From 02334f607fe74e868dae6805ad005e414165302d Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Fri, 20 Nov 2026 17:53:46 +0900 Subject: [PATCH 01/38] fix: a resource script compiles with its package's include_dirs expanded, not as globs A relative include_dirs entry is a glob; the compile steps expand it (package_root.cpp), but the resource compiler received it as written. On Windows LLVM's tools expand wildcards in their own command line against files, so llvm-windres's preprocessor saw several inputs and refused with "cannot specify -o when generating multiple output files". compat.protobuf's protoc host tool failed so on windows-2025, which blocked gRPC's code generation there. The resources e2e case 837 gains R4: a member's `*/inc` reaches the resource compiler expanded. --- CHANGELOG.md | 13 +++++++++++++ src/build/prepare/windows_resources.cpp | 21 +++++++++++++++++++-- tests/e2e/windows/resources.cpp | 16 +++++++++++++--- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7282ec32..bccb78669 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ > Each `## []` section is that release's notes. Entries are written in English > from 2026.9.28.3 on; earlier entries remain as written. +## [Unreleased] + +### Fixed + +- A program's resource script is compiled with its package's `include_dirs` expanded, as the + compile steps expand them: a relative entry is a glob, and `*/include` names the directories it + matches. It reached `llvm-windres` as written, and LLVM's tools on Windows expand wildcards in + their own command line against files, so the preprocessor saw several inputs and refused + ("cannot specify -o when generating multiple output files"); on other hosts the literal `*` + found nothing. Every program of a package whose `include_dirs` name an archive's wrap layer + failed so on Windows, among them compat.protobuf's `protoc` host tool, which gRPC's code + generation builds. + ## [2026.11.20.1] - 2026-11-20 ### Added diff --git a/src/build/prepare/windows_resources.cpp b/src/build/prepare/windows_resources.cpp index c6ad7d931..84cc17862 100644 --- a/src/build/prepare/windows_resources.cpp +++ b/src/build/prepare/windows_resources.cpp @@ -15,6 +15,7 @@ import mcpp.home; import nlohmann.json; import mcpp.log; import mcpp.manifest; +import mcpp.modgraph.scanner; import mcpp.source_kind; import mcpp.toolchain.clang; import mcpp.toolchain.hostflags; // the compile-token producer the package std module reuses @@ -386,9 +387,25 @@ std::expected step13_windows_resources(PrepareState& state, B // Where the resource compiler looks for a script's includes and // files: the package's directory, then its include_dirs. The // scan resolves them the same way. + // + // A relative entry is a glob, expanded as the compile steps + // expand it (package_root.cpp): `*/include` names the + // directories it matches. Passed as written, the `*` reached + // llvm-windres, and LLVM's tools on Windows expand wildcards + // in their own command line -- against FILES -- so the + // preprocessor saw several inputs and refused ("cannot specify + // -o when generating multiple output files"). Every program of + // an index package with a wrap-layer `*/` include failed so + // (compat.protobuf's protoc host tool). std::vector rcIncludes{S.dir}; - for (auto const& d : M.buildConfig.includeDirs) - rcIncludes.push_back(d.is_absolute() ? d : (S.dir / d)); + for (auto const& d : M.buildConfig.includeDirs) { + if (d.is_absolute()) { + rcIncludes.push_back(d); + continue; + } + for (auto& dir : mcpp::modgraph::expand_dir_glob(S.dir, d.generic_string())) + rcIncludes.push_back(dir); + } std::error_code mkEc; std::filesystem::create_directories(resDir, mkEc); diff --git a/tests/e2e/windows/resources.cpp b/tests/e2e/windows/resources.cpp index 430d924e9..e2f82f883 100644 --- a/tests/e2e/windows/resources.cpp +++ b/tests/e2e/windows/resources.cpp @@ -339,14 +339,16 @@ E2E resources{"resources", [] { "837 a members resources reach its image"_test[s::mingw_cross] = [] { // R1 each member's program carries its own icon and not the other's; R2 the synthesised // script of a member is under res// with that member's version and metadata; R3 a - // member's script compiles with that member's include_dirs. + // member's script compiles with that member's include_dirs; R4 a glob among them (an + // archive's wrap layer, `*/inc`) reaches the resource compiler expanded, never as `*`. s::project ws{{"mcpp.toml", "[workspace]\nmembers = [\"cli\", \"gui\"]\n"}, {"cli/assets/app.ico", icon(icon_a)}, {"cli/src/main.cpp", "int main() { return 0; }\n"}, {"cli/mcpp.toml", "[package]\nname = \"cli\"\nversion = \"1.2.3\"\nauthors = [\"Cli Corp\"]\n\n[resources]\nicon = \"assets/app.ico\"\n\n[targets.cli]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}, {"gui/assets/app.ico", icon(icon_b)}, {"gui/src/main.cpp", "int main() { return 0; }\n"}, {"gui/include/gui_ids.h", "#define GUI_ICON 101\n"}, - {"gui/gui.rc", "#include \"gui_ids.h\"\nGUI_ICON ICON \"assets/app.ico\"\n"}, - {"gui/mcpp.toml", "[package]\nname = \"gui\"\nversion = \"4.5.6\"\n\n[build]\ninclude_dirs = [\"include\"]\n\n[resources]\nfiles = [\"gui.rc\"]\nextra-inputs = [\"assets/app.ico\"]\n\n[targets.gui]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}}; + {"gui/vendor-1.0/inc/vendor_ids.h", "#define VENDOR_ICON 102\n"}, + {"gui/gui.rc", "#include \"gui_ids.h\"\n#include \"vendor_ids.h\"\nGUI_ICON ICON \"assets/app.ico\"\nVENDOR_ICON ICON \"assets/app.ico\"\n"}, + {"gui/mcpp.toml", "[package]\nname = \"gui\"\nversion = \"4.5.6\"\n\n[build]\ninclude_dirs = [\"include\", \"*/inc\"]\n\n[resources]\nfiles = [\"gui.rc\"]\nextra-inputs = [\"assets/app.ico\"]\n\n[targets.gui]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}}; auto r = ws.mcpp({"build", "--workspace", "--target", "x86_64-windows-gnu"}); expect(fatal(e2e::succeeded(r))) << "workspace build: " << s::text(r); const auto cli = s::glob(ws.at("target"), "*/*/bin/cli/cli.exe"); @@ -366,6 +368,14 @@ E2E resources{"resources", [] { if (std::regex_search(lines[i], std::regex(R"(rc_object .*gui\.rc)"))) for (std::size_t j = i; j < lines.size() && j <= i + 2; ++j) with_includes = with_includes || lines[j].contains("gui/include"); expect(fatal(with_includes)) << "R3 gui.rc is not compiled with gui's include_dirs"; + bool expanded = false, literal = false; + for (std::size_t i = 0; i < lines.size(); ++i) + if (std::regex_search(lines[i], std::regex(R"(rc_object .*gui\.rc)"))) + for (std::size_t j = i; j < lines.size() && j <= i + 2; ++j) { + expanded = expanded || lines[j].contains("vendor-1.0/inc") || lines[j].contains("vendor-1.0\\inc"); + literal = literal || lines[j].contains("*/inc") || lines[j].contains("*\\inc"); + } + expect(fatal(expanded && !literal)) << "R4 gui's `*/inc` does not reach the resource compiler expanded"; }; "922 dependency images keep their resources"_test[s::windows][s::python3] = [] { From e941e6cf3e20f44333ed9ac3dc72d976135870b0 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Fri, 20 Nov 2026 20:35:06 +0900 Subject: [PATCH 02/38] fix: the resource include list is a function of its own (step13_windows_resources stays under the 400-line limit) --- src/build/prepare/windows_resources.cpp | 48 +++++++++++++------------ 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/src/build/prepare/windows_resources.cpp b/src/build/prepare/windows_resources.cpp index 84cc17862..261d4b119 100644 --- a/src/build/prepare/windows_resources.cpp +++ b/src/build/prepare/windows_resources.cpp @@ -244,6 +244,30 @@ static bool target_takes_code_page(const mcpp::manifest::Target& t) return t.is_program() || t.kind == mcpp::manifest::Target::TestBinary; } +// Where the resource compiler looks for a script's includes and files: the +// package's directory, then its include_dirs. A relative entry is a glob, +// expanded as the compile steps expand it (package_root.cpp): `*/include` +// names the directories it matches. Passed as written, the `*` reached +// llvm-windres, and LLVM's tools on Windows expand wildcards in their own +// command line -- against FILES -- so the preprocessor saw several inputs and +// refused ("cannot specify -o when generating multiple output files"). Every +// program of an index package with a wrap-layer `*/` include failed so +// (compat.protobuf's protoc host tool). +static std::vector +resource_include_dirs(const std::filesystem::path& packageDir, + const std::vector& includeDirs) { + std::vector dirs{packageDir}; + for (auto const& d : includeDirs) { + if (d.is_absolute()) { + dirs.push_back(d); + continue; + } + for (auto& dir : mcpp::modgraph::expand_dir_glob(packageDir, d.generic_string())) + dirs.push_back(dir); + } + return dirs; +} + // Whether an image carries an application manifest of mcpp's: a UTF-8 code // page or long paths asked for. static bool target_manifested(const mcpp::manifest::Target& t, @@ -384,28 +408,8 @@ std::expected step13_windows_resources(PrepareState& state, B const auto resRel = rootSubject ? std::filesystem::path("res") : std::filesystem::path("res") / S.owner; const auto resDir = ctx.plan.outputDir / resRel; - // Where the resource compiler looks for a script's includes and - // files: the package's directory, then its include_dirs. The - // scan resolves them the same way. - // - // A relative entry is a glob, expanded as the compile steps - // expand it (package_root.cpp): `*/include` names the - // directories it matches. Passed as written, the `*` reached - // llvm-windres, and LLVM's tools on Windows expand wildcards - // in their own command line -- against FILES -- so the - // preprocessor saw several inputs and refused ("cannot specify - // -o when generating multiple output files"). Every program of - // an index package with a wrap-layer `*/` include failed so - // (compat.protobuf's protoc host tool). - std::vector rcIncludes{S.dir}; - for (auto const& d : M.buildConfig.includeDirs) { - if (d.is_absolute()) { - rcIncludes.push_back(d); - continue; - } - for (auto& dir : mcpp::modgraph::expand_dir_glob(S.dir, d.generic_string())) - rcIncludes.push_back(dir); - } + // The scan resolves the script's includes the same way. + std::vector rcIncludes = resource_include_dirs(S.dir, M.buildConfig.includeDirs); std::error_code mkEc; std::filesystem::create_directories(resDir, mkEc); From 94e8193a9cfd69a39586c3084876d901c296cf34 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 00:32:06 +0900 Subject: [PATCH 03/38] docs: the test and CI design for mtest, mcpp's test system and CI (mcpp#986, #976, #927, #802) --- ...-11-20-test-architecture-upgrade-design.md | 812 ++++++++++++++++++ .agents/docs/README.md | 7 +- 2 files changed, 818 insertions(+), 1 deletion(-) create mode 100644 .agents/docs/2026-11-20-test-architecture-upgrade-design.md diff --git a/.agents/docs/2026-11-20-test-architecture-upgrade-design.md b/.agents/docs/2026-11-20-test-architecture-upgrade-design.md new file mode 100644 index 000000000..ec24f3075 --- /dev/null +++ b/.agents/docs/2026-11-20-test-architecture-upgrade-design.md @@ -0,0 +1,812 @@ +--- +subject: test +status: active +--- + +# 生态的测试与 CI 方案:mtest 的框架与运行控制、mcpp 自身的测试系统、CI 的形态(mcpp#986、#976、#927、#802) + +**日期:** 2026-11-20 · **版本:** v4(未实施。负责人 2026-11-20 确认 N1 至 N6,N7 不发出,确定本文为生态级的测试与 CI 方案(1.1),确认评审中补充的 5.16 至 5.22;2026-11-21 确认 N8 至 N10,5.19 不每夜运行,5.22 按树认定而不以合并队列为前提,mcpp 的拉取请求复用 #984) · **基线:** mcpp 2026.11.20.1(`722ec1b6`)、mtest 0.10.0、 +compat.doctest 2.4.12、mcpplibs/boost-m 拉取请求 5(`e7cc105c`) · **前文:** +[CI 与发布的时间成本](2026-11-14-ci-and-release-time-cost-design.md)、 +[生态规则收敛方案](2026-11-19-ecosystem-rule-convergence-design.md)(第 11 节,包的实例)、 +[大型工作区的构建空间与复用](2026-10-31-workspace-space-and-reuse-design.md) · **规范:** SPEC-017(`docs/specs/test.md`) + +## 1. 来源、范围与结论 + +本文处理测试与 CI 的成本。来源有五类:四个 issue、一次 CI 探针、一个下游仓库的 CI、一组框架对照测量,以及对已有机制的 +清点。每一项先核对 `origin/main` 的实现,再判定是否纳入。 + +| 来源 | 判定 | 性质 | 章节 | +|---|---|---|---| +| mcpp#986 | 纳入,范围大于报告 | 包实例的摘要含单元集合;同一类问题另有两处(缓存键含工程路径、含 home 的绝对路径) | 5.1、5.2 | +| mcpp#976 · `trees`、`sets` 超出时限 | 纳入 | 三个因素相乘:机器类别、嵌套并行的争用、平台运行时在每个工程中重新编译 | 3.1、5.2、5.5 | +| mcpp#976 · 选择一个套件的用例时不带套件的时限 | 纳入 | 时限陈述在两处,选择只带其中一处 | 5.8 | +| mcpp#976 · `toolchain/custom` 875 在共享 home 下受干扰 | 纳入 | 共享 home 是未声明的运行夹具 | 5.10 | +| mcpp#976 · 骨架的 mtest 版本 | 纳入 | 版本一致性的检查不含骨架 | 5.14 | +| mcpp#976 · `mingw/182` 的下载用时 | 不纳入 | 下载源的选择属于 xlings;本文只使其成为夹具的一部分(5.10),用例不再各自下载 | — | +| mcpp#927 · 第 2 项(耗时汇总有无名的一段) | 纳入 | 构建在运行记录中是一个整体 | 5.6 | +| mcpp#927 · 第 5 项(升级工具后整个配置重编) | 纳入 | 根工程的产物以修改时间与引擎指纹判定,不以内容寻址 | 5.3 | +| mcpp#927 · 第 4 项(测试程序占配置体积七成) | 纳入 | 程序的大小主要是依赖的调试信息 | 5.15 | +| mcpp#927 · 第 1 项(`programs = "directory"` 对 `tests/` 根无效) | 已有 | `programs = "package"`(2026.11.13.1)与「留下直接位于固定目录的文件时说明其数目」已实现(SPEC-017 B1.2) | — | +| mcpp#802 | 纳入 | 内存准入与处理器预算是同一个准入点 | 5.5 | +| mcpp#293 | 纳入 | 测试的 home 以符号链接指向开发者的载荷 | 5.10 | +| 拉取请求 984(本轮复用它) | 核对后补一处 | 资源脚本的包含目录另有一份推导;修复使其展开 glob,但丢掉了规划时尚不存在的字面目录 | 5.1 | +| 探针(拉取请求 985,三轮) | 纳入 | 见 3.1 | 5.2、5.5、5.9 | +| boost-m 的 CI | 纳入基础设施的部分 | 见 3.3;其自身的用法(两组各测一遍、每个文件一份 doctest 实现)由其维护者决定 | 5.6、5.11、5.13、7 | +| mtest 与 doctest 的对照 | 纳入 | 见 3.4 | 5.12、5.15 | +| 评审中的补充(负责人 2026-11-20 确认) | 纳入 | 面的覆盖、预算的门禁、用例结束时的检查、抖动、等待事件、契约性输出、一个提交验证一次 | 5.16 至 5.22 | + +**结论。** + +1. 测试与 CI 的时间主要花在构建上,而测试系统只在用例的粒度上看得见它。mcpp 的 C++ e2e 分片把 2,579 至 7,255 s 的边 + 用时花在平台运行时的编译上;boost-m 的 `mcpp test` 一步有 95% 是构建。 +2. 构建被重复的原因是产物的地址含有不决定其内容的量。已核实三处:包实例的摘要含单元集合(#986);缓存键含构建程序的 + 输出目录,该目录在工程之内;缓存键含 home 的绝对路径。另有一处是缺少寻址:根工程的单元只按修改时间判定。 +3. 嵌套的并行没有共同的预算:四个测试进程各自启动并行构建,4 个处理器上平均 6.8 至 9.4 个进程排队。同一工作的 + 墙钟因此为单独运行的 2.3 倍,时限按墙钟陈述,于是失败。 +4. GitHub 的同一种 runner 是三种处理器,同一组命令的处理器时间相差 1.62 倍。按墙钟陈述的时限与估计在机器之间不可比。 +5. mtest 相对 doctest:断言多的文件编译更快(Clang 上 2 至 4.5 倍),每个文件的首次使用成本在 GCC 上高一个数量级 + (1.1 s 对 0.07 s),链接更慢、程序更大,在协议下每个用例多约 2 ms。四项各有可定位的成因(5.12)。 +6. 一个提交被验证至少两次,而被发布的产物没有被完整验证:CI 测试宿主构建,发布另行构建静态包(5.22)。 +7. 有三类行为今天的测试看不见:没有被任何测试用到的选项与键、通过的用例中未声明的降级提示、没有被固定的次序 + (5.16、5.18、5.19)。 +8. 此前已实施的机制(一次构建、分片、按测量调度、main 的测试构建复用、脚本 e2e 的覆盖图)保留;本文不重复提出, + 并记录其中被测量否定的两项(第 2 节)。 + +**不在范围:** 测试的书写形式(对象名与标识的重复);另行讨论。 + +### 1.1 三个部分 + +本文是生态级的方案:mtest 是各仓库共用的框架,mcpp 的测试系统是它最大的使用者,CI 的形态由二者的能力决定。 + +| 部分 | 内容 | 决定 | +|---|---|---| +| mtest:功能、框架、运行时的控制与配置、数据的粒度 | 协议的步骤、用到的面、构建记录、预算与抖动;机器与用量;时限与预算的门禁;运行夹具;账本;用例结束时的检查;golden;适配器;库自身的成本 | 5.5 至 5.10 的协议与引擎一侧,5.12 至 5.14,5.16 至 5.19,5.21 | +| mcpp 自身的测试系统 | 产物的身份与复用;运行器实现上述协议;测试 kit;按变更选择;测试的规则;以自身的构建与测试为基准 | 5.1 至 5.4,5.5 至 5.11 的运行器一侧,5.15,5.20,第 6 节 | +| CI 的形态 | 档位与树的标记、发布的提升、缓存的写入与传输、分片与预算、夹具的恢复、记录与摘要、下游仓库共用的动作 | 5.22,第 7 节 | + +实施的任务与依赖在第 12 节,自评在第 13 节。 + +## 2. 现状清点 + +下表是 `origin/main` 已有、本文以其为前提的机制。 + +| 机制 | 位置 | 状态 | +|---|---|---| +| 被测程序每个宿主构建一次,各作业取用 | `ci.yml`、`build.yml`、`use-built-mcpp` | 已实施(2026-10-02) | +| 测试构建一次,四个分片运行;按存储测得的用时排序、划分、分片;只重跑失败部分 | `ci-unit.yml`、SPEC-017 O10 | 已实施(2026-11-14) | +| main 的测试构建目录经 artifact 复用,文件修改时间按 blob 恢复(W1) | `mcpp-unit`、`source_mtimes.py` | 已实施(2026-11-15) | +| CI 的测试 profile(`-O0`、无调试信息) | `[profile.ci]` | 已实施 | +| 脚本 e2e 由 mtest 引擎运行;按宿主的历史分片(`mtest ci plan`);每夜以插桩的 mcpp 测得覆盖图,拉取请求按它选择 | `tests/e2e/mtest.toml`、`e2e-coverage.yml`、`e2e-mtest` | 已实施(2026-11-08) | +| 按变更选择:模块导入图、`MCPP_BIN_` 规则、夹具文件 | `mcpp test --since`(docs/08) | 已实施(2026.11.8.1、2026.11.13.1) | +| 运行存储(程序、用例、夹具;`machine`、`monitor` 字段) | SPEC-017 B4 | 已实施;mcpp 写 `machine = ""`、`monitor = "off"` | +| 用量(P10)、运行夹具(P11)、条款矩阵(P6.4)、界限(P6.5) | SPEC-017 O4、O6、O7 | 框架与 mtest 引擎已实现;`mcpp test` 不服务运行夹具,不陈述监测程度 | +| 每次构建一个编译器的 generation,执行器(`__compiler-cap`)从中取槽位 | `detach_codegen.cppm`、`ninja_backend.cppm` | 已实施(2026.11.7.1) | +| 依赖包的全局构建缓存,以包为单位 | `dependency_cache.cpp`、`cache_key.cppm` | 已实施;根工程不进入 | +| 构建引擎的对照基准(mcpp、cmake、xmake) | `bench/`、`bench.yml`(手动) | 已实施 | +| 集合的预算与 `mtest gate`;golden 文件(`MTEST_GOLDEN=update`) | mtest | 已有;没有单个用例的预算、脱敏与更新的摘要;mcpp 的测试没有使用 golden | +| 运行的档位:拉取请求为 `pr`,main 与 `ci:full` 为 `full` | `ci.yml` 的 `changes` | 已实施;只改变 e2e 的集合与一个 macOS 镜像的子集,作业不减少 | + +**此前被测量否定、本文不再提出的两项。** + +- **按文件粒度的覆盖图为 C++ e2e 收窄选择。** 2026-11-14 的记录第 4 节:覆盖图中 `src/` 的文件被执行它的用例覆盖的比例 + 中位数为 99%。本文只把更细的粒度列为一次测量(5.11)。 +- **无条件地按声明的用例划分长套件。** 同一记录 6.3:`sets` 划分为三组后各为 909、1109、1306 s(整体 1375 s),原因是 + 各组重新付出第一个用例填充的共享状态。该记录的结论是缩短这两个套件的途径为共享状态的复用;本文的 5.2 与 5.10 + 即这条途径。 + +## 3. 测量 + +### 3.1 CI 探针(拉取请求 985,2026-11-20,三轮;数据在草稿 release `probe-data`) + +对象是 `e2e/openkal/trees` 的用例 955,在分片中与单独运行两种条件下,各两遍。 + +| 项 | 测量 | +|---|---| +| runner 的处理器 | `ubuntu-24.04`(4 vCPU)出现三种:AMD EPYC 7763、Intel Xeon Platinum 8573C、Intel Xeon 6975P-C | +| 同一组 973 条命令的处理器时间 | EPYC 为 6975P-C 的 1.62 倍(中位数;p10 1.42,p90 1.79);8573C 比 EPYC 快约 15% | +| 分片内的争用 | 四个工作进程各自并行构建;运行队列平均 6.8 至 9.4,峰值 18 | +| 用例 955 的步骤 A(第一次构建) | 单独运行 206 s(EPYC,冷);分片内 478 s(EPYC):墙钟 2.3 倍,处理器时间多 17% | +| 同一 home、新工程的第二遍(单独运行) | 步骤 A 115 s 与 100 s;重新编译的恰是 `x86_64-openkal`、`aarch64-openkal`、`x86_64-windows-gnu` 三个配置。步骤 D(全部命中)7 s | +| 每个配置重新编译的单元 | 约 1,350 个,另加 400 至 500 个(`openkal-musl` 或 `openkal-windows`,与其上的 `llvm-runtime`) | +| 本机的缓存 | `openkal-musl@0.30.0` 的 97 个条目,把输出目录归一后合为 27 个键 | +| 一个分片中平台运行时的边用时 | 2,579 至 7,255 s | + +归档运行的选择另有三个缺陷,均在探针中触发:部分的模式(`e2e/openkal/trees`)在归档运行中选不中任何测试 +(`run_test_archive` 只按程序过滤);`--case 'trees/955*'` 运行整个 `trees`(`selection()` 对已列出用例之下的通配); +`--cases --archive` 以 `no test ... ran on this machine` 结束。 + +### 3.2 mcpp#986 + +报告的最小复现在 2026.11.20.1 上成立。`plan.cppm` 以 `cu.object.generic_string() + flags_text(cu)` 逐单元累加摘要, +设计记录 2026-11-19 第 11.1 节陈述的摘要不含单元集合。在 boost-m 的一条腿上(linux、GCC 16.1.0、A 组),测试步骤的 +进度在 3 分 14 秒到达 1068/1086,重复编译的上限即这一段,约占该步骤 1202 s 的六分之一。 + +### 3.3 boost-m 的 CI(拉取请求 5 的四次运行) + +416 个测试程序,全部使用 doctest,每个文件一个程序,各定义 `DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN`。五个宿主各分 A、B +两组,共十条腿,每条腿运行全部 416 个程序。 + +| 项 | 测量 | +|---|---| +| 整次运行墙钟 | 48、53、65、75 分钟 | +| 最慢的腿(44726358869) | linux GCC B 组 45.5 分钟,其中 `mcpp test` 2507 s | +| 测试步骤的构成(linux GCC A 组) | 1202 s = 规划 7 + 构建 1142 + 运行 53 | +| 构建的长尾 | A 组在 3:14 到达 1068/1086,其余 18 条边约 16 分钟;B 组最后 11 条边约 34 分钟;linux LLVM A 组最后 1 条边约 10 分钟 | +| 这段时间的输出 | 无:日志在 16 至 34 分钟内没有一行,CI 不保留构建记录 | +| 同一条腿在四次运行之间 | x86_64 各腿的同一步骤相差 1.4 至 1.9 倍;linux arm64 相差 1.03 至 1.14 倍(四次为同一拉取请求的不同提交) | +| 子工程步骤(`tests/dll`、`tests/test/utf`) | GCC 腿 7 至 21 s;LLVM、macOS、Windows 腿 292 至 618 s | + +长尾对应哪些边没有直接证据。测试文件的大小中位数为 4 KB,超过 100 KB 的有 13 个(最大 500 KB,为合并移植的官方 +示例),与剩余边数相符;这是推断。 + +### 3.4 mtest 0.10.0 与 doctest 2.4.12 + +方法取自 doctest 的 `doc/markdown/benchmarks.md`:三个场景,单线程构建,以工程自己的目标文件的边用时之和为编译成本。 +本机 x86_64,GCC 16.1.0 与 Clang 23.1.3(libc++),`dev`(`-O0 -g`),mcpp 2026.11.20.1;生成脚本与结果在探针的 +工作目录,不入库。doctest 的文件以 `#include` 取得头文件,mtest 的文件以 `import mcpplibs.mtest;`。 + +**场景一:200 个各含一个函数的文件。** + +| 每个文件增加的编译时间 | GCC doctest | GCC mtest | Clang doctest | Clang mtest | +|---|---|---|---|---| +| 只包含或导入 | 39 ms | 26 ms | 256 ms | 4 ms | +| 另声明一个用例、写一条断言 | 71 ms | 1,111 ms | 251 ms | 270 ms | +| 运行器的实现(一次) | 2.3 s | 无(库的编译 9 至 16 s,见下) | 1.2 s | 无(库的编译 4 至 11 s) | + +**场景二:10 个文件,各 50 个用例,各 100 条断言(共 5 万条)。** + +| 编译时间(s,减去基线前) | GCC | Clang | +|---|---|---| +| doctest `CHECK(a == b)` | 40.7 | 33.7 | +| doctest `CHECK(a == b)`,`SUPER_FAST_ASSERTS` | 15.2 | 16.3 | +| doctest `CHECK_EQ(a, b)` | 72.1 | 14.7 | +| doctest `CHECK_EQ(a, b)`,`SUPER_FAST_ASSERTS` | 11.0 | 4.6 | +| mtest `expect(eq(a, b))` | 22.6 | 7.6 | +| mtest `expect(a == b)` | 13.4 | 4.6 | + +**场景三:一个用例中 1000 万条通过的断言(处理器时间,s)。** + +| | GCC | Clang | +|---|---|---| +| doctest `CHECK(i == i)` | 0.67 | 0.79 | +| doctest `INFO(i); CHECK(i == i)` | 0.89 | 2.31 | +| mtest `expect(eq(i, i))` | 2.14 | 2.35 | +| mtest `expect(eq(i, i)) << i` | 2.15 | 2.43 | + +**其余四项。** + +| 项 | doctest | mtest | +|---|---|---| +| 一个测试程序的链接(GCC / Clang) | 0.34 s / 0.06 s | 0.92 s / 0.21 s | +| 一个测试程序的大小(GCC / Clang,`dev`) | 4.1 MB / 1.4 MB | 11.6 MB / 7.2 MB | +| 协议下 510 个用例、5 万条断言的程序 | 不适用(一个程序一个结论) | 独立运行 0.03 s;协议下 1.05 至 1.27 s,其中处理器时间 0.1 s:每个用例建两个文件并 `fsync`(共 4,044 次) | +| 库在每个工程中 | 仅头文件 | 重新编译:`mcpplibs.mtest` 的目标文件单线程 4 至 16 s,其依赖均命中缓存 | + +**`release`(`-O2`,GCC 16.1.0;Clang 未测)。** + +| 项 | doctest | mtest | +|---|---|---| +| 每个文件:只包含或导入 / 另声明一个用例、写一条断言 | 16 ms / 37 ms | 37 ms / 1,600 ms | +| 5 万条断言:缺省形式 | `CHECK` 175.8 s;`CHECK_EQ` 114.6 s | `expect(eq)` 82.5 s;`expect(a == b)` 19.3 s | +| 5 万条断言:doctest 的快速形式 | 48.9 s 与 12.2 s | — | +| 1000 万条通过的断言(处理器时间) | 0.30 s | 0.14 s | +| 库的编译(每个工程,单线程) | 无 | 16 至 33 s | + +**读法。** + +- doctest 的成本集中在实现的那一份:只在一个文件中实现时,其余文件在 GCC 上每个 39 至 71 ms。boost-m 的 416 个文件 + 各实现一次。Clang 上每个文件的 0.25 s 不随断言增加;推断来自 doctest 的头文件包含的标准库头文件与标准库模块并存, + 未单独测量。 +- mtest 的导入几乎不花时间,成本在首次使用:GCC 上声明第一个用例约 0.3 s,第一条 `eq` 再约 0.3 s(单独编译实测), + 在场景一中每个文件 1.1 s。断言本身比 doctest 的缺省宏便宜(GCC 上 22.6 s 对 40.7 s),在 GCC 上仍高于 doctest 的 + 两种快速形式(15.2 s 与 11.0 s)。 +- mtest 的程序更大、链接更慢,来自调试信息:11.5 MB 中调试段 8.5 MB,代码 1.1 MB,其中 `nlohmann.json` 0.13 MB、 + `mcpplibs.os` 与 openkal 0.09 MB(5.15)。 +- mtest 库不由缓存提供:新的构建目录中,2026.11.17.2、2026.11.18.1 与 2026.11.20.1 都重新编译它(mtest 0.9.0 与 + 0.10.0;并行构建约 6 s),也不写入条目;它的依赖 `mcpplibs.os`、`openkal`、`openkal-linux`、`nlohmann.json` 均显示 + `Cached`。缓存中有 `mcpplibs.mtest@0.9.0` 的一个条目,由别的依赖图写入。mcpp 不说明原因。 + +### 3.5 mcpp 的一次 main 运行(44691464513) + +76 个作业,runner 时间 649 分钟。本提交的 mcpp 之外,各作业另以不同的工具链或目标从源码构建 mcpp: +`bootstrap` 1152 s 与 755 s、容器中的 e2e 1408 s、交叉到 `aarch64-linux-musl` 661 s、Windows 宿主交叉到 Linux +2091 s、musl-gcc 657 s、三个 LLVM 自举 310 至 347 s、macOS 自举 389 s。这些构建各自从零开始。四个单元分片中 +三个为 14.5、17.5、27.4 分钟,macOS 的单元步骤 2142 与 2261 s,Windows 1421 s。 + +### 3.6 测试的写法与合入、发布的现状 + +| 项 | 测量 | +|---|---| +| C++ 测试中的 `sleep_for` | 77 处 | +| C++ e2e 中的 `contains(` 与 `e2e::golden` | 2,302 处与 0 处 | +| 一次合入的全量运行 | 至少两次:拉取请求上一次,main 的推送一次 | +| 最近一周每天合入的拉取请求 | 1 至 4 个 | +| CI 测试的程序与发布的程序 | 前者为 `mcpp build --strict` 的宿主构建,后者为 `release.yml` 另行构建的 `x86_64-linux-musl` 静态包 | +| 发布的构建 | 2026.11.20.1 约 30 分钟 | +| main 的规则集(`main-branch-protect`) | 禁止删除与强推、线性历史、签名、拉取请求须两个批准、只允许 squash 与 rebase,两个可绕过的身份;没有必需的状态检查 | + +## 4. 成因 + +| 编号 | 成因 | 表现 | +|---|---|---| +| C1 | 产物的地址含有不决定其内容的量,或缺少由内容决定的地址 | #986;平台运行时在每个工程中重编;键含 home 的绝对路径;mtest 库不进缓存而无提示;每个作业从零构建 mcpp;升级工具后全部重编(#927) | +| C2 | 嵌套的并行没有共同的预算 | 分片内争用使墙钟为 2.3 倍;内存耗尽(#802) | +| C3 | 构建在运行记录中是一个整体 | 16 至 34 分钟无输出;长尾不可定位;汇总有无名的一段(#927);按用时的调度不含构建 | +| C4 | 时限陈述在两处,并以墙钟对所有机器陈述一次 | 套件 40 分钟、kit 的每条命令 10 分钟;选择丢掉套件的时限(#976);同一工作在两类机器上相差 1.6 倍 | +| C5 | 共享状态是隐式的 | 第一个用例付出填充 home 的成本;修改 home 的用例独占机器;875 受干扰;#293 | +| C6 | 框架的固定成本 | mtest 的首次使用、每个用例的 `fsync`;依赖的调试信息进入每个测试程序;doctest 的实现每个文件一份(用法) | +| C7 | 同一个提交被验证多次,被发布的却没有被验证 | 两次全量运行;发布另行构建(3.6) | +| C8 | 测试看不见的行为 | 没有被任何测试用到的选项与键;通过的用例中未声明的降级提示;没有被固定的次序 | + +## 5. 决定 + +每项陈述决定、机制、它取代或关闭的事项,以及验收。验收的数字来自第 3 节;标为「推算」者在实施中以测量替换。 + +### 5.1 包实例的摘要是包的配置的函数(mcpp;关闭 #986) + +**决定。** 一个包的实例由它的配置决定:声明的标志、宏、包含目录、按 glob 的规则、生效的特性、语言标准,以及其依赖的 +实例摘要。实例目录的摘要与缓存键的配置部分由同一个函数产生。单元集合与单元的输出路径不进入。 + +**机制。** `plan.cppm` 中逐单元累加 `cu.object + flags_text(cu)` 的一段改为读取包的配置;`cache_key.cppm` 的 +`fill_package_config` 是这个函数的现有形态,二者合一。按 glob 的规则以规则本身进入,而不是以它匹配到的单元。 + +**后果。** `mcpp build` 与 `mcpp test` 在特性相同时共用一个实例;测试计划以 `[test] features` 打开特性时配置不同, +实例不同,这是应有的区分。模块映射文件的名字含实例目录,随之稳定。`clean --stale` 与 `clean --report` 读取的 +「每个配置一个目录」不变。 + +**包含目录是配置的一部分,只有一份推导(拉取请求 984)。** 一个包展开后的包含目录现有三份推导:依赖包的编译 +(`package_root.cpp`:展开 glob,丢掉不存在的字面目录)、根清单的编译(`plan.cppm` 的 +`expand_manifest_include_entry`:展开 glob,保留不存在的字面目录,因为它可能由稍后的构建步骤建立,#390)、资源脚本 +(`windows_resources.cpp`)。#984 之前第三份不展开 glob,`*/include` 原样到达 `llvm-windres`,在 Windows 上失败; +#984 改为调用 `expand_dir_glob`,方向正确,CI 全部通过。核对代码后有一处:它没有第二份的回退,规划时尚不存在的 +字面目录此前被原样拼接,现在被丢掉,资源脚本包含其中生成的头文件时首次构建找不到、第二次构建找到。这一点来自 +读代码,没有运行复现。本轮把资源脚本改为调用 `expand_manifest_include_entry`,并在用例 837 中增加「稍后建立的字面 +目录仍在 `rc_object` 的边上」;三份推导随配置函数合为一份。 + +**验收。** #986 的最小复现:`mcpp build` 之后 `mcpp test` 只编译 `src/b.cppm` 与 `tests/t.cpp`。SPEC-004 §9 第 17 项 +写明摘要的输入。 + +### 5.2 路径按角色进入键;可缓存的包的构建程序输出在 home 中按身份寻址(mcpp) + +**决定。** 键中的路径以其角色出现:``(每一个存储根)、``、``(构建程序的输出)。绝对路径不进入键。 +来源被命名的包(索引的版本、git 的提交、路径依赖的内容摘要,#974),其构建程序的输出目录位于 +`/build-cache/v1/out/<包>@<版本>/<键>`,键为该包不含输出目录一项的配置键。 + +**成因的细节。** `state.programOutDir` 把输出目录放在 `<工程>/target/.build-mcpp/out//<包>`;构建程序以 +`mcpp:include-dir=` 声明的目录进入包的键,`relativize` 只认识 `` 与 ``。`openkal-musl`(openkal +目标)与 `openkal-windows`(windows 目标)因此每个工程一个键,其上的 `llvm-runtime` 经 `upstream` 一项随之改变。 +另一处:`nlohmann.json` 的包含目录以 `/home/<用户>/.mcpp/registry/data/xpkgs/...` 进入 `mcpplibs.mtest` 的键 +(`mcpp cache info`),`relativize` 只替换第一个存储根。 + +**不采用「在键中抹去路径」。** 构建程序收到 `MCPP_OUT_DIR`,可以把它写进生成的文件;输出的位置因此是内容的一部分。 +把位置固定为身份的函数,内容与地址才一致。输出按 2026-11-19 记录的事务方式发布:两个工程同时运行同一个构建程序时, +先完成者发布,后完成者核对后采用。 + +**验收。** 探针的第二遍:同一 home 的新工程中,三个配置的步骤 A 不编译平台运行时的任何单元。预期用例 955 的步骤 A +由 478 s 降到与步骤 D(7 s)同一量级加打包的时间(推算)。新增 e2e:两个工程依次构建同一 openkal 目标,第二个工程的 +编译边数为它自己的单元数。 + +### 5.3 根工程与未命名来源的单元按内容寻址(mcpp;取代 W1 的修改时间恢复;关闭 #927 第 5 项) + +**决定。** 执行器在运行编译器之前计算这次编译的键,命中时从缓存发布输出,不运行编译器。键的输入是决定输出的全部: +工具链的身份、归一后的命令、源文件的内容、它读取的每个文件的内容、它导入的每个 BMI 的内容。引擎的版本不进入。 + +**机制。** + +- 位置是 `__compiler-cap`:它包住有上限的图中的每一条编译命令(2026.11.7.1 起),持有输出的发布(2026-11-19 记录 + 10.6)。本项使每个图都经由它。 +- 读取的文件在编译之前未知,采用清单的方式:第一级键(命令、源文件内容)指向若干份清单,每份清单是一次编译实际读取 + 的文件及其摘要(来自依赖文件;MSVC 来自 `/sourceDependencies`);清单中的文件都未变时,其结果即命中。 +- 输出为目标文件、BMI、依赖文件与拆分的调试信息。GCC 的 BMI 以模块映射的拼写记录其导入的路径(BMI 目录与包的 + 实例),随 5.1 稳定;Clang 与 MSVC 的 BMI 在验收中各自核对。 +- 源码根的绝对路径进入键,除非工具链的行陈述其输出可重定位。CI 的检出路径不变,命中不受影响;本机的两个工作树之间 + 不共享,这是正确的结果。 +- 条目位于 `/build-cache/v1/unit/`,由 `mcpp cache gc` 按最近使用回收。CI 中只有 main 的推送运行写入(R3,5.22:它登记一次已通过的 + 全量运行的条目,或自己运行全量),拉取请求的运行只读;经 artifact 传输:仓库的 Actions 缓存已用 9.98 GB(2026-11-14 记录 6.1)。 +- ninja 仍决定哪些边要运行;新的检出使全部边变脏,执行器逐条命中并以 `restat` 结束。链接不缓存。 +- 文件的摘要在一次构建中按(路径、大小、修改时间)记忆,一个头文件只计算一次。 +- 修剪按使用:全量运行在上传之前去掉本次没有读取的条目,传输的大小随当前的树,不随历史。 + +**取代。** W1 的 `source_mtimes.py` 与 `.source-blobs`;`bootstrap-mcpp` 恢复 `target/` 后仍运行 830 条边中的 830 条; +第 3.5 节中各作业从零构建 mcpp;工具升级后的全部重编(指纹目录改变,条目命中)。 + +**验收。** 在一个已填充缓存的 home 上,新检出的无改动构建不运行编译器,其用时记入本文(现为 11 分 37 秒);改动一个接口模块时,运行的编译器数为其 +导入闭包的单元数。逐字节核对:同一键的两次冷编译产出相同的目标文件与 BMI(GCC、Clang、MSVC 各自测量;不相同的 +工具链在其行中标为不可缓存,并以 `diag::degraded` 说明)。 + +### 5.4 缓存的每一次拒绝都有原因(mcpp;mtest 库不由缓存提供) + +**决定。** 一个包或一个单元未由缓存提供时,构建记录写出原因标记,`--verbose` 打印它。`serve_or_fill` 现有的五条返回 +路径(无身份、来源未命名、受未命名来源影响、不在存储之内、有不可寻址的单元)各一个标记。 + +**mtest 库。** `mcpplibs.mtest` 作为根工程的 dev 依赖不由缓存提供,也不被写入(3.4)。成因未定:它落在上述五条路径 +之一,而这五条路径今天不可区分,这正是本项要改变的。成因在实施中以原因标记确定,并按成因修复。每个使用 mtest 的 +工程、每个 e2e 用例建立的工程都付出这次编译。 + +**验收。** 新工程第二次构建时 `mcpplibs.mtest` 显示为 `Cached`;新增 e2e 断言 dev 依赖由缓存提供。 + +### 5.5 一台机器一份预算:处理器槽位与内存(mcpp;SPEC-017 新增 O11;关闭 #802) + +**决定。** 运行测试的 mcpp 持有机器的预算。它的工作进程与这些进程启动的每一次构建从同一份预算取槽位。预算另带内存的 +阈值:可用内存低于阈值时不发放新的槽位,已运行的编译器不被结束。 + +**机制。** + +- 预算是一个目录,其中有与槽位数相同的文件;持有一个文件的锁即持有一个槽位。运行器建立它并在环境中陈述其路径 + (SPEC-017 O11 `budget`)。任何程序都能遵守:`mcpplibs.os` 的文件锁,或脚本中的 `flock`。不认识它的程序不受影响。 +- 一个测试进程持有一个槽位;它启动的构建以这个槽位为第一个编译器,其余向预算申请。不构建的测试以自己的槽位运行。 +- mcpp 的执行器在环境陈述了预算时,启动编译器之前先取预算的槽位,再取本次构建的 generation 的槽位;没有陈述时 + 只取后者(现状)。 +- 内存的准入在同一处:执行器在取得槽位后、启动编译器前读取可用内存(`mcpplibs.os.machine`)。阈值是数据 + (`[build] memory_threshold`,四层取值)。`bmi_schedule = "on"` 的分离代码生成经同一执行器,因此被覆盖。 +- mtest 引擎运行脚本 e2e 时同样建立并陈述预算。 + +**验收。** 探针的分片条件重跑:运行队列的平均值不超过处理器数加一;用例 955 的步骤 A 在分片内的墙钟不超过单独运行的 +1.3 倍(现为 2.3 倍)。分片的总墙钟不增加。 + +### 5.6 构建是运行图的节点(mcpp、SPEC-017 B4;关闭 #927 第 2 项) + +**决定。** 运行存储新增 `runs//builds.ndjson`:每条运行过的边一行(输出、所属的包与测试程序、开始、用时、 +处理器时间、最大驻留集);由缓存提供的边按包汇总为一行。`mcpp test` 与 `mcpp build` 的汇总行列出各阶段,并列出最慢的若干条边。 +一条边运行超过一分钟而无输出时,进度行写出正在运行的边及其已用时间。 + +**机制。** ninja 的日志已有每条边的开始与结束,`progress::Attribution` 已把边归属到包;处理器时间与最大驻留集由 +执行器在回收编译器时取得。本项把它们写出。 +P12.3 规定的关键路径由此可以下到边。按测量的调度(O10)在排序测试程序时计入其构建的用时。 + +**验收。** boost-m 的一条腿重跑后,存储回答第 3.3 节的问题:长尾是哪些边、各用多久。`mcpp test -p` 的汇总各段之和 +与总时间之差小于 1 s。 + +### 5.7 步骤是用例的一部分(SPEC-017 P5、mtest、测试 kit) + +**决定。** 协议新增 `step` 事件:属于一个用例,带名字、开始、用时、处理器时间、退出状态。测试 kit 为它运行的每条 +mcpp 命令写一个步骤;该命令是一次构建时,步骤带上那次构建的记录标识。 + +**后果。** 用例 955 的四个步骤各自可见;`mtest analyze path` 从运行下到用例、步骤、边。探针三轮得到的分解成为一次查询。 + +### 5.8 时限只陈述一次,按处理器时间比较(mcpp、mtest;关闭 #976 的一项) + +**决定。** + +- 用例或套件声明的 `timeout` 是唯一的时限。选择一个套件之下的用例时,运行器取它所属的各级作用域声明的时限 + (`--case 'sets/*'` 与 `--failed` 现取运行的缺省 300 s 或 `--timeout`)。 +- kit 的每条命令缺省 10 分钟的时限改为取用例剩余的时间。 +- 时限防止挂起。性能以指标判定:每个用例的处理器时间进入记录(5.9),界限与趋势按它陈述(P6.5),不按墙钟。 +- 报告列出余量:最近若干次运行中用时超过时限五分之四的用例,连同机器的类别。 + +**验收。** #976 的重现:`--case 'sets/*'` 在套件的 30 分钟内运行。 + +### 5.9 机器与用量进入记录(mcpp) + +**决定。** `run.json` 的 `machine` 写出处理器型号、核数、内存(型号取自 `mcpplibs.os.machine` 新增的接口,任务 O1;没有回答的系统上机器的类别取架构与核数);`mcpp test` 缺省以 `basic` 陈述监测程度,并按 P10.4 +从自己的进程测量不写 `usage` 的程序。按测量的调度与分片在估计用时时按机器的类别取历史;没有该类别的历史时取全部。 + +**验收。** 同一测试在两类机器上的估计分开;3.1 的 1.62 倍在报告中表现为类别的差异,而不是回归。 + +### 5.10 共享的 home 是运行夹具(mcpp 实现 O6;测试 kit;关闭 #293,处理 #976 的 875) + +**决定。** `mcpp test` 实现运行夹具(P11):以 `MTEST_SERVE` 服务,把端点交给使用它的程序。测试 kit 把现在隐式共享的 +home 声明为运行夹具:其内容(工具链、平台包、索引)是声明,摘要是它的身份。 + +- 夹具的准备用时记在夹具上(`fixtures.ndjson`),不记在碰巧第一个运行的用例上。 +- 用例陈述它对 home 的使用:只读,或修改。只读的用例共用一个;修改的用例得到自己的 home,其载荷以只读的存储根 + 提供,而不是指向共享载荷的符号链接。`home::shared` 之下以 `machine` 独占来避免干扰的用例改为后者。 +- CI 以夹具的身份为键恢复它;5.2 使其中的构建缓存对每个工程有效。 + +**验收。** 875 与并行的用例同时运行而不受干扰;`sets`、`trees` 的第一个用例不含填充的时间;#293 的复现不改写 +开发者的载荷。 + +### 5.11 按变更选择的两处补充(mcpp) + +- **头文件。** `--since` 现把任何头文件的变更视为影响全部测试。改为:构建目录或存储中有基线提交的依赖记录时, + 头文件影响依赖记录列出它的单元及其导入者;没有记录时保持现状。记录是 5.3 的清单(每个单元实际读取的文件), + 随 main 登记的单元缓存到达拉取请求。对头文件构成的库(boost-m)这是按变更选择能否生效的条件。 +- **C++ e2e。** 文件粒度已被否定(第 2 节)。本轮测量函数粒度:覆盖图记录每个用例执行的函数,统计一个函数被多少 + 用例执行。中位数低于 50% 时实施(变更的函数与用例执行的函数相交),否则把测量记入本文并不实施。 + +### 5.12 mtest 库的四项成本(mtest;其中一项归 mcpp) + +| 项 | 成因 | 决定 | 验收(第 3.4 节的方法) | +|---|---|---|---| +| 每个文件的首次使用(GCC 1.1 s) | 接口中的模板与标准库类型在每个导入者中实例化 | 用例的声明与 `expect` 的常见形式(整数、字符串、布尔)在库中有非模板的入口;接口不要求导入者实例化 `std::function`、流与格式化 | GCC 上每个文件不超过 150 ms | +| 链接与程序大小 | 依赖与运行时库的调试信息(8.5 MB 中的大部分);JSON 库的代码只有 0.13 MB | 不在库中处理:库继续依赖 JSON 库(mtest 的规则:已有的库被依赖,不重写);由 5.15 处理 | 见 5.15 | +| 协议下每个用例约 2 ms | 每个用例为输出归属建两个文件并 `fsync` | 输出经由一个按用例标记的通道;不 `fsync`(进程崩溃不丢失已写入的数据) | 510 个用例的程序在协议下不超过 0.1 s | +| 每个工程重新编译库 | mcpp 不由缓存提供它(5.4) | — | — | + +断言的运行成本(每条 0.2 µs)不处理。 + +### 5.13 需求、用例与理由的账本(mtest) + +**决定。** `mtest.toml` 的 `[ledger.<名>]` 陈述一组需求的来源(一个文件,或一条产生列表的命令)。用例以 `clause{}` +陈述它覆盖的需求。每条需求必须有通过的用例,或一条带理由的豁免;豁免的数目只减不增。`mtest gate` 判定, +`mtest report` 生成表。 + +需求由产品给出、证据取自记录的形式见 5.16。 + +**取代三份手工维护的账本。** mcpp 的 `tests/e2e/coverage-debt.tsv` 与 `coverage-exceptions.tsv`;`docs/specs/` 中 +每条条款的实现状态;boost-m 的 `coverage.toml` 与 508 行的 `test_coverage.py`。 + +### 5.14 其余 + +- **doctest 适配器(mtest)。** 与 GoogleTest 适配器同级(1 级):一个 doctest 的报告器与入口,使现有的 doctest 程序 + 列出、选择并报告用例。测试文件去掉 `DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN` 一行,其余不变:实现与入口由适配器提供 + 一次,每个文件各实现一次的成本(GCC 上约 1.6 s)随之消失。 +- **归档运行的选择(mcpp)。** 3.1 的三个缺陷:归档运行接受部分的模式;已列出用例之下的通配选中匹配的用例; + `--cases --archive` 列出而不报错。 +- **骨架的 mtest 版本(mcpp)。** `kMtestRelease` 进入 `check_version_pins.sh`。 + +### 5.15 依赖的调试信息是 profile 的数据(mcpp;mcpp#927 第 4 项) + +**测量。** 一个 mtest 的测试程序(GCC、`dev`)11.5 MB,其中调试段 8.5 MB(`.debug_info` 6.0),代码 1.1 MB。按符号归属的代码: +标准库与运行时 0.74 MB,mtest 0.14 MB,`nlohmann.json` 0.13 MB,`mcpplibs.os` 与 openkal 0.09 MB。链接已带 +`--gc-sections`。程序的大小与链接用时因此来自依赖与运行时库的调试信息,不来自 JSON 库的代码。#927 的工作区中, +349 个测试程序占 22 GB 中的 15.4 GB,去掉调试信息并剥离后一个程序由 24.5 MB 降为 4.3 MB。 + +**决定。** profile 按包的类别陈述调试信息的程度:工作区的成员一类,来自存储与 git 的依赖一类。`dev` 之下成员保持完整的 +调试信息,依赖缺省为 `debug = "lines"`(已有的取值:只带行表,回溯仍有文件与行号);两者都可在 profile 中覆盖。 +陈述依赖一类的键的拼写与缺省的程度在 SPEC-004 中陈述(N8)。 + +**机制。** 标志是工具链行的数据(GCC `-g1`,Clang `-gline-tables-only`,MSVC 的对应项);它进入包的配置(5.1), +依赖因此在缓存中另有条目。工具链自带的静态运行时库中的调试信息不由编译标志控制,其份额在实施的第一步测量。 + +**验收。** 3.4 的方法重测:程序的大小、链接用时与依赖的编译用时记入本文;#927 的工作区重测测试程序的总大小。 + +### 5.16 产品面的覆盖(mtest 的账本,mcpp 给出面与记录;扩充 5.13) + +**决定。** 账本的需求可以由产品自己给出,证据可以取自运行的记录,而不只取自用例手写的 `clause{}`。 + +- **面。** `mcpp emit surface` 写出 mcpp 的面:每个子命令、每个选项、清单的每个键、每个原因标记,一行一项 + (`command build`、`flag build --target`、`key build.sources`、`reason build/cache`)。命令与选项取自命令行的 + 定义,键取自 `mcpp.manifest`,原因标记取自 `check_reason_tokens.sh` 所核对的那些;本项只是写出。 +- **证据。** 协议新增 `exercise` 事件:属于一个用例,列出它用到的项。测试 kit 为它运行的每条 mcpp 命令写出命令与选项; + 一次调用读取的键与发出的原因标记由 mcpp 写入该次调用的记录(环境变量指明文件),kit 转为 `exercise`。 +- **判定。** 面中没有任何用例用到的项使门禁失败,除非账本中有带理由的豁免;指向已被用到的项的豁免同样失败。 + 不设百分比的门槛。 + +**取代。** `check_e2e_coverage.py` 之外没有检查回答「哪个选项、哪个键从未被测试用到」;AGENTS.md 的「未知的键被报告」 +与本项互为两面。 + +### 5.17 预算是门禁(mtest;mcpp 的分片) + +**决定。** + +- `mtest.toml` 的 `[budget]` 陈述一个用例的处理器时间上限与一个作业的墙钟上限。超过用例上限的用例使 `mtest gate` + 失败,除非有带理由的豁免(5.13 的形式)。处理器时间取 5.9 的记录,按机器的类别比较。 +- 作业的上限决定分片数:`mtest ci plan` 已按宿主的历史为脚本 e2e 计算分片;`mcpp test --shard` 的 C++ 测试改为同一 + 来源(现为固定的四片)。 + +**验收。** 用例 955 与其余超过上限的用例各有一条理由;一个新加入而未豁免的超限用例使门禁失败。 + +### 5.18 用例结束时的检查(mtest 的 e2e 模块;测试 kit) + +**决定。** e2e 的用例结束时检查两件事,任一不满足即失败。 + +- **进程。** 用例启动的进程都已结束。`mcpplibs.os` 的进程单元已能回答一个单元是否为空。 +- **诊断行。** 被运行的程序的输出中,匹配工程声明的诊断模式的行,必须由用例声明;用例声明而没有出现的行同样失败。 + mcpp 的 kit 声明的模式是 `warning:`、`error:` 与降级的提示。 + +**依据。** AGENTS.md:做得少不能静默。今天一个用例通过时,mcpp 在其中打印的降级提示不被任何检查看见。 + +**迁移。** 现有用例中合法出现的诊断行在本轮逐个声明;其数目在实施中以报告模式测得。 + +### 5.19 抖动(SPEC-017 新增 O12;mtest、mcpp 的钩子) + +**决定。** 运行器可以陈述抖动的上限(毫秒)与种子。框架在声明的竞争点、kit 在每条命令的前后、mcpp 在取锁与发布处, +各等待一段不超过上限的随机时间。它在三种场合运行,不每夜运行: + +- **结论翻转时。** 存储中一个用例的结论在同一棵树上发生改变(重试后通过,或两次运行不同)时,CI 以若干个种子带抖动 + 重跑这一个用例,把结果与种子写入摘要。 +- **每周一次**,在 main 上运行全部测试。 +- **手动。** + +**依据。** 偶发失败的来源是没有被固定的次序。这类缺陷只在并发相关的代码改动时新增,每夜扫描在绝大多数夜里没有发现; +已有的证据是 2026-11-15 并行暴露的两个锁的缺陷(#976 的四个偶发失败中,两个与次序无关,一个由 5.10 解决)。 +能力本身很小,定位一个翻转的用例时才用得上它。 + +**边界。** 抖动不进入拉取请求的运行与全量的验证。mcpp 的钩子是一个环境变量与一处等待,位于取锁与发布所经过的 +一个函数中。 + +### 5.20 等待事件,不等待时间(mcpp 的测试) + +**决定。** 测试只等待被测程序给出的事件(一行输出、一个文件、退出)。`sleep_for` 与轮询不出现在新的测试中;现有的 +77 处记入基线文件(`.github/tools/` 中已有两份同类的基线),基线只减不增。时间本身是断言的用例在基线中注明。 +测试需要而产品没有的事件,加到产品中。 + +### 5.21 契约性的输出整段比较(mtest 的 golden;mcpp 只用于契约) + +**决定。** `e2e::golden` 增加脱敏的规则(路径、用时、大小、摘要)与更新时的摘要(哪些块改变、新增、移除)。mcpp 的 +测试把它用于其格式本身是契约的输出:机器可读的记录(docs/50)与带原因标记的拒绝。现有的 2,302 处 `contains(` 不迁移。 + +### 5.22 一棵树验证一次,被使用的就是被验证的(CI;发布规程) + +**现状。** 一次合入至少有两次全量运行:拉取请求上一次,合入后 main 上一次(发布提交的那一次现在被手动取消)。CI 测试的是 +`mcpp build --strict` 的宿主构建;发布的是 `release.yml` 另行构建的 `mcpp pack --target x86_64-linux-musl` 静态包, +它在发布前只经过冒烟,完整的验证在发布之后(`ci-fresh-install`)。main 的规则集没有必需的状态检查:CI 的结论由 +负责人查看后合入(3.6)。 + +**决定。** 验证的对象是源码树,不是提交,也不是推送。 + +- **标记。** 一次全量运行通过后,留下以树的标识命名的标记,连同它的归档、缓存条目与记录。任何之后的使用都按树查找。 +- **全量运行属于检查点。** 拉取请求的推送运行 `pr` 档;工作到达一个检查点时加 `ci:full`,运行全量。实施中的提交是 + 本地的检查点,推送发生在包含多个提交的检查点上。 +- **main 的推送。** 合入的提交的树已有标记时(验证之后 main 没有移动),不运行测试:把那次运行的产物登记为 main 的, + 保存缓存。没有标记时运行全量。供其他运行使用的缓存与历史只由这一步写入(R3 不变)。 +- **发布。** 标签所指的树已有标记时,提升那次运行的归档;没有时按现在的方式构建,并写明这次是构建。 +- **被测的即被发布的。** CI 中被测的二进制改为发布的形态:`build.yml` 产出打包后的归档,测试使用其中的程序。 + +| 时机 | 运行 | 产物 | +|---|---|---| +| 拉取请求的推送 | `pr` 档:每个宿主的构建与门;Linux 的全部单元分片与脚本 e2e 的 `pr` 集合;macOS 与 Windows 各一个单元作业;容器中的 e2e 与各宿主的 e2e | 不被使用 | +| 拉取请求加 `ci:full`(检查点) | `full` 档:全部作业 | 通过时留下树的标记 | +| 推送到 main | 树已有标记:只登记与保存;没有:`full` 档 | main 的缓存、历史、覆盖记录 | +| 发布 | 树已有标记:提升;没有:构建 | release 的归档 | + +**合并队列不是前提。** GitHub 的合并队列在合入之前对「main 加上排队的拉取请求」运行必需的检查,是给出同一个标记的 +又一个来源。工作流接受它的事件,启用与否是仓库的设置:在规则集中加入「要求合并队列」与「要求状态检查」(指向汇总 +作业 `verdict`)两条。每天合入 1 至 4 个拉取请求时不启用;并发的合入使 main 在合入后才发现失败成为常态时,由负责人 +启用,工作流与本文不需改动。 + +**代价。** 拉取请求验证之后 main 又移动时,验证发生在合入之后,与今天相同。次要作业的问题到检查点才暴露。 + +**前提,在实施中核对。** 测试是否依赖宿主构建而在静态二进制上不成立;macOS 与 Windows 的 CI 构建与发布构建的差别; +mcpp 的二进制不含提交的标识(同一棵树的两次构建给出同一个程序)。 + +## 6. mcpp 的构建与测试即规模基准 + +基准不另设工作量。每次 CI 已经构建 mcpp(830 条边)并构建、运行它的全部测试;5.6 与 5.9 使这两件事留下记录。 + +| 类别 | 内容 | 判定 | +|---|---|---| +| 不变量(每个拉取请求,小工程的 e2e,带指标与界限) | 无改动的重建运行 0 条编译边;`build` 之后的 `test` 只编译新增的单元(#986);同一 home 的第二个工程编译 0 个平台单元(5.2);新检出在已填充的缓存上运行 0 个编译器(5.3);dev 依赖由缓存提供(5.4) | 超出界限即失败 | +| 规模记录(每次全量运行) | 自建:边数、运行的边数、编译器的处理器时间、最大驻留集、最慢的边;测试构建:同上;测试运行:处理器时间、每个用例的开销;一次无事可做的 mcpp 调用的用时(e2e 调用它成千上万次) | 趋势;拉取请求与同类机器的中位数比较,写入摘要,不判定 | +| 引擎对照(`bench/`,手动) | 不变 | 不变 | + +第一类中的每一项对应本文的一个缺陷:它们在引入时都没有测试能察觉。 + +## 7. CI 的形态与实现 + +### 7.1 形态 + +| 阶段 | 现在 | 本文之后 | +|---|---|---| +| 触发与档位 | 拉取请求与 main 的推送各运行全部作业;`pr` 与 `full` 只改变 e2e 的集合与一个 macOS 镜像的子集 | 拉取请求的推送运行 `pr` 档,检查点(`ci:full`)运行 `full` 档;main 的推送在树已有标记时不运行测试(5.22) | +| 被测程序的构建 | 每个宿主一次,从零;宿主构建 | 每个宿主一次,单元由缓存提供(5.3);发布的形态(5.22) | +| 其他工具链与目标的自举构建 | 各自从零(3.5),每次运行 | 只在 `full` 档;各自的配置在缓存中命中 | +| 测试的构建 | 一次;main 的构建目录按修改时间复用(W1) | 一次;按内容命中(5.3) | +| 分片 | 固定四片,按宿主的历史分配 | 片数来自作业的预算(5.17),按机器类别的历史分配(5.9);一份预算(5.5) | +| 共享状态 | 第一个用例填充 home | 夹具按身份恢复(5.10) | +| 记录 | 程序与用例 | 另有边、步骤、机器、处理器时间、用到的面 | +| 选择 | 脚本 e2e 按覆盖图;C++ e2e 全部 | 不变;头文件按清单;函数粒度视测量(5.11) | +| 门禁 | 测试的结论;每个 e2e 在某处运行 | 另有预算(5.17)、面的覆盖(5.16)、不变量(第 6 节) | +| 定时 | 每夜:覆盖图、金丝雀、已发布版本的安装 | 不变;抖动每周一次,并在结论翻转时运行(5.19) | +| 发布 | 四个平台各自构建,约 30 分钟 | 提升已验证的归档(5.22) | + +### 7.2 实现:逐个文件 + +| 文件 | 改动 | +|---|---| +| `ci.yml` | `changes` 计算本次运行的树的标识;`set` 的含义扩大到作业:`bootstrap`、`cross`、`target-matrix`、`upgrade`、`openkal`、`macos-ios`、`windows-msvc-xlings`、`linux` 中其他工具链的作业、第二个 macOS 镜像只在 `full` 档运行;main 的推送先查树的标记,已有时只运行登记与保存的作业;新增汇总作业 `verdict`,在 `full` 档通过时上传标记 `verified-<树>`;`on:` 接受 `merge_group`(未启用合并队列时不触发) | +| `build.yml` | 构建后以 `mcpp pack` 产出发布形态的归档(Linux 为 `x86_64-linux-musl` 的静态包),`mcpp-built-` 取其中的程序;`full` 档另上传 `mcpp-release-` 与单元缓存中本次用到的条目(5.3 的修剪) | +| `ci-unit.yml`、`mcpp-unit` | 分片的清单来自计划作业的输出,不再是固定的 `[1,2,3,4]`;归档阶段取 main 登记的单元缓存,不再取构建目录与 `.source-blobs`;运行阶段陈述预算(5.5)与监测程度(5.9);重跑仍为 `--failed` | +| `e2e-mtest`、`ci-*-e2e.yml` | mtest 引擎陈述预算;存储照旧上传 | +| `newest_artifact.sh` | 增加按树的标记查找一次运行;「main 最近一次成功的运行」的含义不变(登记之后产物在 main 的运行之下) | +| `release.yml` | 新增 `promote`:由标签得到树,按标记找到运行,下载各宿主的 `mcpp-release-`,核对校验和与版本,创建 release;没有标记时运行现有的构建作业,摘要写明这次是构建而非提升 | +| 新增 `jitter.yml`(每周、手动,及 CI 运行结束后有结论翻转时) | 取 main 登记的归档与测试,带抖动运行;翻转时只运行翻转的用例,若干个种子;结果与种子写入摘要 | +| `e2e-coverage.yml` | 覆盖图记录函数;首次运行给出 5.11 的统计 | +| `test-report` | 报告增加各段:构建的关键路径与最慢的边、步骤、余量、预算的超出、面中未被用到的项、结论翻转的用例 | +| 新增 `.github/actions/project-test` | 下游仓库的动作:安装固定版本的 mcpp,恢复 home(拉取请求只读),`mcpp test`,上传存储,写摘要 | +| `.github/tools/` | 新增 `waits.baseline` 与其检查(5.20);`check_version_pins.sh` 纳入骨架的 mtest 版本;`check_e2e_coverage.py` 与两份 tsv 由账本取代(5.13);`source_mtimes.py` 在固定版本移动后移除 | +| `docs/92-release.md`、`mcpp-release` 技能 | 发布的步骤改为:发布的拉取请求在 `ci:full` 的检查点通过后合入,打标签,`release.yml` 提升;不再取消 main 的运行(它已不运行测试) | + +仓库的设置不需改动。 + +### 7.3 自举版本的约束 + +构建 mcpp 自身的是 `.xlings.json` 固定的已发布版本。因此 5.3 对「构建 mcpp」的各个作业(3.5)生效,要等含有它的版本 +发布并成为固定版本之后;这是发布规程已有的一步(固定版本的移动),W1 的移除与它同一个提交。测试的构建由本提交的 +mcpp 完成的部分(各宿主的单元作业)在本轮即生效。 + +### 7.4 下游仓库 + +`project-test` 之外,下游得到的是引擎与框架的能力本身:实例的复用(5.1)、构建的记录(5.6)、头文件的选择(5.11)、 +doctest 适配器(5.14)、依赖的调试信息(5.15)。boost-m 自己的三项用法(两组各运行全部测试、每个文件一份 doctest +实现、合并移植的示例文件)由其维护者决定。 + +## 8. 各仓库的范围 + +每个仓库一个拉取请求,提交作为检查点。mcpp 的拉取请求是 #984(分支 `fix/rc-include-dir-globs`):它已有的两个提交 +是最早的检查点,本文与各任务的提交接在其后,标题与说明在内容齐备时改写。任务、位置与依赖在第 12 节。 + +| 仓库 | 范围 | +|---|---| +| mcpp | 本文;SPEC-004 与 SPEC-017 的条款;引擎(5.1 至 5.6、5.8 至 5.11、5.14 至 5.16、5.19 的运行器与钩子);测试 kit 与测试(5.7、5.10、5.18、5.20、5.21、第 6 节);CI 与发布规程(5.22、第 7 节);docs 两种语言与 CHANGELOG | +| mtest | 协议的实现与一致性套件;5.12、5.13、5.16 至 5.19、5.21 的框架与引擎一侧;doctest 适配器;报告与分析 | +| mcpplibs.os | 处理器的型号(O1) | +| boost-m | 不由本文改动 | + +## 9. 预期 + +| 对象 | 基线 | 预期 | 依据 | +|---|---|---|---| +| 用例 955 的步骤 A(分片内,EPYC) | 478 s | 与步骤 D 同一量级 | 5.2;探针第二遍中命中的配置为 7 s | +| 分片中平台运行时的边用时 | 2,579 至 7,255 s | 每个配置在夹具的构建缓存中命中 | 5.2、5.10 | +| 分片内争用 | 墙钟 2.3 倍 | 不超过 1.3 倍 | 5.5 | +| 各作业的 mcpp 自举构建 | 310 至 2091 s,各自从零 | 与改动的单元数成正比 | 5.3(推算) | +| boost-m 的测试步骤 | 916 至 2507 s | #986 的部分减少至多六分之一;长尾在有记录之后定位 | 5.1、5.6 | +| 新工程中的 mtest 库 | 每次 4 至 16 s | 0 | 5.4 | +| mtest 的每个文件(GCC) | 1.1 s | 不超过 0.15 s | 5.12 | +| 一次合入的全量运行 | 至少两次 | 一次 | 5.22 | +| 一个推送三次的拉取请求的 runner 时间 | 约四次全量 | 三次 `pr` 档加一次全量,约少四成 | 5.22;按 3.5 中次要作业约占 649 分钟中的 350 分钟估算 | +| 发布 | 约 30 分钟的构建,产物未经完整验证 | 提升,数分钟;产物即被验证的那一份 | 5.22 | + +拉取请求的整次墙钟不给出预期:它受组织内的排队支配(2026-11-14 记录 6.1),本文改变的是工作量。 + +## 10. 风险与未采纳 + +- **5.3 的正确性取决于键的完备。** 遗漏一个输入会提供错误的目标文件。清单取自编译器自己的依赖记录;逐字节的核对是 + 验收的一部分;不可复现的工具链不缓存。 +- **5.5 改变每一次嵌套构建的并行度。** 单独运行的构建不受影响(没有父 generation)。 +- **5.10 把 home 的内容变成声明。** 现有用例中隐含依赖共享 home 里某个包的,会在迁移时暴露;逐个改为声明。 +- **5.12 的接口改动**不改变已发布名字的含义(SPEC-017 的「只增不改」针对协议;库的源码兼容性另由 mtest 的版本规则约束)。 +- **5.22 之下次要作业的问题到检查点才暴露。** 需要提前看到时加 `ci:full`;验证之后 main 又移动的情形与今天相同。 +- **5.18 使现有用例中未声明的诊断行成为失败。** 其数目未知,在实施中先以报告模式测得。 +- **5.15 改变依赖的缺省调试程度。** 调试进依赖的代码时需要在 profile 中覆盖;缓存中依赖的条目重建一次。 +- **未采纳:以 GNU make 的 jobserver 作为预算的形式。** 它需要命名管道,`mcpplibs.os` 没有;槽位文件以已有的文件锁实现。 +- **未采纳:在 Linux 上交叉构建各平台的测试、到各平台只运行。** macOS 的原生目标与 MSVC 不能从 Linux 交叉;macOS 的 + 瓶颈是 runner 的排队(2026-11-14 记录)。 +- **未采纳:更大的 runner。** 2026-11-15 已定。 +- **未采纳:以 boost-m 作为规模基准。** 规模基准是 mcpp 自己的构建与测试(第 6 节)。 + +## 11. 已定与待定事项 + +**已定(负责人 2026-11-20)。** + +| 编号 | 事项 | 决定 | +|---|---|---| +| N1 | 5.3(单元的内容寻址)与其余各项同一个拉取请求 | 是 | +| N2 | #802(内存准入)并入 5.5 | 是 | +| N3 | 5.11 的函数粒度选择,阈值取 50%,测量先于实施 | 是 | +| N4 | 5.13 的账本取代 mcpp 的两份 tsv 与规范的状态列;boost-m 的由其维护者决定 | 是 | +| N5 | doctest 适配器进入 mtest | 是 | +| N6 | 下游动作放在 mcpp 仓库 | 是 | +| N7 | 向 boost-m 的维护者提供测量与用法的建议 | 不发出 | +| — | 评审中补充的各项(5.16 至 5.22,第 6 节的调用用时,5.3 的按使用修剪) | 并入 | +| N8 | 5.15:`dev` 之下依赖的缺省调试程度为 `"lines"`;键在 `[profile.<名>]` 之下,名字在 S2 中按 SPEC-004 的命名规则给出 | 是(2026-11-21) | +| N9 | 5.22:只在 `full` 档运行的作业清单按 7.2;容器中的 e2e 与各宿主的 e2e 留在 `pr` 档 | 是(2026-11-21) | +| N10 | `mcpplibs.os` 进入本轮的范围(O1),其形式须符合该仓库的规则并且通用 | 是(2026-11-21) | +| N11 | 合并队列 | 不是前提,不启用;工作流接受其事件,启用与否是仓库的设置(5.22)(2026-11-21) | +| — | 5.19 不每夜运行:结论翻转时、每周、手动 | 是(2026-11-21) | +| — | 5.22 按树认定,全量运行在 `ci:full` 的检查点 | 是(2026-11-21) | +| — | mcpp 的拉取请求复用 #984;实施中的提交是本地的检查点,推送发生在包含多个提交的检查点上 | 是(2026-11-21) | + +没有待定事项。 + +## 12. 实施方案:任务与依赖 + +每个仓库一个拉取请求,下表的每个任务是其中的一个提交或一组相邻的提交。提交是本地的检查点:一组任务在本地做完并 +核对之后才推送,全量的验证(`ci:full`)发生在包含多个提交的检查点上,不随每个提交。任务按依赖排列;同一仓库内没有依赖关系的任务 +次序任意。「位置」写明代码所在:新的机制进入 `modules/` 下的新包(AGENTS.md §4:缩小 `src/` 是方向; +`execute.cppm` 已有 8,649 行),按 `check_modules_wiring.sh` 的四处接入。 + +### 12.1 规范(mcpp 仓库,先于实现) + +| 编号 | 任务 | 位置 | 依赖 | +|---|---|---|---| +| S1 | SPEC-017 v2.3 的条款,状态为未实现:P5 的 `step` 与 `exercise`;B4 的 `builds.ndjson` 与 `machine` 的字段;O5 在选择之下的时限;O11 `budget`;O12 `jitter` | `docs/specs/test.md` | — | +| S2 | SPEC-004 的条款:实例摘要的输入(§9 第 17 项);键中路径的角色;`[build] memory_threshold`;profile 中依赖的调试信息 | `docs/specs/manifest-semantics.md` | — | + +### 12.2 mcpplibs.os + +| 编号 | 任务 | 位置 | 依赖 | +|---|---|---|---| +| O1 | 处理器的型号:与 `machine::memory()` 同一形式,回答为 `std::optional`,各系统一个实现单元并取自其自身的服务,映像不回答;它是机器的一个事实,不为测试而设(基准的报告、诊断同样读取) | `src/machine.cppm` 与各系统的实现单元 | — | +| O2 | 发布 | — | O1 | + +### 12.3 mtest(发布 0.11.0) + +| 编号 | 任务 | 位置 | 依赖 | +|---|---|---|---| +| T1 | 协议:库写出 `step` 与 `exercise`;引擎的模型与存储读写 `builds.ndjson`、步骤、用到的项;一致性套件 | `lib/src/mtest.*`、`engine/src/model.*`、`store.*`、`conformance/` | S1 | +| T2 | 预算与抖动:引擎为所有者,陈述给它运行的程序;库的竞争点 | `engine/src/exec.*`、`schedule.*`、`lib/src/mtest.*` | S1 | +| T3 | 机器与用量:引擎写 `machine`;估计与 `ci plan` 按机器类别取历史 | `engine/src/schedule.*`、`cicheck.*` | S1、O2 | +| T4 | 库的成本:用例声明与常见 `expect` 的非模板入口;输出归属不再每个用例建文件并 `fsync` | `lib/src/mtest.*` | — | +| T5 | 账本与面的覆盖:`[ledger.*]`、豁免、判定与报告 | `engine/src/config.*`、`commands.*`、`report.*` | T1 | +| T6 | 预算的门禁:`[budget]`、豁免;`ci plan` 接受任何存储 | `engine/src/cicheck.*`、`commands.*` | T3 | +| T7 | e2e 模块:用例结束时的进程与诊断行检查;golden 的脱敏与更新摘要 | `lib/src/e2e.*` | — | +| T8 | doctest 适配器(1 级),随一致性套件 | `adapters/doctest/` | — | +| T9 | 覆盖图记录函数 | `engine/src/coverage.*` | — | +| T10 | 报告与分析读取新增记录:关键路径下到边与步骤,余量,机器类别 | `engine/src/dag.*`、`analysis.*`、`report.*` | T1、T3 | +| T11 | 3.4 的三个场景成为 mtest 自己的 `Bench` 用例,带 5.12 的界限 | `lib/tests/`、`engine/tests/` | T4 | +| T12 | 发布 0.11.0 | — | T1 至 T11 | + +### 12.4 mcpp:引擎 + +| 编号 | 任务 | 位置 | 依赖 | +|---|---|---|---| +| M0 | 资源脚本的包含目录调用 `expand_manifest_include_entry`;用例 837 增加稍后建立的字面目录(5.1,接在 #984 的两个提交之后) | `src/build/prepare/windows_resources.cpp`、`tests/e2e/windows/resources.cpp` | — | +| M1 | 包的配置是一个函数;实例摘要与缓存键共用;展开后的包含目录由它给出,三份推导合一(5.1) | `src/build/cache_key.cppm`、`plan.cppm`、`prepare/package_root.cpp`、`prepare/windows_resources.cpp` | S2、M0 | +| M2 | 路径的角色;可缓存的包的构建程序输出在 home 中(5.2) | `cache_key.cppm`、`prepare/graph_provisioning.cpp`、`build_programs.cpp`;docs/25 | M1 | +| M3 | 缓存的拒绝各有原因标记;以它定位并修复 mtest 库(5.4) | `prepare/dependency_cache.cpp`;docs/50 的登记 | — | +| M4 | 构建记录:每条边一行,处理器时间与最大驻留集,各阶段的汇总,长边的进度(5.6) | 新包 `modules/build-record`;`src/build/progress.cppm` 接入 | S1 | +| M5 | 单元的内容寻址:键、清单、条目、发布;每个图经由执行器;工具链行陈述输出是否可复现;按使用修剪(5.3) | 新包 `modules/unit-cache`;`schedule/compiler_transaction.*`、`ninja_backend.cppm` 接入;`modules/toolchain-model/data/builtin.toml` | M1、M2、M4 | +| M6 | 预算:所有者与客户,槽位的目录,内存的阈值;执行器在启动编译器前取槽位(5.5) | 新包 `modules/budget`;`schedule/detach_codegen.cppm` 接入 | S1、S2 | +| M7 | 运行器的记录:`machine`、缺省的监测程度、从自己的进程测量;按机器类别的估计(5.9) | 新包 `modules/test-run`;`src/build/test_schedule.cppm` | S1、O2 | +| M8 | 时限:选择保留作用域声明的时限(5.8 的运行器一侧) | `modules/test-run`;`src/build/test_protocol.cppm` | S1 | +| M9 | 运行夹具的服务(P11)(5.10 的运行器一侧) | `modules/test-run` | S1 | +| M10 | 归档运行的三个选择缺陷;骨架的 mtest 版本进入版本检查(5.14) | `src/build/execute.cppm`、`test_protocol.cppm`;`.github/tools/check_version_pins.sh` | — | +| M11 | 依赖的调试信息(5.15) | `modules/toolchain-model/data/builtin.toml`、`src/build/flags.cppm`、`mcpp.manifest` | S2、M1 | +| M12 | `mcpp emit surface`;一次调用读取的键与发出的原因标记写入记录(5.16) | `src/cli.cppm`、`modules/manifest`、`src/diag.cppm` | S1 | +| M13 | `--since` 的头文件规则读取清单(5.11) | `src/build/test_impact.cppm` | M5 | +| M14 | 抖动的钩子:取锁与发布处的一处等待(5.19) | `src/build/dirlock.cppm` 与发布所经过的函数 | S1 | + +### 12.5 mcpp:测试 kit 与测试 + +| 编号 | 任务 | 位置 | 依赖 | +|---|---|---|---| +| K1 | kit:每条命令一个步骤;命令的时限取用例的余量;写出用到的项 | `tests/support/` | T12、M12 | +| K2 | home 的运行夹具;用例陈述只读或修改;修改的用例得到自己的 home 与只读的存储根 | `tests/support/`、`tests/e2e/` | T12、M9、M2 | +| K3 | 第 6 节的不变量用例与规模记录 | `tests/e2e/cache/`、`tests/e2e/self/` | M1 至 M5 | +| K4 | 等待的基线;能以事件替换的即替换 | `tests/`、`.github/tools/waits.baseline` | — | +| K5 | 诊断行的声明:以报告模式测得数目,逐个声明 | `tests/e2e/` | T12 | +| K6 | 契约性输出的 golden | `tests/e2e/`(机器可读输出、拒绝) | T12 | +| K7 | 账本:两份 tsv 与规范的状态列迁入;面的账本与豁免 | `tests/e2e/mtest.toml`、`docs/specs/` | T12、M12 | +| K8 | 函数粒度的统计,写入本文;达到阈值时实施选择 | `e2e-coverage.yml`、`src/build/test_impact.cppm` | T12 | +| K9 | 预算的豁免:超过上限的用例各一条理由 | `tests/e2e/mtest.toml` | T12、M7 | + +### 12.6 CI + +| 编号 | 任务 | 依赖 | +|---|---|---| +| CI1 | 树的标识与标记、档位扩大到作业、汇总作业 `verdict`、main 的推送按标记决定(7.2 的 `ci.yml`) | — | +| CI2 | 被测的二进制为发布的形态;核对测试在其上成立(7.2 的 `build.yml`) | — | +| CI3 | 发布的提升与回退;发布规程与技能的文字 | CI1、CI2 | +| CI4 | 分片来自计划;预算与监测程度的陈述 | M6、M7、T12 | +| CI5 | 单元缓存的传输、登记、查找与修剪 | M5、CI1 | +| CI6 | home 夹具按身份恢复 | K2 | +| CI7 | 报告的各段 | M4、T12 | +| CI8 | 抖动的工作流(每周、手动、结论翻转时);覆盖图的函数 | M14、T12、K8 | +| CI9 | 下游的动作 `project-test` | M4 | +| CI10 | 固定版本移动之后:W1 的移除,自举构建的命中(7.3) | 发布 | + +### 12.7 发布的次序 + +`mcpplibs.os`(O2)→ mtest 0.11.0(T12)→ mcpp(其 dev 依赖指向前两者)→ 固定版本的移动(CI10)。mtest 的拉取请求 +对照 mcpp 拉取请求分支中的 S1 实现;S1 的条款在 mcpp 的实现完成的提交中由未实现改为已实现。 + +### 12.8 关键的依赖链 + +- **身份到缓存到 CI:** S2 → M1 → M2 → M5 → CI5 → CI10。5.3 的收益在这条链的末端。 +- **协议到记录到报告:** S1 → T1、M4 → T10 → CI7。 +- **机器类别:** O1 → O2 → T3、M7 → T6、CI4。 +- **夹具:** S1 → M9 → K2 → CI6;K2 另需 M2,使夹具中的构建缓存对每个工程有效。 +- **树的标记与发布:** CI1、CI2 → CI3。它们不依赖引擎的改动,可以是最早的检查点。 + +## 13. 自评 + +对照 mcpp 的规则(AGENTS.md)、mtest 的规则(其 AGENTS.md)、通用性与成本,逐项核对。下列是核对后改动了本文的各处。 + +| 核对 | 发现 | 改动 | +|---|---|---| +| mtest:依赖已有的库,不重写 | v1 的 5.12 要库自己写出事件、去掉 JSON 库 | 撤回。测量表明 JSON 的代码为 0.13 MB,大小来自调试信息(8.5 MB);改为 5.15 | +| mtest:只经 `mcpplibs.os` 到达系统 | 5.9 需要处理器的型号,`mcpplibs.os.machine` 没有 | 新增 O1,范围多一个仓库 | +| 通用性:预算的形式 | v2 的 5.5 以「父 generation」陈述,是 mcpp 的内部形式,别的程序无从遵守 | 改为规范中的一个目录与其中的槽位文件,持有文件锁即持有槽位;`mcpplibs.os` 的文件锁即可实现,脚本以 `flock` 遵守 | +| 通用性:GNU make 的 jobserver | 它是事实上的标准,但需要命名管道,`mcpplibs.os` 没有 | 不采用,记入第 10 节 | +| 引擎不出现框架的名字 | home 的夹具是测试的概念 | 位于 `tests/support/`,引擎只实现 P11 | +| 每个缺省是数据,且可覆盖 | 内存阈值、依赖的调试信息、输出是否可复现 | 分别为清单的键、profile 的键、工具链行的数据 | +| 做得少不能静默 | 缓存的拒绝、发布的回退、找不到标记 | 5.4 的原因标记;7.2 中「写明」的各处 | +| 成因还是表象 | 把 `trees` 的时限调大、把套件划分得更细,都是表象 | 不提出;5.2、5.5、5.10 处理成因 | +| 已有的机制 | v1 的草稿曾把覆盖图当作新设计,把 mtest 库不进缓存当作回归 | 第 2 节的清点;3.4 改为实测的三个版本 | +| 5.11 的依据 | v2 写头文件的依赖记录来自构建记录,构建记录不含它 | 改为 5.3 的清单 | +| 5.6 的体积 | 一个分片的嵌套构建有上万条边 | 由缓存提供的边按包汇总为一行 | +| 5.3 的开销 | 每条边的摘要计算 | 文件的摘要在一次构建中按(路径、大小、修改时间)记忆;验收增加无改动重建的用时 | +| 5.3 的安全 | 缓存的条目被别的运行使用 | 只有 main 的推送运行登记或写入;拉取请求的运行只读 | +| 5.14 的适配器 | v1 写「不改测试代码」 | 改为去掉每个文件的实现一行 | +| 5.22 的前提 | v3 以合并队列为前提;仓库没有必需的状态检查,每天合入 1 至 4 个,启用它要改规则集与每个会话的合入方式 | 改为按树认定,全量运行在检查点;合并队列是可选的设置 | +| 5.19 的频率 | v3 每夜运行;这类缺陷只在并发相关的改动时新增 | 改为结论翻转时、每周、手动 | +| 已有的修复(#984) | 它称与编译步骤同一展开,但少了字面目录的回退 | 5.1 增加一段,任务 M0 | +| 任务的编号 | CI 的任务以 C 编号,与第 4 节的成因编号相同 | 改为 CI1 至 CI10 | +| 自举 | 构建 mcpp 自身的是已发布的版本 | 7.3 写明哪些收益在固定版本移动之后 | + +仍未核对、列为实施中的前提的各项:5.5 的槽位协议与现有 generation 的衔接(只读了注释);5.3 的 BMI 在 Clang 与 MSVC 上 +是否逐字节可复现;5.22 的两项前提;5.15 中工具链运行时库的调试信息的份额;mtest 库不由缓存提供的成因。 diff --git a/.agents/docs/README.md b/.agents/docs/README.md index 37e6dce75..1987adf76 100644 --- a/.agents/docs/README.md +++ b/.agents/docs/README.md @@ -18,7 +18,7 @@ superseded_by: 2026-09-07-....md # when status is superseded --- ``` -367 records. +368 records. ## By subject @@ -173,6 +173,10 @@ Records that declare one. Everything else is listed by date below. - [SDK toolchains, the payload/engine seam, and openkal across iOS, Android and Web](2026-09-11-sdk-toolchains-and-ios-local-verification.md) — landed - [Where a platform's knowledge belongs: iOS, Android and Web across the engine, the index and the plugins](2026-09-11-platform-targets-design-review.md) — active +### test + +- [生态的测试与 CI 方案:mtest 的框架与运行控制、mcpp 自身的测试系统、CI 的形态(mcpp#986、#976、#927、#802)](2026-11-20-test-architecture-upgrade-design.md) — active + ### toolchain - [目标基座与执行宿主:sysroot 的通用模型、运行器契约、自备输入,以及同批的发布产物路径(#832)](2026-10-23-target-base-and-execution-host-design.md) — landed @@ -206,6 +210,7 @@ Records that declare one. Everything else is listed by date below. ### 2026-11 +- [生态的测试与 CI 方案:mtest 的框架与运行控制、mcpp 自身的测试系统、CI 的形态(mcpp#986、#976、#927、#802)](2026-11-20-test-architecture-upgrade-design.md) — active - [程序起点模型:启动者、两层入口与 `main` 的三种陈述(mcpp#845、mcpp#981)](2026-11-20-program-start-model.md) — landed - [生态规则收敛方案:2026.11.18.1 之后十个 issue 的五类成因、七项决定与实施范围](2026-11-19-ecosystem-rule-convergence-design.md) — landed - [第二部分之后的补充方案:带修正的 ninja、模块重编译的级联、工作区发布与本轮遗留项](2026-11-17-ninja-cascade-publish-supplement-design.md) — active From d82e2d904485acf2135c3735abca0ccec74c4774 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 00:42:37 +0900 Subject: [PATCH 04/38] fix: a resource script's include dirs are expanded as the root manifest's are, keeping a literal directory a build step creates later resource_include_dirs called expand_dir_glob, which drops a literal entry that does not exist when the build is planned; before mcpp#984 such an entry was joined as written. It now calls expand_manifest_include_entry, the expansion the root manifest's compile steps use, which keeps it (#390). e2e 837 states R5: gui's generated/inc reaches the rc_object edge. --- src/build/prepare/windows_resources.cpp | 30 +++++++++++-------------- tests/e2e/windows/resources.cpp | 12 ++++++++-- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/build/prepare/windows_resources.cpp b/src/build/prepare/windows_resources.cpp index 261d4b119..c299b2e63 100644 --- a/src/build/prepare/windows_resources.cpp +++ b/src/build/prepare/windows_resources.cpp @@ -15,7 +15,6 @@ import mcpp.home; import nlohmann.json; import mcpp.log; import mcpp.manifest; -import mcpp.modgraph.scanner; import mcpp.source_kind; import mcpp.toolchain.clang; import mcpp.toolchain.hostflags; // the compile-token producer the package std module reuses @@ -245,26 +244,23 @@ static bool target_takes_code_page(const mcpp::manifest::Target& t) } // Where the resource compiler looks for a script's includes and files: the -// package's directory, then its include_dirs. A relative entry is a glob, -// expanded as the compile steps expand it (package_root.cpp): `*/include` -// names the directories it matches. Passed as written, the `*` reached -// llvm-windres, and LLVM's tools on Windows expand wildcards in their own -// command line -- against FILES -- so the preprocessor saw several inputs and -// refused ("cannot specify -o when generating multiple output files"). Every -// program of an index package with a wrap-layer `*/` include failed so -// (compat.protobuf's protoc host tool). +// package's directory, then its include_dirs, each expanded as the root +// manifest's compile steps expand it (plan.cppm's +// expand_manifest_include_entry): a glob names the directories it matches, +// and a literal directory that does not exist yet is kept, since a build step +// may create it before the script compiles (#390). Passed as written, the `*` +// of `*/include` reached llvm-windres, and LLVM's tools on Windows expand +// wildcards in their own command line -- against FILES -- so the preprocessor +// saw several inputs and refused ("cannot specify -o when generating multiple +// output files"). Every program of an index package with a wrap-layer `*/` +// include failed so (compat.protobuf's protoc host tool). static std::vector resource_include_dirs(const std::filesystem::path& packageDir, const std::vector& includeDirs) { std::vector dirs{packageDir}; - for (auto const& d : includeDirs) { - if (d.is_absolute()) { - dirs.push_back(d); - continue; - } - for (auto& dir : mcpp::modgraph::expand_dir_glob(packageDir, d.generic_string())) - dirs.push_back(dir); - } + for (auto const& d : includeDirs) + for (auto& dir : mcpp::build::expand_manifest_include_entry(packageDir, d)) + dirs.push_back(std::move(dir)); return dirs; } diff --git a/tests/e2e/windows/resources.cpp b/tests/e2e/windows/resources.cpp index e2f82f883..b148096c3 100644 --- a/tests/e2e/windows/resources.cpp +++ b/tests/e2e/windows/resources.cpp @@ -340,7 +340,9 @@ E2E resources{"resources", [] { // R1 each member's program carries its own icon and not the other's; R2 the synthesised // script of a member is under res// with that member's version and metadata; R3 a // member's script compiles with that member's include_dirs; R4 a glob among them (an - // archive's wrap layer, `*/inc`) reaches the resource compiler expanded, never as `*`. + // archive's wrap layer, `*/inc`) reaches the resource compiler expanded, never as `*`; + // R5 a literal directory that does not exist when the build is planned (one a build step + // creates, #390) still reaches it, as it reaches the compile steps. s::project ws{{"mcpp.toml", "[workspace]\nmembers = [\"cli\", \"gui\"]\n"}, {"cli/assets/app.ico", icon(icon_a)}, {"cli/src/main.cpp", "int main() { return 0; }\n"}, {"cli/mcpp.toml", "[package]\nname = \"cli\"\nversion = \"1.2.3\"\nauthors = [\"Cli Corp\"]\n\n[resources]\nicon = \"assets/app.ico\"\n\n[targets.cli]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}, @@ -348,7 +350,7 @@ E2E resources{"resources", [] { {"gui/include/gui_ids.h", "#define GUI_ICON 101\n"}, {"gui/vendor-1.0/inc/vendor_ids.h", "#define VENDOR_ICON 102\n"}, {"gui/gui.rc", "#include \"gui_ids.h\"\n#include \"vendor_ids.h\"\nGUI_ICON ICON \"assets/app.ico\"\nVENDOR_ICON ICON \"assets/app.ico\"\n"}, - {"gui/mcpp.toml", "[package]\nname = \"gui\"\nversion = \"4.5.6\"\n\n[build]\ninclude_dirs = [\"include\", \"*/inc\"]\n\n[resources]\nfiles = [\"gui.rc\"]\nextra-inputs = [\"assets/app.ico\"]\n\n[targets.gui]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}}; + {"gui/mcpp.toml", "[package]\nname = \"gui\"\nversion = \"4.5.6\"\n\n[build]\ninclude_dirs = [\"include\", \"*/inc\", \"generated/inc\"]\n\n[resources]\nfiles = [\"gui.rc\"]\nextra-inputs = [\"assets/app.ico\"]\n\n[targets.gui]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}}; auto r = ws.mcpp({"build", "--workspace", "--target", "x86_64-windows-gnu"}); expect(fatal(e2e::succeeded(r))) << "workspace build: " << s::text(r); const auto cli = s::glob(ws.at("target"), "*/*/bin/cli/cli.exe"); @@ -376,6 +378,12 @@ E2E resources{"resources", [] { literal = literal || lines[j].contains("*/inc") || lines[j].contains("*\\inc"); } expect(fatal(expanded && !literal)) << "R4 gui's `*/inc` does not reach the resource compiler expanded"; + bool later = false; + for (std::size_t i = 0; i < lines.size(); ++i) + if (std::regex_search(lines[i], std::regex(R"(rc_object .*gui\.rc)"))) + for (std::size_t j = i; j < lines.size() && j <= i + 2; ++j) + later = later || lines[j].contains("generated/inc") || lines[j].contains("generated\\inc"); + expect(fatal(later)) << "R5 gui's `generated/inc`, created after planning, is not on the resource compile"; }; "922 dependency images keep their resources"_test[s::windows][s::python3] = [] { From 7ae560ab40bb1ad0830ffdb615644d35e62f100c Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 00:58:39 +0900 Subject: [PATCH 05/38] fix: a package's instance is a function of its configuration, not of its units (#986); each cache refusal names its reason The instance digest hashed each unit's object path, so mcpp test, which keeps a unit an inactive feature gates, gave the package a second instance and compiled every unit again. It now hashes the package's distinct unit configurations and its imports' digests. A package's include directories have one expansion, expand_manifest_include_entry, for its compile steps, its private part and its resource script. serve_or_fill names why a package is not served (no-identity, unnamed-sources, unnamed-upstream, outside-store, unaddressable-unit, miss) under --verbose and in the build context. The token found why mcpplibs.mtest was never cached: its synthesized entry main had no cache address, which took the whole package out. An entry unit now compiles with the image that links it, and the package's other units are cached; a second project's test shows mcpplibs.mtest as Cached. e2e: 986 (the issue's reproduction), 1037 B (mtest cached in the second project). --- src/build/plan.cppm | 26 +++++++-- src/build/prepare.cppm | 12 ++++ src/build/prepare/dependency_cache.cpp | 28 ++++++++-- src/build/prepare/package_root.cpp | 77 +++++++++----------------- tests/e2e/test/skeleton.cpp | 5 ++ tests/e2e/workspace/inheritance.cpp | 26 +++++++++ 6 files changed, 113 insertions(+), 61 deletions(-) diff --git a/src/build/plan.cppm b/src/build/plan.cppm index 43cee646b..6dffa9c6a 100644 --- a/src/build/plan.cppm +++ b/src/build/plan.cppm @@ -74,6 +74,14 @@ struct CompileUnit { // the one kind of unit `trim_paths` leaves its paths, since a test finds // its fixtures through `__FILE__` and runs where it was built. bool testUnit = false; + // A target's entry `main`, synthesized for the image that links it (a test + // program's `mtest/main`). It provides no module, so it compiles where the + // image is built, beside the package's other units: the global cache + // serves and fills those and passes this one by, and the instance digest + // does not count it, as it does not count the root's units (design + // 2026-11-20 §5.4: an entry without a cache address took the whole of + // mcpplibs.mtest out of the cache in every project). + bool entryUnit = false; // The key of this unit's module map in `BuildPlan::moduleScopes`: the BMI // of every module the unit imports, directly or through another module, // and of the module it provides. Empty when every such BMI lies at its @@ -3505,6 +3513,7 @@ make_plan(const mcpp::manifest::Manifest& manifest, : scanned.declaration; cu.object = object_for(entry, owner, std::filesystem::relative(entry, pkgRoot), pkgIdx).object; + cu.entryUnit = true; plan.compileUnits.push_back(cu); object = cu.object; } @@ -4719,13 +4728,17 @@ make_plan(const mcpp::manifest::Manifest& manifest, // include directories and flags, its features' defines among them -- and // the digests of the packages whose modules it imports, whose BMI paths // its command names. It holds no version and no compiler build, so an - // upgrade keeps the paths. The root's units keep their place: one plan - // has one root, and its command line is the configuration's. + // upgrade keeps the paths. It holds no unit either (mcpp#986): `mcpp test` + // keeps a package's units that an inactive feature gates and `mcpp build` + // drops them, and a digest of the unit set gave the two plans two + // instances of one configuration, so every unit compiled again in the + // second. The root's units keep their place: one plan has one root, and + // its command line is the configuration's. const auto rootPackageName = manifest.package.virtualRoot ? std::string{} : qualified_package_name(manifest); std::map instanceDirs; // package -> `/` { - std::map own; // package -> its units' flags + std::map> own; // package -> its configurations std::map> importsOf; // package -> providers // A directory is spelled relative to the store or the project that // holds it: the digest names a directory a cached BMI records, so it @@ -4756,8 +4769,8 @@ make_plan(const mcpp::manifest::Manifest& manifest, return out; }; for (auto const& cu : plan.compileUnits) { - if (cu.packageName.empty() || cu.packageName == rootPackageName || cu.testUnit) continue; - own[cu.packageName] += cu.object.generic_string() + "\n" + flags_text(cu); + if (cu.packageName.empty() || cu.packageName == rootPackageName || cu.testUnit || cu.entryUnit) continue; + own[cu.packageName].insert(flags_text(cu)); importsOf[cu.packageName].merge(providers_of(cu)); } std::map digest; @@ -4765,7 +4778,8 @@ make_plan(const mcpp::manifest::Manifest& manifest, std::function digest_of = [&](const std::string& pkg) { if (auto it = digest.find(pkg); it != digest.end()) return it->second; if (!own.contains(pkg) || !visiting.insert(pkg).second) return std::string{}; - std::string text = own[pkg]; + std::string text; + for (auto const& configuration : own[pkg]) text += configuration + "\x1e"; for (auto const& dep : importsOf[pkg]) text += "D" + dep + "=" + digest_of(dep) + "\n"; visiting.erase(pkg); return digest[pkg] = mcpp::toolchain::hash_string(text).substr(0, 8); diff --git a/src/build/prepare.cppm b/src/build/prepare.cppm index a931442c8..ebdef0f2f 100644 --- a/src/build/prepare.cppm +++ b/src/build/prepare.cppm @@ -680,6 +680,18 @@ export struct BuildContext { std::size_t units = 0; }; std::vector cachedDeps; + // Each package the global cache did not serve, and why, as a token + // (design 2026-11-20 §5.4): `no-identity`, `unnamed-sources`, + // `unnamed-upstream`, `outside-store`, `unaddressable-unit`, or `miss` + // when the entry is filled after the build. `--verbose` prints them and + // the build record keeps them; a refusal that says nothing cannot be + // told from a cache that does not work. + struct CacheDecision { + std::string name; + std::string version; + std::string reason; + }; + std::vector cacheDecisions; // What the dependency walk actually RESOLVED, keyed by the root manifest's // dependency map key. The "Compiling v" banner used to read diff --git a/src/build/prepare/dependency_cache.cpp b/src/build/prepare/dependency_cache.cpp index 2e25aefc6..115494845 100644 --- a/src/build/prepare/dependency_cache.cpp +++ b/src/build/prepare/dependency_cache.cpp @@ -10,6 +10,7 @@ import :state; import std; import mcpp.diag; import mcpp.home; +import mcpp.log; import mcpp.build.source_digest; import nlohmann.json; import mcpp.manifest; @@ -296,6 +297,16 @@ static std::expected package_keys(PrepareState& state, return out; } +// Why a package was not served, said once (design 2026-11-20 §5.4): kept for +// the build record and printed under --verbose. +static void not_served(BuildContext& ctx, const mcpp::modgraph::PackageRoot& pkg, + std::string_view reason, std::string_view why) { + const auto name = mcpp::build::qualified_package_name(pkg.manifest); + ctx.cacheDecisions.push_back({name, pkg.manifest.package.version, std::string(reason)}); + mcpp::log::verbose("cache", std::format("{} {} not served [{}]: {}", name, + pkg.manifest.package.version, reason, why)); +} + // One package's entry: its units are served from the cache when the entry holds // what this build asks for; otherwise a task fills it after the build. static void serve_or_fill(PrepareState& state, BuildContext& ctx, std::size_t i, const PackageKeys& keys) { @@ -325,9 +336,14 @@ static void serve_or_fill(PrepareState& state, BuildContext& ctx, std::size_t i, // resolved commit, or the digest of its tree's content, is in its // key (`sources_named`), so a change to the sources is a change of // key. The project's own trees are not named and stay out. - if (!depIdent || !sources_named(state, i)) return; + if (!depIdent) + return not_served(ctx, pkgRoot, "no-identity", "the resolution recorded no identity for it"); + if (!sources_named(state, i)) + return not_served(ctx, pkgRoot, "unnamed-sources", + "its sources are the project's, or have no version, commit or content digest"); // ...and neither may anything it was built against be unnamed. - if (keys.localTaint[i]) return; + if (keys.localTaint[i]) + return not_served(ctx, pkgRoot, "unnamed-upstream", "a package it is built against has unnamed sources"); // ...and the package's sources must ACTUALLY be in the immutable // store, not merely labelled as coming from it. // @@ -358,7 +374,8 @@ static void serve_or_fill(PrepareState& state, BuildContext& ctx, std::size_t i, // statement about the path, not about the inode. const bool fromStore = depIdent->sourceKind == "version"; if (fromStore && !mcpp::build::path_is_under_any(pkgRoot.root, state.storeRoots)) - return; + return not_served(ctx, pkgRoot, "outside-store", + std::format("its root {} is not below a store", pkgRoot.root.generic_string())); const auto& depName = depIdent->packageName; const auto depVer = (depIdent->version.empty() @@ -389,6 +406,7 @@ static void serve_or_fill(PrepareState& state, BuildContext& ctx, std::size_t i, // below a package nested in this one's tree is that package's. if (keys.unitPackage[u] != i) continue; auto& cu = ctx.plan.compileUnits[u]; + if (cu.entryUnit) continue; // compiled with the image that links it // ALL OR NOTHING. A unit plan.cppm could not give a // machine-independent entry address to takes its whole package @@ -415,7 +433,8 @@ static void serve_or_fill(PrepareState& state, BuildContext& ctx, std::size_t i, cu.object}); unitIdx.push_back(u); } - if (!addressable) return; + if (!addressable) + return not_served(ctx, pkgRoot, "unaddressable-unit", "a unit has no machine-independent address"); // Validate the entry against THIS build's artifact list, not // against the entry's own (mcpp#344). Anything short of a full @@ -468,6 +487,7 @@ static void serve_or_fill(PrepareState& state, BuildContext& ctx, std::size_t i, arts.bmiFiles.size() + arts.objFiles.size(), probe.layoutMismatch.front())); } + not_served(ctx, pkgRoot, "miss", std::format("no entry under key {} yet; it is filled after the build", key.keyHex)); for (auto u : unitIdx) ctx.plan.compileUnits[u].fillsCache = true; BuildContext::CacheTask task{ std::move(key), std::move(arts) }; // The tree is read again before its entry is written: a source diff --git a/src/build/prepare/package_root.cpp b/src/build/prepare/package_root.cpp index 56d5496e2..22fe48c68 100644 --- a/src/build/prepare/package_root.cpp +++ b/src/build/prepare/package_root.cpp @@ -10,81 +10,56 @@ import mcpp.build.prepare_inputs; import std; import mcpp.diag; +import mcpp.build.plan; import mcpp.manifest; -import mcpp.modgraph.glob; import mcpp.modgraph.graph; namespace mcpp::build { -static std::vector expandIncludeDirs( +// A package's include directories, each entry expanded as the root manifest's +// are (plan.cppm's expand_manifest_include_entry): an absolute entry in its +// native spelling (a TOML `C:/SDL2/include` stays mixed on MSVC and leaks into +// the CDB's -I otherwise, mcpp#230), a glob as the directories it matches, and +// a literal directory that does not exist yet as written, since a build step +// may create it (#390). One expansion serves `include_dirs`, +// `include_dirs_after` (#249: the -idirafter channel) and +// `private_include_dirs`, the resource compiler's search list and the root's +// compile steps, so a package's configuration has one set of directories +// whichever consumer reads it. +static std::vector expandEntries( PrepareState& state, const std::filesystem::path& packageRoot, - const mcpp::manifest::Manifest& manifest) + const std::vector& entries) { std::vector dirs; - for (auto const& inc : manifest.buildConfig.includeDirs) { - if (inc.is_absolute()) { - // Native spelling: a TOML `C:/SDL2/include` stays mixed on - // MSVC and leaks into the CDB's -I otherwise. Direct - // make_preferred — no generic_string round trip, which can - // throw for names the ANSI codepage cannot spell (mcpp#230). - auto n = inc; - n.make_preferred(); - state.appendUniquePath(dirs, std::move(n)); - continue; - } - for (auto& dir : mcpp::modgraph::expand_dir_glob( - packageRoot, inc.generic_string())) { - state.appendUniquePath(dirs, dir); - } - } + for (auto const& inc : entries) + for (auto& dir : expand_manifest_include_entry(packageRoot, inc)) + state.appendUniquePath(dirs, std::move(dir)); return dirs; } - // #249: same glob expansion for `include_dirs_after` (the -idirafter - // channel — searched after the toolchain's system dirs). +static std::vector expandIncludeDirs( + PrepareState& state, + const std::filesystem::path& packageRoot, + const mcpp::manifest::Manifest& manifest) +{ + return expandEntries(state, packageRoot, manifest.buildConfig.includeDirs); +} + static std::vector expandIncludeDirsAfter( PrepareState& state, const std::filesystem::path& packageRoot, const mcpp::manifest::Manifest& manifest) { - std::vector dirs; - for (auto const& inc : manifest.buildConfig.includeDirsAfter) { - if (inc.is_absolute()) { - auto n = inc; - n.make_preferred(); - state.appendUniquePath(dirs, std::move(n)); - continue; - } - for (auto& dir : mcpp::modgraph::expand_dir_glob( - packageRoot, inc.generic_string())) { - state.appendUniquePath(dirs, dir); - } - } - return dirs; + return expandEntries(state, packageRoot, manifest.buildConfig.includeDirsAfter); } - // The same expansion for `private_include_dirs`, so a private entry may be - // a glob and still name exactly the directories it expands to. static std::vector expandPrivateIncludeDirs( PrepareState& state, const std::filesystem::path& packageRoot, const mcpp::manifest::Manifest& manifest) { - std::vector dirs; - for (auto const& inc : manifest.buildConfig.privateIncludeDirs) { - if (inc.is_absolute()) { - auto n = inc; - n.make_preferred(); - state.appendUniquePath(dirs, std::move(n)); - continue; - } - for (auto& dir : mcpp::modgraph::expand_dir_glob( - packageRoot, inc.generic_string())) { - state.appendUniquePath(dirs, dir); - } - } - return dirs; + return expandEntries(state, packageRoot, manifest.buildConfig.privateIncludeDirs); } std::expected makePackageRoot( diff --git a/tests/e2e/test/skeleton.cpp b/tests/e2e/test/skeleton.cpp index 586cedd0e..deba406a3 100644 --- a/tests/e2e/test/skeleton.cpp +++ b/tests/e2e/test/skeleton.cpp @@ -47,6 +47,11 @@ E2E skeleton{"skeleton", [] { auto bt = in("prog", {"test"}); expect(e2e::succeeded(bt)); expect(s::has_line(s::text(bt), R"(test_smoke \.\.\. ok \(.*1 case\))")); + // The second project's test takes the framework from the global cache: its entry `main` + // compiles with the test, and its other units are the entry the first project filled + // (design 2026-11-20 §5.4; the entry without a cache address kept the package out). + expect(s::has_line(s::text(bt), R"(Cached mcpplibs\.mtest v[0-9.]+ \([0-9]+ units?\))")) + << "B: the second project compiled mtest again: " << s::text(bt); // C: a framework whose entry offers to begin tests. t.write("fw/mcpp.toml", R"([package] name = "fw" diff --git a/tests/e2e/workspace/inheritance.cpp b/tests/e2e/workspace/inheritance.cpp index aa7648927..eb5cf74ab 100644 --- a/tests/e2e/workspace/inheritance.cpp +++ b/tests/e2e/workspace/inheritance.cpp @@ -644,6 +644,32 @@ main = "src/main.cpp" expect(fatal(e2e::succeeded(ws.mcpp({"test", "-p", "app"})))) << "C: the build after the sweep failed"; }; + "986 a package keeps its instance when a command adds units"_test[s::gcc][s::elf] = [] { + // mcpp#986. `mcpp test` keeps a unit that an inactive feature gates and `mcpp build` drops + // it; the features and flags are the same. The instance is a function of the package's + // configuration, not of its units, so the test after a build compiles only what it adds: + // the gated unit and the test, not `a.cppm` again in a second instance directory. + const auto tmp = s::case_dir(); + s::project ws(tmp / "ws", { + {"mcpp.toml", "[package]\nname = \"lib\"\nversion = \"0.1.0\"\n\n[build]\nsources = [\"src/a.cppm\", \"src/b.cppm\"]\n\n" + "[features]\nb = { sources = [\"src/b.cppm\"] }\n\n[targets.lib]\nkind = \"lib\"\n\n[workspace]\nmembers = [\"app\"]\n"}, + {"src/a.cppm", "export module lib986.a;\nexport int a() { return 1; }\n"}, + {"src/b.cppm", "export module lib986.b;\nexport int b() { return 2; }\n"}, + {"tests/t.cpp", "import lib986.a;\nint main() { return a() == 1 ? 0 : 1; }\n"}, + {"app/mcpp.toml", "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[dependencies]\nlib = { path = \"..\" }\n\n[targets.app]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}, + {"app/src/main.cpp", "import lib986.a;\nint main() { return a() == 1 ? 0 : 1; }\n"}, + }); + expect(fatal(e2e::succeeded(ws.mcpp({"build"})))) << "the build failed"; + const auto objects = s::find_files(ws.at("target"), R"(a\.m\.o)"); + expect(fatal(objects.size() == 1u)) << "the build does not have one object of a.cppm: " << objects.size(); + const auto stamp = fs::last_write_time(objects.front()); + auto r = ws.mcpp({"test", "--verbose"}); + expect(fatal(e2e::succeeded(r))) << "the test failed: " << s::text(r); + expect(s::find_files(ws.at("target"), R"(a\.m\.o)").size() == 1u) << "a.cppm has a second instance under the test: " << s::text(r); + expect(fs::last_write_time(objects.front()) == stamp) << "a.cppm was compiled again under the test: " << s::text(r); + expect(s::find_files(ws.at("target"), R"(b\.m\.o)").size() == 1u) << "the gated unit the test keeps was not compiled: " << s::text(r); + }; + "954 the workspace's abi table is the root's statement"_test[s::gcc][s::elf] = [] { // mcpp#954. A workspace whose root is only `[workspace]` states the build's abi once, in // `[workspace.target..abi]`; its members inherit the row and no member is told From 2277ffca5550dc08aa0930968600a0d27766d211 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 01:52:17 +0900 Subject: [PATCH 06/38] feat: a test and a build share the machine's budget of slots and a memory threshold; an archive run selects case parts; SPEC-017 2.5 (#976, #927) --- .github/tools/check_version_pins.sh | 18 ++++- docs/specs/test.md | 40 ++++++++-- modules/manifest/src/key_registry.cppm | 2 + modules/manifest/src/toml.cppm | 2 + modules/manifest/src/types.cppm | 6 ++ src/build/execute.cppm | 87 ++++++++++++++++++++- src/build/ninja_backend.cppm | 19 +++++ src/build/plan.cppm | 3 + src/build/prepare/plan.cpp | 11 ++- src/build/schedule/compiler_transaction.cpp | 2 + src/build/schedule/detach_codegen.cppm | 82 ++++++++++++++++++- src/build/schedule/policy.cppm | 46 +++++++++++ src/build/test_protocol.cppm | 27 +++++-- src/config.cppm | 8 ++ src/home/files.cpp | 26 ++++++ src/home/files.cppm | 7 ++ tests/e2e/test/streams.cpp | 22 ++++++ 17 files changed, 388 insertions(+), 20 deletions(-) diff --git a/.github/tools/check_version_pins.sh b/.github/tools/check_version_pins.sh index 2238ba297..2505b736d 100755 --- a/.github/tools/check_version_pins.sh +++ b/.github/tools/check_version_pins.sh @@ -13,6 +13,8 @@ # # 3. The launchers a pack builds use mcpp's own mcpplibs.os release. # +# 4. The test skeleton of `mcpp new` uses mcpp's own mcpplibs.mtest release. +# # Why this exists: src/provider/xlings/xlings.cppm used to say "keep in lock-step with the # XLINGS_VERSION pins in release.yml / cross-build-test.yml / ci-linux-e2e.yml" # and that list was ALREADY incomplete — it omitted both composite actions, @@ -191,7 +193,21 @@ elif [ "$os_toml" != "$os_launcher" ]; then bad "src/pack/distribution.cppm builds the launchers on mcpplibs.os '$os_launcher', mcpp.toml builds mcpp on '$os_toml'" fi +# ── 4. A new project's tests use mcpp's own mtest ───────────────────────── +# +# `mcpp new` writes `mcpplibs.mtest = ""` (src/build/test_protocol.cppm), and mcpp's +# own tests use the mtest of mcpp.toml. A skeleton on an older release than mcpp tests itself with +# is a framework nobody here runs: 2026.11.20.1 scaffolded 0.9.0 while mcpp used 0.10.0 (mcpp#976). +mtest_toml=$(awk -F '"' '/^mcpplibs\.mtest[[:space:]]*=/{print $2; exit}' mcpp.toml) +mtest_skeleton=$(grep -oE 'kMtestRelease[[:space:]]*=[[:space:]]*"[^"]+"' src/build/test_protocol.cppm \ + | grep -oE '"[^"]+"' | tr -d '"' | head -1) +if [ -z "$mtest_toml" ] || [ -z "$mtest_skeleton" ]; then + bad "could not read mcpplibs.mtest from mcpp.toml ('$mtest_toml') or kMtestRelease from src/build/test_protocol.cppm ('$mtest_skeleton')" +elif [ "$mtest_toml" != "$mtest_skeleton" ]; then + bad "src/build/test_protocol.cppm scaffolds mcpplibs.mtest '$mtest_skeleton', mcpp.toml tests mcpp with '$mtest_toml'" +fi + if [ "$fail" = 0 ]; then - echo "OK: xlings pins all at $XLINGS_EXPECTED; building mcpp $v_toml, bootstrapping from $v_xl; launchers on mcpplibs.os $os_launcher" >&2 + echo "OK: xlings pins all at $XLINGS_EXPECTED; building mcpp $v_toml, bootstrapping from $v_xl; launchers on mcpplibs.os $os_launcher; skeleton on mcpplibs.mtest $mtest_skeleton" >&2 fi exit "$fail" diff --git a/docs/specs/test.md b/docs/specs/test.md index cc7d7ae17..6135478b9 100644 --- a/docs/specs/test.md +++ b/docs/specs/test.md @@ -4,16 +4,16 @@ |---|---| | 规范编号 | SPEC-017 | | 标题 | mcpp 测试规范:测试程序、测试协议、记录、存储、启动描述与框架声明;基础能力与可选能力 | -| 状态 | 评审中 v2.2 | -| 版本 | 2.2 | -| 最后修改 | 2026-11-14 | -| 对应实现 | 逐条标注。B1–B4 与 O1、O2、O4–O7 mcpp >= 2026.11.8.1 至 2026.11.10.1(逐条);B5、B6、O3 的运行器一侧、O8 mcpp >= 2026.11.10.1;O9 由框架的工具实现(mtest >= 0.6.0);O10 mcpp >= 2026.11.13.1 | +| 状态 | 评审中 v2.5 | +| 版本 | 2.5 | +| 最后修改 | 2026-11-21 | +| 对应实现 | 逐条标注。B1–B4 与 O1、O2、O4–O7 mcpp >= 2026.11.8.1 至 2026.11.10.1(逐条);B5、B6、O3 的运行器一侧、O8 mcpp >= 2026.11.10.1;O9 由框架的工具实现(mtest >= 0.6.0);O10 mcpp >= 2026.11.13.1;P5 的 `step` 与 `exercise`、B4 的 `builds.ndjson` 与机器的字段、O5 在选择之下的时限、O11、O12 mcpp >= @NEXT@,mtest >= 0.11.0 | | 取代 | SPEC-017 v1.0(`test-runner.md`);mtest 测试协议版本 1(mtest `spec/protocol.md`)的规范正文;SPEC-004 中 `[test]` 的键 | | 相关规范 | [SPEC-004](manifest-semantics.md)(清单的平面与命名)、[SPEC-006](build-database.md)(构建数据库) | | 相关 issue | mcpp#914、mcpp#915、mcpp#917 | | 使用文档 | [08 —— 测试](../zh/08-testing.md)、[50 —— 机器可读输出](../zh/50-machine-output.md) §8 | -用语按 RFC 2119:**必须 / 禁止**为强制,**应当**为建议,**可以**为可选。条款编号有三类:B1–B6 为基础能力,O1–O10 为可选能力,P1–P12 为协议条款(编号与 mtest 协议版本 1 相同,原有引用不变)。 +用语按 RFC 2119:**必须 / 禁止**为强制,**应当**为建议,**可以**为可选。条款编号有三类:B1–B6 为基础能力,O1–O12 为可选能力,P1–P12 为协议条款(编号与 mtest 协议版本 1 相同,原有引用不变)。 ## 0. 范围与分层 @@ -132,10 +132,13 @@ | `usage` | `user_ms`、`sys_ms`,可选 `peak_bytes`、`read_bytes`、`written_bytes`(O7) | 程序结束时 | | `artifact` | `id`、`path`、`kind` | 用例产生的文件 | | `fixture` | `id`、`name`、`ms` | 用例促成为共享它的用例而构造的对象,及构造耗时 | +| `step` | `id`、`name`、`t`(自程序启动起的毫秒)、`ms`,可选 `cpu_ms`、`status`(步骤运行的程序的退出状态)、`build`(该步骤是一次构建时,构建记录的标识) | 用例的一个步骤结束(mtest >= 0.11.0) | +| `exercise` | `id`、`items`:用例用到的产品面的项(字符串的列表,形式由产品给出,例如 `command build`、`flag build --target`、`key build.sources`、`reason build/cache`) | 用例运行期间,可多次(mtest >= 0.11.0) | | `end` | `id`、`status`、`ms`,可选 `reason`、`checks`(O3) | 用例结束 | | `missing` | `id` | P4.1 | | `done` | `status`、`counts`(状态到数目的对象) | 末个 | +- **P5.6 步骤与用到的面(mtest >= 0.11.0;mcpp >= @NEXT@)。** 一个用例由若干步骤组成时(端到端用例运行的每条命令),程序**可以**为每个步骤写一个 `step` 事件;运行器把它们按到达顺序归入用例。`exercise` 列出用例用到的产品面的项;运行器把一个用例的各次 `exercise` 合并为一个集合。二者都不影响用例的结论。 - **P5.3** 用例的 `status` 为以下之一:`pass`、`fail`、`skip`(带 `reason`)、`xfail`(如预期地失败)、`xpass`(预期失败却通过;运行失败)、`invalid`(无法判定的测量;P6.3)。 - **P5.4** 运行器**必须**忽略未知字段与未知事件种类。运行器**必须**接受没有 `done` 的报告(程序崩溃):有 `begin` 而无 `end` 的用例失败,以程序的退出状态为原因。 - **P5.5** 程序的退出状态为零,当且仅当没有用例失败或意外通过,且没有被选用例缺失。 @@ -168,11 +171,13 @@ | 路径 | 内容 | |---|---| | `index.ndjson` | 每次运行一行:去掉程序与用例的运行记录 | -| `runs//run.json` | 运行记录:`schema`(`mtest.run/1`)、`id`、`started`、`finished`、`status`、`system`、`arch`、`machine`、`commit`、`set`、`selection`、`group`、`monitor`、`ms`、`jobs`、`counts` | +| `runs//run.json` | 运行记录:`schema`(`mtest.run/1`)、`id`、`started`、`finished`、`status`、`system`、`arch`、`machine`、`commit`、`set`、`selection`、`group`、`monitor`、`ms`、`jobs`、`counts`;`processors`、`memory`、`budget`、`jitter`(@NEXT@) | | `runs//programs.ndjson` | 每个程序进程一行:`name`、`argv`、`status`、`exit`、`ms`、`started_ms`、`worker`、`select`、`resources`、`fixtures`,运行器构建它时的 `build_started_ms` 与 `build_ms`,`usage`(带 `source`:`program` 或 `engine`) | -| `runs//cases.ndjson` | 每个用例一行:`program`、`id`、`kind`、`status`、`ms`、`started_ms`、`reason`、`failures`、`metrics`、`bounds`、`tags`、`clauses` | +| `runs//cases.ndjson` | 每个用例一行:`program`、`id`、`kind`、`status`、`ms`、`started_ms`、`reason`、`failures`、`metrics`、`bounds`、`tags`、`clauses`;`cpu_ms`、`steps`(`step` 事件的字段,`id` 除外)、`exercised`(@NEXT@) | +| `runs//builds.ndjson` | 运行器为这次运行所做的构建(@NEXT@):每条运行过的边一行,`build`(构建记录的标识)、`output`、`package`、`program`(该边所服务的测试程序,可为空)、`started_ms`、`ms`、`cpu_ms`、`peak_bytes`;由缓存提供的包一行,`build`、`package`、`cached`(`true`)、`units` | | `runs//fixtures.ndjson` | 每个运行夹具一行:`name`、`program`、`started_ms`、`ready_ms`、`ended_ms`、`status` | +- **P12.4 机器(@NEXT@)。** `machine` 是机器的类别:处理器的型号,系统不给出型号时为 `/<逻辑处理器数>`;`processors` 为逻辑处理器数,`memory` 为物理内存的字节数。比较不同运行的用时(O10 的估计、界限、趋势)时,读者**应当**按 `machine` 分组;没有同类的历史时取全部。处理器时间(`cpu_ms`)在同类机器之间可比,墙钟只在同一次运行之内可比。 - **P12.2** 运行标识按时间排序。同一 `group` 的运行(一次 CI 运行的各作业)是一个整体的各部分,读者合并报告。 - **P12.3** 一次运行中程序、用例、构建与夹具的时间构成其工作的有向无环图;读者的关键路径是按实测时间加权的最长路径。 - **mcpp 的实现。** 存储位于 `[test] store`(缺省 `target/mtest`;`"none"` 不保存)。`run.json` 的 `commit` 与 `group` 在 CI 中取 `GITHUB_SHA` 与 `GITHUB_RUN_ID`;分片运行(O10)另写 `shard`(`I/N`)。程序进程一行在其为一部分时写 `part_of`(mcpp >= 2026.11.13.1)。mcpp 在一次构建中构建全部测试程序,不写逐个程序的 `build_started_ms`。 @@ -218,6 +223,8 @@ | O8 | `output` | 每个用例的输出归属 | 框架写出 `output` 事件;mcpp 归属与转交 | 程序的输出整体显示 | | O9 | `session` | 测试会话:运行、加入、取消、状态 | 框架的工具 | 客户端以多个并发的 `mcpp test` 进程运行 | | O10 | `measured-schedule` | 按存储测得的用时排序、划分与分片,重跑失败部分 | mcpp | 按列出顺序运行,不分片 | +| O11 | `budget` | 一台机器一份处理器预算:运行器与被运行的程序启动的并行工作从同一份槽位中取得 | 运行器建立;任何程序遵守 | 每个程序按自己的并行度运行 | +| O12 | `jitter` | 在竞争点等待随机的时间,使未被固定的次序显现 | 运行器陈述;框架与被测程序遵守 | 不等待 | ### O1. 各部分 已实现(mcpp >= 2026.11.10.1) @@ -256,6 +263,7 @@ - **P7.2** `artifacts` 写出用例所读之物:`target:<名>`、`file:<路径>`、`program:<名>`;运行器据此按变更选择用例。 - **P7.3** 资源**可以**有容量:组件带来的池陈述同时持有它的用例数;运行器**禁止**超过容量。 - 为每个用例启动进程的运行器在用例的 `timeout` 时结束该进程;整体运行程序的运行器(`mcpp test`)运行其中每个用例,按报告判定。 +- **时限只陈述一次(@NEXT@)。** 用例的时限是它与其所属各级作用域声明的 `timeout` 中最小者。运行器选择一个用例或其下声明的用例(P3.4、P4)时,**必须**取这些声明,而不是运行的缺省;没有任何声明时取运行的缺省。 ### O6. 组件与运行夹具(P11) @@ -318,6 +326,20 @@ 4. **分片。** `--shard I/N` 把每个测试按估计从长到短(相同时按名字)分配给已分配估计之和最小的分片(相同时取编号最小者),只运行第 I 个分片的测试。分配**必须**只取决于存储与本次运行的测试列表。分片**禁止**把本组(`GITHUB_RUN_ID`,无则为空)的分片运行读作历史,使同一存储上先后启动的各分片读到相同的历史。 5. **失败重跑。** `--failed` 读取存储中最近的一次运行:状态为 `fail` 或 `build-fail` 的程序记录与状态为 `fail` 或 `xpass` 的用例。所选程序为这些记录的 `part_of`(无则为其名字);每个失败的程序都有失败用例时,只运行这些用例(与 `--case` 相同);否则整体运行这些程序。最近一次运行没有失败时,运行器不运行任何测试,退出码为 0。 +### O11. 预算(@NEXT@;mtest >= 0.11.0) + +1. **形式。** 预算是一个目录,其中有与槽位数相同的文件 `slot-`(`i` 自 0 起)。持有一个文件的独占锁即持有一个槽位;进程结束时锁随之释放,因此崩溃不会遗失槽位。 +2. **陈述。** 建立预算的运行器在它启动的程序的环境中陈述 `MTEST_BUDGET`(目录)与 `MTEST_BUDGET_HELD`(运行器为该程序持有的槽位文件)。程序以持有的槽位运行它的第一份并行工作;其余每一份并行工作(一个编译器、一个子进程)在开始前锁住一个空闲的槽位文件,结束后释放。没有空闲的槽位时等待。 +3. **传递。** 程序把这两个变量原样交给它启动的程序:被启动的程序可以使用预算,但**禁止**使用父程序持有的槽位,除非父程序在此期间不使用它(例如等待一个它启动的构建)。 +4. **不认识它的程序不受影响。** 预算只约束遵守它的程序;运行器以槽位数作为它自己的并行度。 +5. **mcpp 的实现。** `mcpp test` 与 `mtest run` 建立预算,槽位数为其并行度;mcpp 的编译执行器在环境陈述了预算时,启动编译器之前先取预算的槽位,再取本次构建的槽位,并在取得之后、启动之前读取可用内存:低于 `[build] memory_threshold`(SPEC-004)时等待已运行的编译器结束,不结束它们。 + +### O12. 抖动(@NEXT@;mtest >= 0.11.0) + +1. 运行器**可以**陈述 `MTEST_JITTER`(毫秒,上限)与 `MTEST_JITTER_SEED`(十进制)。程序在其声明的竞争点等待一段 [0, 上限] 内的随机时间,随机数由种子与竞争点的序号决定,同一种子给出同一序列。 +2. 框架的竞争点:用例开始之前、共享对象构造之前;被测程序的竞争点由其自身声明(mcpp:取目录锁与发布产物之前)。 +3. 运行记录写出 `jitter`:`{max_ms, seed}`。抖动下的运行**禁止**作为合入或发布的验证,其结论只用于定位次序缺陷。 + ## 4. 宣告 mcpp 以 feature(docs/50 §4)宣告它实现的条款;客户端按需门控。 @@ -333,6 +355,9 @@ mcpp 以 feature(docs/50 §4)宣告它实现的条款;客户端按需门 | `test-frameworks` | B6 的 `capabilities` 与 `session` | 2026.11.10.1 | | `test-checks` | O3 的运行器一侧 | 2026.11.10.1 | | `test-output` | O8 | 2026.11.10.1 | +| `test-steps` | P5.6 的运行器一侧;B4 的 `steps`、`exercised` 与 `builds.ndjson` | @NEXT@ | +| `test-budget` | O11 | @NEXT@ | +| `test-jitter` | O12 的运行器一侧 | @NEXT@ | 框架实现的可选能力由 B6 宣告,不由 mcpp 的 feature 宣告。 @@ -358,4 +383,5 @@ mcpp 以 feature(docs/50 §4)宣告它实现的条款;客户端按需门 | 2.1 | 2026-11-14 | P3.3 的位置与能力(`only`、`except`、`capabilities`、列为跳过时的 `reason`;mtest 0.8.0),及 mcpp 的实现 | | 2.2 | 2026-11-14 | O10 按测量调度(顺序、`split = "auto"`、`--shard`、`--failed`);B4 的 `shard` 与 `part_of` | | 2.4 | 2026-11-17 | B1 第 6 条 `[test] files`;B5 的 `cwd` 为已备好的工作目录(mcpp 2026.11.17.1,设计 2026-11-16 §5) | +| 2.5 | 2026-11-21 | P5.6 的 `step` 与 `exercise`;B4 的 `builds.ndjson`、`cpu_ms`、`steps`、`exercised` 与机器的字段(P12.4);O5 的时限只陈述一次;O11 预算;O12 抖动(设计 2026-11-20 §5.5 至 §5.9、§5.16、§5.19) | | 2.3 | 2026-11-15 | P4.3 排除与按声明选择(mtest 0.9.0);O10 按声明的用例划分并以测量判定是否保留(`divides`),用例的用时按程序与标识记取 | diff --git a/modules/manifest/src/key_registry.cppm b/modules/manifest/src/key_registry.cppm index 4797fc6fa..397d40661 100644 --- a/modules/manifest/src/key_registry.cppm +++ b/modules/manifest/src/key_registry.cppm @@ -50,6 +50,8 @@ inline constexpr KeyClass kKeyRegistry[] = { {"build", "accel", KeyScope::Configuration, KeyMerge::Scalar, false, false}, {"build", "allow_host_libs", KeyScope::Package, KeyMerge::Scalar, false, false}, {"build", "bmi_schedule", KeyScope::Configuration, KeyMerge::Scalar, false, false}, + // The memory a compiler may start below (SPEC-004 §9, design 2026-11-20 §5.5, mcpp#802). + {"build", "memory_threshold", KeyScope::Configuration, KeyMerge::Scalar, false, false}, {"build", "build_program_timeout", KeyScope::Package, KeyMerge::Scalar, false, false}, {"build", "c_standard", KeyScope::Package, KeyMerge::Scalar, true, false}, {"build", "cache", KeyScope::Configuration, KeyMerge::Scalar, false, false}, diff --git a/modules/manifest/src/toml.cppm b/modules/manifest/src/toml.cppm index 522ba9c98..117b7d3d8 100644 --- a/modules/manifest/src/toml.cppm +++ b/modules/manifest/src/toml.cppm @@ -3219,6 +3219,8 @@ std::expected parse_document(const t::Document& documen // the whole manifest unloadable. (A published package carrying an unknown // key must never break an older mcpp — same rule the dependency keys follow.) if (auto v = doc->get_string("build.bmi_schedule")) m.buildConfig.bmiSchedule = *v; + if (auto v = doc->get_string("build.memory_threshold")) m.buildConfig.memoryThreshold = *v; + else if (auto n = doc->get_int("build.memory_threshold")) m.buildConfig.memoryThreshold = std::to_string(*n); if (auto v = doc->get_string("build.private_fragment")) m.buildConfig.privateFragment = *v; if (auto v = doc->get_string("build.jobs")) m.buildConfig.jobs = *v; else if (auto n = doc->get_int("build.jobs")) m.buildConfig.jobs = std::to_string(*n); diff --git a/modules/manifest/src/types.cppm b/modules/manifest/src/types.cppm index 7fa839d52..13b6dcc4e 100644 --- a/modules/manifest/src/types.cppm +++ b/modules/manifest/src/types.cppm @@ -1029,6 +1029,12 @@ struct BuildConfig : BuildInputs { // compiler doing the build, and resolving it at parse time would freeze one // machine's answer into a manifest that travels. std::string bmiSchedule; + // `[build] memory_threshold` -- the memory the system must have available + // before a compiler starts ("2GiB", "512MiB", or bytes; design 2026-11-20 + // §5.5, mcpp#802). Below it the compiler waits for one already running to + // end; none is ended. Text, resolved with MCPP_MEMORY_THRESHOLD, the + // machine's config.toml and the engine's default (1 GiB) where it is used. + std::string memoryThreshold; // `[build] private_fragment` -- how a private module fragment is built // (SPEC-019): "unit" (the default: the fragment is a unit of its own, so an // edit of it reaches no importer) or "as-written". Text, validated where it diff --git a/src/build/execute.cppm b/src/build/execute.cppm index 3695fc082..0d5c32bc2 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -38,6 +38,7 @@ import mcpp.buildopt.keep; import mcpp.build.dirlock; // the build directory's lock (mcpp#839) import mcpp.build.graph_engine; // the engine a graph names (2026-11-12 §17) import mcpplibs.os.process; // the identity of this invocation (its protocol directory) +import mcpplibs.os.fs; // the budget's slot locks (SPEC-017 O11) import mcpp.build.result; // the result record and the event stream (design 2026-10-26 §5) import mcpp.wire; // the mcpp.launch envelope import mcpp.build.cxx_runtime; // the island, verified after the link (design 2026-10-26 §6.3) @@ -2617,6 +2618,8 @@ std::optional run_ninja_fast(const std::string& ninjaProgram, if (!opened) { mcpp::ui::error(opened.error()); return 1; } compilerRun.emplace(std::move(*opened)); compilerRun->environment(childEnv); + childEnv.emplace_back("MCPP_MEMORY_THRESHOLD", + std::to_string(mcpp::build::read_memory_threshold_sidecar(outputDir))); } const auto compilerStopped=[&] {return compilerRun&&compilerRun->failed();}; @@ -5600,7 +5603,53 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, std::atomic next{0}; std::mutex reportMutex; + // ONE BUDGET FOR THE MACHINE (SPEC-017 O11, design 2026-11-20 §5.5): a + // directory of a slot file per job of the run; each worker holds one + // while it runs tests and states it to them (MTEST_BUDGET, + // MTEST_BUDGET_HELD), so a build a test starts runs its first compiler + // on that slot and takes the others from the slots no worker holds. + // Four shards' tests each building in parallel queued 7 to 9 processes + // on 4 processors, and a step took 2.3 times its time alone (§3.1). + // Inside another runner's budget (a test that runs `mcpp test`) that + // one is used: the first worker has the slot held for this process. + const auto outerBudget = mcpp::invocation::get("MTEST_BUDGET"); + const bool nestedBudget = outerBudget && !outerBudget->empty(); + const auto outerHeld = mcpp::invocation::get("MTEST_BUDGET_HELD").value_or(""); + std::filesystem::path budgetDir; + if (nestedBudget) { + budgetDir = mcpp::home::from_utf8(*outerBudget); + } else { + std::error_code ec; + budgetDir = std::filesystem::temp_directory_path(ec) + / std::format("mcpp-budget-{}-{}", mcpplibs::os::process::self() ? mcpplibs::os::process::self()->id : 0, + std::chrono::steady_clock::now().time_since_epoch().count()); + std::filesystem::create_directories(budgetDir, ec); + for (int i = 0; i < std::max(1, runJobsHere); ++i) + std::ofstream(budgetDir / std::format("slot-{}", i), std::ios::binary); + } + struct OwnedBudget { + std::filesystem::path dir; + ~OwnedBudget() { std::error_code ec; if (!dir.empty()) std::filesystem::remove_all(dir, ec); } + } ownedBudget{nestedBudget ? std::filesystem::path{} : budgetDir}; + auto take_slot = [&](int workerId) -> std::pair> { + if (nestedBudget && workerId == 0 && !outerHeld.empty()) return {outerHeld, std::nullopt}; + std::vector slots; + std::error_code ec; + for (const auto& e : std::filesystem::directory_iterator(budgetDir, ec)) + if (e.path().filename().string().starts_with("slot-") && mcpp::home::spelling(e.path()) != outerHeld) + slots.push_back(e.path()); + std::ranges::sort(slots); + for (;;) { + for (const auto& f : slots) + if (auto l = mcpplibs::os::fs::lock::acquire(f, mcpplibs::os::fs::lock::kind::exclusive, false)) + return {mcpp::home::spelling(f), std::move(*l)}; + if (slots.empty()) return {{}, std::nullopt}; + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + }; + auto worker = [&](int workerId) { + auto [slotFile, slotLock] = take_slot(workerId); for (;;) { std::size_t i = next.fetch_add(1); if (i >= list.size()) return; @@ -5659,6 +5708,10 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, // it, stated for this run, and no other program (P1.2, P2.2). mcpp::run::Options runOpts; runOpts.env = r.env; + if (!slotFile.empty()) { + runOpts.env.emplace_back("MTEST_BUDGET", mcpp::home::spelling(budgetDir)); + runOpts.env.emplace_back("MTEST_BUDGET_HELD", slotFile); + } runOpts.cwd = r.cwd.string(); runOpts.input = mcpp::run::Input::inherit; runOpts.streams = capture ? mcpp::run::Streams::merged : mcpp::run::Streams::inherit; @@ -6240,12 +6293,33 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, continue; } std::string lines; - for (const auto& id : mcpp::build::test_protocol::selection(report.listed, testOpts.cases)) { + // The cases a run of the store measured under this program: what a glob under a + // listed case names (P4.3). + std::map selectedSecs; // listed case -> its declared time + std::vector declared; + { + const auto prefix = mcpp::build::test_schedule::case_key(r.name, ""); + for (auto it = history.cases.lower_bound(prefix); + it != history.cases.end() && it->first.starts_with(prefix); ++it) + declared.push_back(it->first.substr(prefix.size())); + } + for (const auto& id : mcpp::build::test_protocol::selection(report.listed, testOpts.cases, + declared, report.selectsWithin)) { lines += id + "\n"; // A selected case that holds the machine (P7.1) runs the program alone. for (const auto& c : report.listed) if (c.id == id && std::ranges::find(c.resources, "machine") != c.resources.end()) r.alone = true; + // THE TIME LIMIT IS THE ONE ITS SCOPE STATES (SPEC-017 O5, design 2026-11-20 §5.8): a + // case selected under a listed case runs within that case's `timeout`, as the whole + // case does; `--case 'sets/*'` ran within the run's 300 s (mcpp#976). + const std::string_view bare = id.starts_with('?') ? std::string_view(id).substr(1) : std::string_view(id); + for (const auto& c : report.listed) + if (c.timeout > 0 && (bare == c.id || (bare.size() > c.id.size() && bare.starts_with(c.id) + && bare[c.id.size()] == '/'))) + selectedSecs.emplace(c.id, c.timeout); } + for (const auto& [_, secs] : selectedSecs) r.declaredSecs += secs; + selectedSecs.clear(); if (lines.empty()) { ++notSelected; continue; } r.select = r.protocolDir / "select.txt"; std::ofstream(*r.select, std::ios::binary) << lines; @@ -6819,7 +6893,13 @@ export int run_test_archive(std::span passthrough, BuildOverr // workers, the store (SPEC-015 §4; docs/08). std::vector runs; for (auto const& t : leg.tests) { - if (!selected_by(testOpts.filters, t.name)) continue; + // As a build's run selects: the program, or a part of it (`e2e/openkal/trees`), whose + // parts the run divides and selects after listing; an archive does not carry the + // files a part is named by (design 2026-11-20 §5.14). + const bool names_part = std::ranges::any_of(testOpts.filters, [&](const std::string& f) { + return f.starts_with(t.name + "/"); + }); + if (!names_part && !selected_by(testOpts.filters, t.name)) continue; TestRunnable r; r.name = t.name; r.program = (opened->root / t.program).lexically_normal(); @@ -6877,6 +6957,9 @@ export int run_test_archive(std::span passthrough, BuildOverr "\"not_reached\":{}}}}}", passed, failed, notRun, notReached); std::fflush(libc::stdout()); } + // `--cases` lists the archive's cases and runs none: a listing is its answer (design 2026-11-20 + // §5.14; through 2026.11.20.1 it ended "no test ran"). + if (testOpts.listCases) return notRun ? 2 : 0; if (passed + failed == 0) { std::string why; for (auto const& r : reasons) why += "\n " + r; diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index 032e3813d..c6db362e2 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -347,6 +347,11 @@ read_consumer_include_sidecar(const std::filesystem::path& outputDir); // graph runs no compiler through the capped executor; 0 when absent. void write_compiler_cap_sidecar(const std::filesystem::path& outputDir, int cap); int read_compiler_cap_sidecar(const std::filesystem::path& outputDir); +// The memory threshold the plan resolved (design 2026-11-20 §5.5), beside the +// cap: the fast path replays the graph without a plan and states it to the +// executors as the full path does. +void write_memory_threshold_sidecar(const std::filesystem::path& outputDir, std::uint64_t bytes); +std::uint64_t read_memory_threshold_sidecar(const std::filesystem::path& outputDir); // #693 (M4): whether a Ninja reads build.ninja in the encoding mcpp writes it. // @@ -1282,6 +1287,18 @@ int read_compiler_cap_sidecar(const std::filesystem::path& outputDir) { return f >> cap && cap > 0 ? cap : 0; } +constexpr std::string_view kMemoryThresholdSidecar = ".mcpp-memory-threshold"; + +void write_memory_threshold_sidecar(const std::filesystem::path& outputDir, std::uint64_t bytes) { + std::ofstream(outputDir / kMemoryThresholdSidecar, std::ios::binary | std::ios::trunc) << bytes << '\n'; +} + +std::uint64_t read_memory_threshold_sidecar(const std::filesystem::path& outputDir) { + std::ifstream f(outputDir / kMemoryThresholdSidecar, std::ios::binary); + std::uint64_t bytes = 0; + return f >> bytes ? bytes : 0; +} + std::vector read_consumer_include_sidecar(const std::filesystem::path& outputDir) { std::vector dirs; @@ -4901,6 +4918,7 @@ std::expected NinjaBackend::build(const BuildPlan& plan : std::vector{}); write_consumer_include_sidecar(plan.outputDir, root_include_dirs_of(plan)); write_compiler_cap_sidecar(plan.outputDir, plan.scheduleCompilerCap); + write_memory_threshold_sidecar(plan.outputDir, plan.memoryThreshold); write_graph_c_library_sidecar(plan.outputDir, plan.targetSide.cAbi); std::string placements; mcpp::build::progress::Attribution attribution; @@ -5252,6 +5270,7 @@ std::expected NinjaBackend::build(const BuildPlan& plan for (auto& ev : plan.toolchain.envOverrides) nenv.emplace_back(ev.key, ev.value); if (compilerRun) compilerRun->environment(nenv); + nenv.emplace_back("MCPP_MEMORY_THRESHOLD", std::to_string(plan.memoryThreshold)); const auto compilerStopped=[&] {return compilerRun&&compilerRun->failed();}; // The directory's BMI cutoff for this run (mcpp.buildopt.keep). A graph // compiled with contracts is compared exactly: a contract's violation diff --git a/src/build/plan.cppm b/src/build/plan.cppm index 6dffa9c6a..fb84e03aa 100644 --- a/src/build/plan.cppm +++ b/src/build/plan.cppm @@ -585,6 +585,9 @@ struct BuildPlan { // frontier starves — see the hazard note in schedule/detach_codegen. int scheduleNinjaJobs = 0; int scheduleCompilerCap = 0; + // The memory a compiler may start below (design 2026-11-20 §5.5), in bytes: + // the compiler executor waits below it (MCPP_MEMORY_THRESHOLD in its env). + std::uint64_t memoryThreshold = 0; // One immutable snapshot selected before workspace member substitution. // Build/run/test and cache fast paths consume this value; none may re-read // xlings active/current state. diff --git a/src/build/prepare/plan.cpp b/src/build/prepare/plan.cpp index b60fa7a6b..34517f85d 100644 --- a/src/build/prepare/plan.cpp +++ b/src/build/prepare/plan.cpp @@ -1016,9 +1016,18 @@ static void step13_graph_and_schedule(PrepareState& state, BuildContext& ctx) { // `requireBootstrap=false` because nothing here needs the bootstrap // toolchain. int globalDefaultJobs = 0; - if (auto c = state.get_cfg(/*requireBootstrap=*/false)) + std::string machineMemoryThreshold; + if (auto c = state.get_cfg(/*requireBootstrap=*/false)) { globalDefaultJobs = static_cast((*c)->defaultJobs); + machineMemoryThreshold = (*c)->memoryThreshold; + } ctx.globalDefaultJobs = globalDefaultJobs; + ctx.plan.memoryThreshold = mcpp::build::schedule::resolve_memory_threshold( + ctx.manifest, machineMemoryThreshold, [](std::string_view bad) { + mcpp::ui::warning(std::format( + "ignoring invalid memory threshold '{}' (expected a size such as \"2GiB\" or bytes); " + "the engine's 1 GiB holds", bad)); + }); ctx.intent = state.overrides.intent; // Resolve the module-edge schedule ONCE, here, where both the toolchain and diff --git a/src/build/schedule/compiler_transaction.cpp b/src/build/schedule/compiler_transaction.cpp index 8e30c7763..33441c846 100644 --- a/src/build/schedule/compiler_transaction.cpp +++ b/src/build/schedule/compiler_transaction.cpp @@ -1,5 +1,6 @@ module mcpp.build.schedule.compiler_transaction; import std; +import mcpp.home.files; // contention_point (SPEC-017 O12) namespace mcpp::build::schedule::transaction { @@ -46,6 +47,7 @@ bool replace_record(const fs::path& path,std::string_view bytes,std::string* dia std::error_code ec;fs::remove(temporary,ec);return false; } bool publish(const fs::path& path,std::string_view tag,const Fields& fields) { + mcpp::home::contention_point(); return replace_record(path,pack(tag,fields)); } namespace { diff --git a/src/build/schedule/detach_codegen.cppm b/src/build/schedule/detach_codegen.cppm index 19380cf2b..c170e19d7 100644 --- a/src/build/schedule/detach_codegen.cppm +++ b/src/build/schedule/detach_codegen.cppm @@ -75,6 +75,7 @@ import mcpplibs.os.env; import mcpplibs.os.platform; import mcpplibs.os.stdio; import mcpplibs.os.text; +import mcpplibs.os.machine; import mcpp.build.depfile; import mcpp.build.command_line; import mcpp.run; @@ -386,7 +387,77 @@ bool write_rc(const std::filesystem::path& slot, int rc,std::string* diagnostic= struct Permit { std::optional lock; std::filesystem::path reservation; + // The machine's budget slot (SPEC-017 O11), held for the compile, taken + // before the generation's slot; empty when the slot is the one the runner + // holds for this process, or no budget is stated. + std::optional budget; + std::optional heldClaim; }; + +// ONE BUDGET FOR THE MACHINE (SPEC-017 O11, design 2026-11-20 §5.5). A runner +// that states MTEST_BUDGET holds a slot file of that directory for the process +// it started (MTEST_BUDGET_HELD); the first compiler of this build runs on it, +// and every other compiler locks a free slot file before it starts. Four test +// processes each starting a parallel build put 7 to 9 processes in the run +// queue of 4 processors (measured, design §3.1): the work of a slot that is +// not free waits instead. +std::expected take_budget_slot(Permit& permit, const std::filesystem::path& permits) { + const auto budget = mcpp::invocation::get("MTEST_BUDGET"); + if (!budget || budget->empty()) return {}; + const auto held = mcpp::invocation::get("MTEST_BUDGET_HELD").value_or(""); + const std::filesystem::path dir = mcpp::home::from_utf8(*budget); + // The held slot is claimed by one compiler of the build at a time. + if (!held.empty()) { + auto claim = mcpplibs::os::fs::lock::acquire(permits / "budget-held.lock", + mcpplibs::os::fs::lock::kind::exclusive, false); + if (claim) { permit.heldClaim.emplace(std::move(*claim)); return {}; } + } + std::vector slots; + std::error_code ec; + for (const auto& e : std::filesystem::directory_iterator(dir, ec)) + if (e.path().filename().string().starts_with("slot-") + && mcpp::home::spelling(e.path()) != held) + slots.push_back(e.path()); + if (slots.empty()) return {}; // a budget of one slot: the held one, claimed by another compiler + std::ranges::sort(slots); + for (;;) { + for (const auto& f : slots) { + auto slot = mcpplibs::os::fs::lock::acquire(f, mcpplibs::os::fs::lock::kind::exclusive, false); + if (slot) { permit.budget.emplace(std::move(*slot)); return {}; } + } + // The held slot may have been released by the compiler that claimed it. + if (!held.empty()) { + auto claim = mcpplibs::os::fs::lock::acquire(permits / "budget-held.lock", + mcpplibs::os::fs::lock::kind::exclusive, false); + if (claim) { permit.heldClaim.emplace(std::move(*claim)); return {}; } + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } +} + +// THE MEMORY A COMPILER MAY START BELOW (design 2026-11-20 §5.5, mcpp#802): read +// after the slots are held, before the compiler starts. Below the threshold the +// compiler waits for memory a running compiler gives back; none is ended. A +// minute without enough is said and the compiler starts: waiting for memory no +// compiler of this machine holds would wait forever. +void await_memory() { + std::uint64_t threshold = 0; + if (const auto v = mcpp::invocation::get("MCPP_MEMORY_THRESHOLD")) + std::from_chars(v->data(), v->data() + v->size(), threshold); + if (threshold == 0) return; + const auto started = std::chrono::steady_clock::now(); + for (;;) { + const auto available = mcpplibs::os::machine::memory_available(); + if (!available || *available >= threshold) return; + if (std::chrono::steady_clock::now() - started > std::chrono::minutes(1)) { + std::println(std::cerr, "mcpp: {} MiB available, under the memory threshold of {} MiB for a " + "minute; the compiler starts ([build] memory_threshold)", + *available >> 20, threshold >> 20); + return; + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } +} std::expected acquire_permit(const Actor& actor, const std::filesystem::path& requested, int requestedCap) { const auto dir = actor.owner.permits(requested); @@ -401,7 +472,9 @@ acquire_permit(const Actor& actor, const std::filesystem::path& requested, int r if (!actor.ready) return std::unexpected(130); if (dir.empty() || cap <= 0) { if (actor.owner.managed) return std::unexpected(2); - return Permit{}; + Permit free; + await_memory(); + return free; } std::error_code ec; std::filesystem::create_directories(dir, ec); @@ -409,6 +482,8 @@ acquire_permit(const Actor& actor, const std::filesystem::path& requested, int r std::println(std::cerr, "mcpp: cannot create compiler semaphore {}: {}", dir.string(), ec.message()); return std::unexpected(1); } + Permit budgeted; + if (auto took = take_budget_slot(budgeted, dir); !took) return std::unexpected(took.error()); const auto started = std::chrono::steady_clock::now(); constexpr auto kLimit = std::chrono::hours(2); std::optional unexplainedSince; @@ -425,7 +500,10 @@ acquire_permit(const Actor& actor, const std::filesystem::path& requested, int r if (!available) { std::println(std::cerr,"mcpp: {}",available.error()); return std::unexpected(1); } if (!*available) { orphan = true; continue; } } - return Permit{std::optional{std::move(*permit)},reservation}; + budgeted.lock.emplace(std::move(*permit)); + budgeted.reservation = reservation; + await_memory(); + return budgeted; } // On macOS a slot another compiler holds is now and then reported // "not found" from inside this executor, though the directory and diff --git a/src/build/schedule/policy.cppm b/src/build/schedule/policy.cppm index e8a5d6f3e..491712f8c 100644 --- a/src/build/schedule/policy.cppm +++ b/src/build/schedule/policy.cppm @@ -67,6 +67,21 @@ int resolve_jobs(const manifest::Manifest& m, const std::function& onInvalid = {}, int globalDefault = 0); +// THE MEMORY A COMPILER MAY START BELOW (design 2026-11-20 §5.5, mcpp#802). The +// compiler executor reads the system's available memory after it holds its +// slot and before it starts the compiler, and waits while it is below this many +// bytes for a compiler already running to end; none is ended. A module compile +// peaks at 0.5 to 1.0 GB (measured above), so the engine's default is 1 GiB. +// +// Precedence as every default's (AGENTS.md §2): MCPP_MEMORY_THRESHOLD > +// `[build] memory_threshold` > the machine's config.toml `[build] +// memory_threshold` > 1 GiB. A size ("2GiB", "512MiB", "1.5GB") or bytes; "0" +// waits for nothing. +inline constexpr std::uint64_t kDefaultMemoryThreshold = std::uint64_t{1} << 30; +std::optional parse_size(std::string_view text); +std::uint64_t resolve_memory_threshold(const manifest::Manifest& m, std::string_view machine, + const std::function& onInvalid = {}); + // `requested` is the user's switch: "auto" (default), "on", "off". `hostJobs` is // the already-resolved parallelism (`--jobs` or `[build] jobs`), or 0 meaning // "the user said nothing, leave the backend's own default". @@ -134,4 +149,35 @@ int resolve_jobs(const manifest::Manifest& m, return 0; } +std::optional parse_size(std::string_view text) { + while (!text.empty() && text.front() == ' ') text.remove_prefix(1); + while (!text.empty() && text.back() == ' ') text.remove_suffix(1); + double number = 0; + const auto [end, ec] = std::from_chars(text.data(), text.data() + text.size(), number); + if (ec != std::errc{} || number < 0) return std::nullopt; + std::string_view unit(end, text.data() + text.size() - end); + while (!unit.empty() && unit.front() == ' ') unit.remove_prefix(1); + static constexpr std::pair kUnits[] = { + {"", 1}, {"B", 1}, {"KiB", 1024.0}, {"MiB", 1024.0 * 1024}, {"GiB", 1024.0 * 1024 * 1024}, + {"KB", 1e3}, {"MB", 1e6}, {"GB", 1e9}}; + for (const auto& [name, scale] : kUnits) + if (unit == name) return static_cast(number * scale); + return std::nullopt; +} + +std::uint64_t resolve_memory_threshold(const manifest::Manifest& m, std::string_view machine, + const std::function& onInvalid) { + auto from_text = [&](std::string_view v) -> std::optional { + if (v.empty()) return std::nullopt; + if (auto n = parse_size(v)) return n; + if (onInvalid) onInvalid(v); + return std::nullopt; + }; + if (auto e = mcpp::invocation::get("MCPP_MEMORY_THRESHOLD")) + if (auto n = from_text(*e)) return *n; + if (auto n = from_text(m.buildConfig.memoryThreshold)) return *n; + if (auto n = from_text(machine)) return *n; + return kDefaultMemoryThreshold; +} + } // namespace mcpp::build::schedule diff --git a/src/build/test_protocol.cppm b/src/build/test_protocol.cppm index b65ff8173..57b3bfd7d 100644 --- a/src/build/test_protocol.cppm +++ b/src/build/test_protocol.cppm @@ -18,7 +18,7 @@ inline constexpr int kVersion = 1; // The framework mcpp scaffolds tests with and suggests where a package has // none (docs/01, docs/08): mtest, by this release, from its repository. -inline constexpr std::string_view kMtestRelease = "0.9.0"; +inline constexpr std::string_view kMtestRelease = "0.10.0"; inline constexpr std::string_view kMtestRepo = "https://github.com/mcpp-community/mtest"; // The protocol's variables, removed from the environment a program inherits @@ -173,13 +173,19 @@ variables(std::string_view mode, const std::filesystem::path& report, // it equals or the identifiers it is a `/`-prefix of (`fs` matches `fs/read`). bool matches(std::string_view id, std::string_view pattern); -// The identifiers `mcpp test --case` passes a program (P4.1), from its listing: +// The lines `mcpp test --case` passes a program (P4.1), from its listing: // each listed case a pattern matches; for a pattern under a listed case -- a // case that case declares, which the program reports when it runs and need not -// list (P3.4) -- the pattern itself when it names one case, else the listed -// case, all of whose cases then run. +// list (P3.4) -- the pattern itself when it names one case. A glob under a +// listed case selects the declared cases it matches that a run of the store +// measured (`declared`), each as an optional line `?` where the program +// selects within (P4.3), so `--case 'trees/955*'` runs case 955 and not the +// whole of `trees` (design 2026-11-20 §5.14); with neither, the listed case, +// all of whose cases then run. std::vector selection(const std::vector& listed, - const std::vector& patterns); + const std::vector& patterns, + const std::vector& declared = {}, + bool selectsWithin = false); // The JSON of a report in a `mcpp test --message-format json` record // (docs/50): `cases` and `missing`. @@ -444,7 +450,9 @@ bool matches(std::string_view id, std::string_view pattern) { } std::vector selection(const std::vector& listed, - const std::vector& patterns) { + const std::vector& patterns, + const std::vector& declared, + bool selectsWithin) { std::vector out; auto add = [&](const std::string& id) { if (std::ranges::find(out, id) == out.end()) out.push_back(id); @@ -458,7 +466,12 @@ std::vector selection(const std::vector& listed, return p.size() > c.id.size() && p.starts_with(c.id) && p[c.id.size()] == '/'; }); if (under == listed.end()) continue; - add(p.find_first_of("*?") == std::string::npos ? p : under->id); + if (p.find_first_of("*?") == std::string::npos) { add(p); continue; } + bool known = false; + if (selectsWithin) + for (const auto& id : declared) + if (id.starts_with(under->id + "/") && matches(id, p)) { add("?" + id); known = true; } + if (!known) add(under->id); } return out; } diff --git a/src/config.cppm b/src/config.cppm index e3ebd5621..bcd932abc 100644 --- a/src/config.cppm +++ b/src/config.cppm @@ -142,6 +142,8 @@ struct GlobalConfig { // From config.toml [build] std::int64_t defaultJobs = 0; + // `[build] memory_threshold`: this machine's (design 2026-11-20 §5.5); empty: the engine's. + std::string memoryThreshold; // `[build] compile_commands`: the default for a project that does not say // where its compile database goes (design 2026-10-26 D14). Empty: none. std::string compileCommands; @@ -482,6 +484,10 @@ unused_days = 30 [build] default_jobs = 0 +# A compiler starts only while the system has this much memory available; below +# it, it waits for one already running to end ("2GiB", "512MiB", or bytes). +# Unset: the project's [build] memory_threshold, else 1 GiB. +# memory_threshold = "2GiB" # After a build, at most once a day, the build directories of this project that # no configuration used for this many days are removed. 0 keeps all. keep_days = 3 @@ -1010,6 +1016,8 @@ std::expected load_or_init( cfg.cacheUnusedDays = doc->get_int("cache.unused_days").value_or(30); cfg.buildKeepDays = doc->get_int("build.keep_days").value_or(3); cfg.defaultJobs = doc->get_int("build.default_jobs").value_or(0); + if (auto v = doc->get_string("build.memory_threshold")) cfg.memoryThreshold = *v; + else if (auto n = doc->get_int("build.memory_threshold")) cfg.memoryThreshold = std::to_string(*n); cfg.compileCommands = doc->get_string("build.compile_commands").value_or(""); cfg.defaultToolchain = doc->get_string("toolchain.default").value_or(""); cfg.defaultTarget = doc->get_string("toolchain.default_target").value_or(""); diff --git a/src/home/files.cpp b/src/home/files.cpp index 1af3e8306..f02f98dc9 100644 --- a/src/home/files.cpp +++ b/src/home/files.cpp @@ -4,9 +4,34 @@ module mcpp.home.files; import std; import mcpplibs.os.fs; +import mcpplibs.os.env; namespace mcpp::home { +void contention_point() { + struct jitter { + int max_ms = 0; + std::mt19937_64 random; + std::mutex lock; + jitter() { + const auto max = mcpplibs::os::env::get("MTEST_JITTER").value_or(""); + std::from_chars(max.data(), max.data() + max.size(), max_ms); + std::uint64_t seed = 0; + const auto given = mcpplibs::os::env::get("MTEST_JITTER_SEED").value_or(""); + std::from_chars(given.data(), given.data() + given.size(), seed); + random.seed(seed); + } + }; + static jitter j; + if (j.max_ms <= 0) return; + int wait = 0; + { + std::lock_guard g(j.lock); + wait = std::uniform_int_distribution(0, j.max_ms)(j.random); + } + std::this_thread::sleep_for(std::chrono::milliseconds{wait}); +} + std::optional FileLock::try_acquire(const std::filesystem::path& dir, std::string_view name) { std::error_code ec; std::filesystem::create_directories(dir, ec); @@ -34,6 +59,7 @@ std::optional FileLock::acquire( const std::function& waiting, bool* waited) { namespace fs = std::filesystem; if (waited) *waited = false; + contention_point(); const auto start = std::chrono::steady_clock::now(); auto pause = std::chrono::milliseconds(20); bool told = false; diff --git a/src/home/files.cppm b/src/home/files.cppm index 689f493f5..8cce90532 100644 --- a/src/home/files.cppm +++ b/src/home/files.cppm @@ -8,6 +8,13 @@ import std; export namespace mcpp::home { +// A POINT WHERE AN ORDER BETWEEN PROCESSES IS NOT FIXED (SPEC-017 O12): under a test runner's +// jitter (MTEST_JITTER milliseconds, MTEST_JITTER_SEED) the process waits here a random time up +// to the stated bound, the same for the same seed; elsewhere nothing happens. mcpp waits at the +// acquisition of a lock and at the publication of a compiler's outputs, the two places its +// concurrent runs meet (design 2026-11-20 §5.19). +void contention_point(); + // ── FileLock ────────────────────────────────────────────────────────────── // // RAII exclusive non-blocking file lock. diff --git a/tests/e2e/test/streams.cpp b/tests/e2e/test/streams.cpp index b8f138e9a..e58e5a84a 100644 --- a/tests/e2e/test/streams.cpp +++ b/tests/e2e/test/streams.cpp @@ -6,6 +6,7 @@ import mcpp.test.support; using namespace mcpplibs::mtest; namespace s = mcpp::test::support; +namespace fs = std::filesystem; namespace { @@ -65,6 +66,27 @@ main = "src/main.cpp" expect(s::lacks(d.err, "workspace result")) << "the workspace result line is on standard error"; }; + "976 a test is told the machine's budget, and a build within a budget of one slot completes"_test[s::gcc] = [] { + // SPEC-017 O11, design 2026-11-20 §5.5. `mcpp test` holds a slot of its budget for each test + // and states it (A). A build under a runner's budget runs its first compiler on the slot held + // for it and takes the others from the budget's free slots: with one slot, the held one, the + // compilers run one at a time and the build completes (B). + s::project p{{"mcpp.toml", "[package]\nname = \"p\"\nversion = \"0.1.0\"\n\n[targets.p]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}, + {"src/main.cpp", "import p.a;\nimport p.b;\nint main() { return a() + b() == 3 ? 0 : 1; }\n"}, + {"src/a.cppm", "export module p.a;\nexport int a() { return 1; }\n"}, + {"src/b.cppm", "export module p.b;\nexport int b() { return 2; }\n"}, + {"tests/t_budget.cpp", "#include \n#include \n#include \n" + "int main() { const char* d = std::getenv(\"MTEST_BUDGET\"); const char* h = std::getenv(\"MTEST_BUDGET_HELD\");\n" + " struct stat st{}; if (!d || !h || stat(h, &st) != 0) { std::puts(\"no budget\"); return 1; }\n" + " std::puts(\"budget stated\"); return 0; }\n"}}; + auto a = p.mcpp({"test"}); + expect(fatal(e2e::succeeded(a))) << "A: a test was not told the budget: " << s::text(a); + const auto budget = s::case_dir() / "budget"; + fs::create_directories(budget); + std::ofstream(budget / "slot-0"); + auto b = p.mcpp({"build"}, {.env = {{"MTEST_BUDGET", budget.string()}, {"MTEST_BUDGET_HELD", (budget / "slot-0").string()}}}); + expect(e2e::succeeded(b)) << "B: a build within a budget of one slot: " << s::text(b); + }; "178 test observability"_test[s::unix_shell] = [] { // `mcpp test` is streamed, timed and bounded. POSIX only: the deadline runner has no // kill-by-handle path on Windows, so --timeout and --build-timeout are documented POSIX-only. From 25700dd262559f904eb61e6bf40ed35fc226eaaa Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 02:10:37 +0900 Subject: [PATCH 07/38] feat: a run records its machine, each case's processor time, steps, items used and limit; estimates read the runs of the machine's class (SPEC-017 P5.6, P12.4) --- docs/specs/test.md | 4 +- mcpp.toml | 2 +- modules/invocation/mcpp.toml | 2 +- modules/manifest/mcpp.toml | 2 +- modules/toolchain-model/mcpp.toml | 2 +- src/build/execute.cppm | 53 +++++++++++++++++++-- src/build/test_protocol.cppm | 77 ++++++++++++++++++++++++++++++- src/build/test_schedule.cppm | 16 +++++-- tests/e2e/test/protocol.cpp | 21 +++++++++ tests/unit/test_test_schedule.cpp | 17 +++++++ 10 files changed, 182 insertions(+), 14 deletions(-) diff --git a/docs/specs/test.md b/docs/specs/test.md index 6135478b9..fd9df9689 100644 --- a/docs/specs/test.md +++ b/docs/specs/test.md @@ -173,7 +173,7 @@ | `index.ndjson` | 每次运行一行:去掉程序与用例的运行记录 | | `runs//run.json` | 运行记录:`schema`(`mtest.run/1`)、`id`、`started`、`finished`、`status`、`system`、`arch`、`machine`、`commit`、`set`、`selection`、`group`、`monitor`、`ms`、`jobs`、`counts`;`processors`、`memory`、`budget`、`jitter`(@NEXT@) | | `runs//programs.ndjson` | 每个程序进程一行:`name`、`argv`、`status`、`exit`、`ms`、`started_ms`、`worker`、`select`、`resources`、`fixtures`,运行器构建它时的 `build_started_ms` 与 `build_ms`,`usage`(带 `source`:`program` 或 `engine`) | -| `runs//cases.ndjson` | 每个用例一行:`program`、`id`、`kind`、`status`、`ms`、`started_ms`、`reason`、`failures`、`metrics`、`bounds`、`tags`、`clauses`;`cpu_ms`、`steps`(`step` 事件的字段,`id` 除外)、`exercised`(@NEXT@) | +| `runs//cases.ndjson` | 每个用例一行:`program`、`id`、`kind`、`status`、`ms`、`started_ms`、`reason`、`failures`、`metrics`、`bounds`、`tags`、`clauses`;`cpu_ms`、`steps`(`step` 事件的字段,`id` 除外)、`exercised`、`limit_ms`(用例所受的时限,O5;未知时省略)(@NEXT@) | | `runs//builds.ndjson` | 运行器为这次运行所做的构建(@NEXT@):每条运行过的边一行,`build`(构建记录的标识)、`output`、`package`、`program`(该边所服务的测试程序,可为空)、`started_ms`、`ms`、`cpu_ms`、`peak_bytes`;由缓存提供的包一行,`build`、`package`、`cached`(`true`)、`units` | | `runs//fixtures.ndjson` | 每个运行夹具一行:`name`、`program`、`started_ms`、`ready_ms`、`ended_ms`、`status` | @@ -278,7 +278,7 @@ - **P10.2** 程度不为 `off` 的程序在 `done` 之前写一个 `usage` 事件。 - **P10.3** 在 `trace` 下,程序每 `MTEST_MONITOR_SAMPLE` 毫秒(缺省 50)采样一次,每个序列报告为一个指标,`samples` 为序列,`value` 为其最大样本,精度为 `sampled`。 - **P10.4** 运行器可以从其自己的进程测量不写 `usage` 的程序,并把结果记作引擎的。 -- **mcpp 的实现(mcpp >= 2026.11.10.1)。** `usage` 属于程序,进入程序记录,并以 `source = "program"` 进入 B4。 +- **mcpp 的实现(mcpp >= 2026.11.10.1)。** `usage` 属于程序,进入程序记录,并以 `source = "program"` 进入 B4。mcpp >= @NEXT@ 以 `basic` 陈述程度,调用者的环境陈述 `MTEST_MONITOR` 时取其值;B4 的 `monitor` 为所陈述的程度。mcpp 不实现 P10.4:它的各个工作线程在同一进程中运行测试程序,系统对子进程的计量只按进程汇总,一个程序的用量无法从中分离。 ### O8. 输出归属(mcpp >= 2026.11.10.1) diff --git a/mcpp.toml b/mcpp.toml index d84fdd0c0..a699a67ab 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -125,7 +125,7 @@ mcpplibs.mwrap.xz = "5.8.3.1" # The operating system is mcpplibs.os's (design 2026-10-26 §25): mcpp keeps no # layer of its own. -mcpplibs.os = "0.4.6" +mcpplibs.os = "0.4.7" # `modules/` holds what gets linked into this binary; `src/` is what has not # been separated yet, and shrinking it is the direction. Each entry is an diff --git a/modules/invocation/mcpp.toml b/modules/invocation/mcpp.toml index 496101574..cf26a37c3 100644 --- a/modules/invocation/mcpp.toml +++ b/modules/invocation/mcpp.toml @@ -13,4 +13,4 @@ description = "The environment of one invocation of mcpp: statements over the st license = "Apache-2.0" [dependencies] -mcpplibs.os = "0.4.6" +mcpplibs.os = "0.4.7" diff --git a/modules/manifest/mcpp.toml b/modules/manifest/mcpp.toml index 9d473104a..d1dc930c9 100644 --- a/modules/manifest/mcpp.toml +++ b/modules/manifest/mcpp.toml @@ -24,7 +24,7 @@ mcpp.source-kind = { path = "../source-kind" } mcpp.toolchain-model = { path = "../toolchain-model" } mcpp.versioning = { path = "../versioning" } -mcpplibs.os = "0.4.6" +mcpplibs.os = "0.4.7" # Subsystem-level tests. They see only this package, which is what lets them # state its contract in isolation; the root `tests/unit/` states the facts two diff --git a/modules/toolchain-model/mcpp.toml b/modules/toolchain-model/mcpp.toml index 18eba6295..155505710 100644 --- a/modules/toolchain-model/mcpp.toml +++ b/modules/toolchain-model/mcpp.toml @@ -23,7 +23,7 @@ license = "Apache-2.0" mcpp.invocation = { path = "../invocation" } mcpp.versioning = { path = "../versioning" } -mcpplibs.os = "0.4.6" +mcpplibs.os = "0.4.7" # Subsystem-level tests. They see only this package, which is what lets them # state its contract in isolation; the root `tests/unit/` states the facts two diff --git a/src/build/execute.cppm b/src/build/execute.cppm index 0d5c32bc2..62701aa21 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -39,6 +39,7 @@ import mcpp.build.dirlock; // the build directory's lock (mcpp#839) import mcpp.build.graph_engine; // the engine a graph names (2026-11-12 §17) import mcpplibs.os.process; // the identity of this invocation (its protocol directory) import mcpplibs.os.fs; // the budget's slot locks (SPEC-017 O11) +import mcpplibs.os.machine; // the machine a run records (SPEC-017 P12.4) import mcpp.build.result; // the result record and the event stream (design 2026-10-26 §5) import mcpp.wire; // the mcpp.launch envelope import mcpp.build.cxx_runtime; // the island, verified after the link (design 2026-10-26 §6.3) @@ -4603,6 +4604,10 @@ struct TestResult { int worker = -1; std::vector select; bool alone = false; // it held the machine (P7.1) + // The limit its process ran within (0: none), and each listed case's own + // (seconds): a case's `limit_ms` in the store (B4). + double limitSecs = 0; + std::map caseSecs; // The program and the files it is compiled from, for a test built and not // run (`--no-run`): what a runner that selects by change reads (docs/50). std::string program; @@ -4614,6 +4619,27 @@ struct TestResult { std::optional launch; }; +// THE MACHINE'S CLASS (SPEC-017 P12.4): its processor's model, else its +// architecture and logical processors. A runner on a hosted pool lands on +// processors 1.6 times apart in speed (design 2026-11-20 §3.1); estimates +// compared within a class tell that from a regression. +static std::string machine_class() { + if (auto model = mcpplibs::os::machine::processor_model(); model && !model->empty()) return *model; + return std::format("{}/{}", mcpplibs::os::machine::arch(), mcpplibs::os::machine::processors().logical); +} + +// The limit a case ran within, in seconds (0: none): what it or the listed +// case it is under states, else its process's when the process ran it alone. +static double case_limit(const TestResult& t, std::string_view id) { + for (std::string_view at = id;;) { + if (auto it = t.caseSecs.find(std::string(at)); it != t.caseSecs.end()) return it->second; + const auto slash = at.rfind('/'); + if (slash == std::string_view::npos) break; + at = at.substr(0, slash); + } + return t.report.cases.size() == 1 ? t.limitSecs : 0; +} + // THE RUN IN THE MTEST STORE (mtest spec P12; design 2026-11-09 §17.3): the // run's record, a line per process of a program and a line per case, so that // `mtest report|analyze|graph --store ` read `mcpp test`'s runs as the @@ -4700,6 +4726,11 @@ void write_mtest_store(const std::filesystem::path& dir, const std::vectorheld}, {"failed", c.checks->failed}}; + // P5.6 and the case's processor time and limit (SPEC-017 B4, 2.5). + if (c.cpuMs) line["cpu_ms"] = *c.cpuMs; + if (!c.steps.empty()) line["steps"] = mcpp::build::test_protocol::steps_json(c.steps); + if (!c.exercised.empty()) line["exercised"] = c.exercised; + if (const auto limit = case_limit(t, c.id); limit > 0) line["limit_ms"] = limit * 1000; cases << mcpp::wire::text(line) << "\n"; } } @@ -4707,12 +4738,17 @@ void write_mtest_store(const std::filesystem::path& dir, const std::vectorempty()) + run["jitter"] = {{"max_ms", std::atoi(jitter->c_str())}, + {"seed", std::strtoull(mcpp::invocation::get("MTEST_JITTER_SEED").value_or("0").c_str(), nullptr, 10)}}; { std::ofstream(runDir / "run.json", std::ios::binary) << mcpp::wire::text(run, 2) << "\n"; } std::ofstream(dir / "index.ndjson", std::ios::binary | std::ios::app) << mcpp::wire::text(run) << "\n"; } @@ -5012,6 +5048,10 @@ struct TestRunnable { // `--timeout`, so that a test that states its length is not ended at the // limit the rest are held to (P-limit, docs/08). double declaredSecs = 0; + // The time each listed case states it may run, by its identifier: the + // limit a case is held to in the store (B4 `limit_ms`), for the report's + // margin (design 2026-11-20 §5.8). + std::map caseSecs; }; // THE PROTOCOL A TEST PROGRAM IMPLEMENTS (P2.3): what its entry states, else @@ -5489,7 +5529,8 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, ? mcpp::build::test_schedule::History{} : mcpp::build::test_schedule::read_history( storeDir, 30, testOpts.shard ? std::optional(mcpp::invocation::get("GITHUB_RUN_ID").value_or("")) - : std::nullopt); + : std::nullopt, + testOpts.shard ? std::nullopt : std::optional(machine_class())); using Runnable = TestRunnable; std::vector runnable; @@ -5666,6 +5707,8 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, t.partOf = r.partOf; t.divides = r.divides; t.alone = r.alone; + t.limitSecs = limit_of(r); + t.caseSecs = r.caseSecs; if (r.select) { std::ifstream in(*r.select); for (std::string line; std::getline(in, line);) @@ -6320,6 +6363,8 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, } for (const auto& [_, secs] : selectedSecs) r.declaredSecs += secs; selectedSecs.clear(); + for (const auto& c : report.listed) + if (c.timeout > 0) r.caseSecs[c.id] = c.timeout; if (lines.empty()) { ++notSelected; continue; } r.select = r.protocolDir / "select.txt"; std::ofstream(*r.select, std::ios::binary) << lines; @@ -6409,6 +6454,8 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, continue; } const auto& report = *listings[at]; + for (const auto& c : report.listed) + if (c.timeout > 0) r.caseSecs[c.id] = c.timeout; // THE MACHINE, HELD ALONE (P7.1). A case that holds `machine` // states a property that holds only while nothing beside it // changes the machine (the shared home, its caches, the default diff --git a/src/build/test_protocol.cppm b/src/build/test_protocol.cppm index 57b3bfd7d..cd31ccf90 100644 --- a/src/build/test_protocol.cppm +++ b/src/build/test_protocol.cppm @@ -10,6 +10,7 @@ export module mcpp.build.test_protocol; import std; import nlohmann.json; +import mcpp.invocation; // MTEST_MONITOR, stated to the run (P10.1) export namespace mcpp::build::test_protocol { @@ -23,11 +24,16 @@ inline constexpr std::string_view kMtestRepo = "https://github.com/mcpp-commu // The protocol's variables, removed from the environment a program inherits // so that an outer runner's do not reach a program mcpp starts (P2.2). -inline constexpr std::array kVariables = { +inline constexpr std::array kVariables = { "MTEST_PROTOCOL", "MTEST_MODE", "MTEST_REPORT", "MTEST_SELECT", - "MTEST_ISOLATION", "MTEST_TMPDIR", "MTEST_SEED", + "MTEST_ISOLATION", "MTEST_TMPDIR", "MTEST_SEED", "MTEST_MONITOR", }; +// The level of monitoring a run states (P10.1): `basic` -- each case's +// processor time, which the store and the budget gate compare (design +// 2026-11-20 §5.9) -- unless the environment of `mcpp test` states another. +std::string monitor_level(); + // The version the marker in a program's file states (P2.3): the bytes // `mtest-protocol=` and a version followed by a byte that is not a digit; 0 // when the file carries none or cannot be read. A file may carry the bytes as @@ -73,6 +79,17 @@ struct Case { double timeout = 0; // seconds the case states it may run; 0: none stated }; +// A step of a case (P5.6): a command an end-to-end case ran, as the program +// states it when the step ends. +struct Step { + std::string name; + double t = 0; // milliseconds after the program started + double ms = 0; + std::optional cpuMs; + std::optional status; // the exit status of the program the step ran + std::string build; // the build record's identifier when it was a build +}; + // A failure an expectation of a case reported (P5.2). struct Failure { std::string file; @@ -111,6 +128,13 @@ struct CaseResult { // for a program that ran this case alone, all that the program wrote. std::string output; bool outputStated = false; // an `output` event came + // P5.6: its steps in the order they ended, and the items of the product's + // surface it used, sorted and each once. + std::vector steps; + std::vector exercised; + // Its processor time: the sum of its `cpu.user` and `cpu.sys` metrics + // (P6.6, B4 `cpu_ms`); unset when it reported neither. + std::optional cpuMs; }; // What one program's report says. @@ -197,6 +221,8 @@ nlohmann::json bounds_json(const std::vector& b); nlohmann::json metric_json(const Metric& m); nlohmann::json usage_json(const Usage& u); nlohmann::json listing_json(const Report& r); +// A case's steps as B4 and the records write them (P5.6). +nlohmann::json steps_json(const std::vector& steps); } // namespace mcpp::build::test_protocol @@ -357,7 +383,28 @@ take(Report& r, std::map& open, const nlohmann::json& if (auto v = k->find("failed"); v != k->end() && v->is_number_integer()) n.failed = v->get(); c.checks = n; } + double cpu = 0; + bool told = false; + for (const auto& m : c.metrics) + if (m.name == "cpu.user" || m.name == "cpu.sys") { + cpu += m.value; + told = true; + } + if (told) c.cpuMs = cpu; return ReportReader::Change{i, true}; + } else if (kind == "step") { + Step st{text(e, "name")}; + if (auto v = e.find("t"); v != e.end() && v->is_number()) st.t = v->get(); + if (auto v = e.find("ms"); v != e.end() && v->is_number()) st.ms = v->get(); + if (auto v = e.find("cpu_ms"); v != e.end() && v->is_number()) st.cpuMs = v->get(); + if (auto v = e.find("status"); v != e.end() && v->is_number_integer()) st.status = v->get(); + st.build = text(e, "build"); + r.cases[case_of(r, open, id)].steps.push_back(std::move(st)); + } else if (kind == "exercise") { + auto& c = r.cases[case_of(r, open, id)]; + for (auto& item : strings(e, "items")) + if (auto at = std::ranges::lower_bound(c.exercised, item); at == c.exercised.end() || *at != item) + c.exercised.insert(at, std::move(item)); } else if (kind == "missing") { r.missing.push_back(id); } else if (kind == "done") { @@ -398,6 +445,16 @@ Report ReportReader::finish(int exitStatus) { return std::move(r_); } +std::string monitor_level() { + static const std::string level = [] { + const auto stated = mcpp::invocation::get("MTEST_MONITOR").value_or(""); + for (std::string_view known : {"off", "basic", "detailed", "trace"}) + if (stated == known) return stated; + return std::string("basic"); + }(); + return level; +} + Report read_report(const std::filesystem::path& file, int exitStatus) { ReportReader reader; reader.poll(file); @@ -414,6 +471,7 @@ variables(std::string_view mode, const std::filesystem::path& report, {"MTEST_REPORT", report.string()}, {"MTEST_TMPDIR", tmpdir.string()}, }; + if (mode == "run") env.emplace_back("MTEST_MONITOR", monitor_level()); if (select) env.emplace_back("MTEST_SELECT", select->string()); if (seed) env.emplace_back("MTEST_SEED", std::to_string(*seed)); return env; @@ -528,9 +586,24 @@ nlohmann::json case_json(const CaseResult& c) { if (!c.bounds.empty()) one["bounds"] = bounds_json(c.bounds); if (c.checks) one["checks"] = {{"held", c.checks->held}, {"failed", c.checks->failed}}; if (!c.output.empty()) one["output"] = c.output; + if (c.cpuMs) one["cpu_ms"] = *c.cpuMs; + if (!c.steps.empty()) one["steps"] = steps_json(c.steps); + if (!c.exercised.empty()) one["exercised"] = c.exercised; return one; } +nlohmann::json steps_json(const std::vector& steps) { + nlohmann::json all = nlohmann::json::array(); + for (const auto& st : steps) { + nlohmann::json one = {{"name", st.name}, {"t", st.t}, {"ms", st.ms}}; + if (st.cpuMs) one["cpu_ms"] = *st.cpuMs; + if (st.status) one["status"] = *st.status; + if (!st.build.empty()) one["build"] = st.build; + all.push_back(std::move(one)); + } + return all; +} + nlohmann::json cases_json(const Report& r) { nlohmann::json cases = nlohmann::json::array(); for (const auto& c : r.cases) cases.push_back(case_json(c)); diff --git a/src/build/test_schedule.cppm b/src/build/test_schedule.cppm index fb99a46e3..e3c22baca 100644 --- a/src/build/test_schedule.cppm +++ b/src/build/test_schedule.cppm @@ -33,8 +33,14 @@ struct History { // the runs of a shard of that group (the CI run, or none) are left out: the // shards of one run, run one after another on one store, read the store as // the first of them read it. +// +// With `machine`, the runs on that class of machine (SPEC-017 P12.4) when the +// store has one, else all: the processors of one hosted pool are 1.6 times +// apart (design 2026-11-20 §3.1). The shards of a run do not pass it: each +// shard lands on its own machine and must compute the one plan the others do. History read_history(const std::filesystem::path& store, std::size_t maxRuns = 30, - std::optional shardsOf = std::nullopt); + std::optional shardsOf = std::nullopt, + std::optional machine = std::nullopt); // The key of a case in `History::cases`: its program (the program a part is of) // and its id. @@ -137,12 +143,16 @@ bool failed_case(std::string_view status) { return status == "fail" || status == } // namespace History read_history(const std::filesystem::path& store, std::size_t maxRuns, - std::optional shardsOf) { + std::optional shardsOf, std::optional machine) { History h; std::error_code ec; if (!std::filesystem::exists(store / "index.ndjson", ec)) return h; - for (const auto& run : runs_newest_first(store)) { + const auto runs = runs_newest_first(store); + if (machine && std::ranges::none_of(runs, [&](const json& r) { return text(r, "machine") == *machine; })) + machine.reset(); + for (const auto& run : runs) { if (h.runs == maxRuns) break; + if (machine && text(run, "machine") != *machine) continue; if (shardsOf && !text(run, "shard").empty() && text(run, "group") == *shardsOf) continue; const auto dir = store / "runs" / text(run, "id"); if (!std::filesystem::is_directory(dir, ec)) continue; diff --git a/tests/e2e/test/protocol.cpp b/tests/e2e/test/protocol.cpp index c305ff742..d3d6d6077 100644 --- a/tests/e2e/test/protocol.cpp +++ b/tests/e2e/test/protocol.cpp @@ -294,6 +294,15 @@ bool cases(std::ofstream& r, bool list, const std::set& selected) { if (on("plain")) { std::println("written by plain"); r << R"({"e":"begin","id":"plain","t":0})" << "\n"; + r << R"({"e":"step","id":"plain","name":"mcpp build","t":1,"ms":5,"cpu_ms":4,"status":0,"build":"b1"})" << "\n"; + r << R"({"e":"exercise","id":"plain","items":["flag build --target","command build"]})" << "\n"; + r << R"({"e":"exercise","id":"plain","items":["command build"]})" << "\n"; + // The level the run stated (P10.1): its processor time where it is `basic`. + const char* level = std::getenv("MTEST_MONITOR"); + if (level && std::string_view(level) == "basic") { + r << R"({"e":"metric","id":"plain","name":"cpu.user","value":3,"unit":"ms"})" << "\n"; + r << R"({"e":"metric","id":"plain","name":"cpu.sys","value":1,"unit":"ms"})" << "\n"; + } r << R"({"e":"end","id":"plain","status":"pass","ms":0})" << "\n"; } return on("stated"); @@ -340,12 +349,24 @@ bool cases(std::ofstream& r, bool list, const std::set& selected) { expect(rec["cases"][0].value("output", "") == "said by the case\n" || rec["cases"][0].value("output", "") == "said by the case") << "the case's stated output: " << rec["cases"][0].dump(); expect(!rec["cases"][1].contains("output")) << "the output of a program of two cases was divided"; + // P5.6 (SPEC-017 2.5): the steps in order, the items once each and sorted, and the case's + // processor time from the metrics of the `basic` level the run stated (P10.1). + const auto& plain = rec["cases"][1]; + expect(plain["steps"] == json::parse(R"([{"name":"mcpp build","t":1.0,"ms":5.0,"cpu_ms":4.0,"status":0,"build":"b1"}])")) + << plain.dump(); + expect(plain["exercised"] == json::parse(R"(["command build","flag build --target"])")) << plain.dump(); + expect(plain.value("cpu_ms", 0.0) == 4.0) << "the case's processor time: " << plain.dump(); bool kept = false; for (const auto& f : s::glob(t.at("app/target/mtest/runs"), "*/cases.ndjson")) { const auto c = s::read_file(f); kept = kept || c.contains(R"("checks":{"failed":1,"held":4})") || c.contains(R"("checks":{"held":4,"failed":1})"); } expect(kept) << "the kept run has no checks"; + // P12.4: the machine the run was on, and the level it stated. + const auto index = s::json_lines(s::read_file(t.at("app/target/mtest/index.ndjson"))); + expect(fatal(!index.empty())); + expect(!index.back().value("machine", "").empty() && index.back().value("processors", 0) > 0 + && index.back().value("monitor", "") == "basic") << index.back().dump(); // O8: one case run alone. auto one = app({"test", "--case", "plain", "--message-format", "json"}); expect(fatal(e2e::succeeded(one))); diff --git a/tests/unit/test_test_schedule.cpp b/tests/unit/test_test_schedule.cpp index f5b875b33..d969689f0 100644 --- a/tests/unit/test_test_schedule.cpp +++ b/tests/unit/test_test_schedule.cpp @@ -83,6 +83,23 @@ UTest a_shard_reads_no_run_of_its_own_group{"TestSchedule/AShardReadsNoRunOfItsO expect(eq(ts::estimate(ts::read_history(s.dir, 30, std::string("8")), "p", {}), 900.0)); }}; +UTest a_run_reads_the_runs_of_its_machine_class{"TestSchedule/ARunReadsTheRunsOfItsMachineClass", [] { + // SPEC-017 P12.4: the runs on the same class of machine when the store has + // one, else all of them. + Store s; + s.run("old", "2026-11-01T00:00:00Z", R"({"name":"p","status":"pass","ms":100})" "\n", ""); + fs::create_directories(s.dir / "runs" / "new"); + std::ofstream(s.dir / "runs" / "new" / "programs.ndjson") << R"({"name":"p","status":"pass","ms":160})" "\n"; + std::ofstream(s.dir / "index.ndjson", std::ios::app) + << R"({"id":"new","started":"2026-11-02T00:00:00Z","machine":"Intel Xeon 6975P"})" "\n"; + expect(eq(ts::estimate(ts::read_history(s.dir), "p", {}), 160.0)); + expect(eq(ts::estimate(ts::read_history(s.dir, 30, std::nullopt, std::string("Intel Xeon 6975P")), "p", {}), + 160.0)); + expect(eq(ts::estimate(ts::read_history(s.dir, 30, std::nullopt, std::string("")), "p", {}), 100.0)); + expect(eq(ts::estimate(ts::read_history(s.dir, 30, std::nullopt, std::string("AMD EPYC 7763")), "p", {}), + 160.0)); +}}; + UTest the_shards_cover_every_test_once{"TestSchedule/TheShardsCoverEveryTestOnce", [] { const std::vector est{50, 400, 10, 400, 90, 5, 300}; const std::vector names{"a", "b", "c", "d", "e", "f", "g"}; From 682afda7fede3d98191514a840723f990c3d5f19 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 02:34:39 +0900 Subject: [PATCH 08/38] feat: a build records each edge it ran with its processor time and largest resident set; a test run keeps its builds; a step that runs a minute is named (#927) --- mcpp.toml | 2 + modules/build-record/mcpp.toml | 20 ++ modules/build-record/src/build_record.cppm | 229 ++++++++++++++++++ .../build-record/tests/test_build_record.cpp | 59 +++++ src/build/execute.cppm | 20 +- src/build/ninja_backend.cppm | 11 + src/build/progress.cppm | 161 +++++++++++- src/cli.cppm | 25 +- tests/e2e/test/streams.cpp | 31 +++ tests/unit/test_build_progress.cpp | 38 +++ 10 files changed, 589 insertions(+), 7 deletions(-) create mode 100644 modules/build-record/mcpp.toml create mode 100644 modules/build-record/src/build_record.cppm create mode 100644 modules/build-record/tests/test_build_record.cpp diff --git a/mcpp.toml b/mcpp.toml index a699a67ab..abc349035 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -153,6 +153,7 @@ mcpp.buildopt = { path = "modules/buildopt" } mcpp.manifest = { path = "modules/manifest" } mcpp.toolchain-model = { path = "modules/toolchain-model" } mcpp.buildmcpp = { path = "modules/buildmcpp" } +mcpp.build-record = { path = "modules/build-record" } [target.linux.build] sources = ["!src/native/windows/**"] @@ -176,6 +177,7 @@ members = [ "modules/manifest", "modules/toolchain-model", "modules/buildmcpp", + "modules/build-record", ] # `mcpp build` ignores [dev-dependencies]; only `mcpp test` resolves them. diff --git a/modules/build-record/mcpp.toml b/modules/build-record/mcpp.toml new file mode 100644 index 000000000..25cb503eb --- /dev/null +++ b/modules/build-record/mcpp.toml @@ -0,0 +1,20 @@ +# mcpp.build-record -- what a build did, edge by edge (SPEC-017 B4 `builds.ndjson`; design +# 2026-11-20 §5.6). +# +# A leaf: a line format the compiler executor appends to (each compile's start, end, processor +# time and largest resident set), the record the build writes from it and ninja's log (each edge +# that ran, its package, and a line per package the cache served), and the slowest edges. It knows +# nothing of ninja's files, the plan or the screen: the build's reader hands it what they say. + +[package] +name = "build-record" +namespace = "mcpp" +version = "0.1.0" +description = "The record of a build: each edge that ran, its time, processor time and memory" +license = "Apache-2.0" + +[dependencies] +nlohmann.json = "3.12.0" + +[dev-dependencies] +mcpplibs.mtest = "0.10.0" diff --git a/modules/build-record/src/build_record.cppm b/modules/build-record/src/build_record.cppm new file mode 100644 index 000000000..899a09717 --- /dev/null +++ b/modules/build-record/src/build_record.cppm @@ -0,0 +1,229 @@ +// mcpp.build.record -- the record of a build (SPEC-017 B4 `builds.ndjson`, design 2026-11-20 +// §5.6): an edge per line, with what ninja's log says of it (its outputs, start and end) and what +// the compiler executor measured (processor time, largest resident set), and a line per package +// the cache served. Three questions a build could not answer before (the long tail of a test +// build: which edges, how long each, on how many processors) are each a query of it. +export module mcpp.build.record; + +import std; +import nlohmann.json; + +export namespace mcpp::build::record { + +// The variable that names the file the compiler executor appends its lines to: the build's +// reader sets it for ninja (`environment()` of a build's progress). +inline constexpr std::string_view kUsageEnv = "MCPP_EDGE_USAGE"; +inline constexpr std::string_view kUsageFile = ".mcpp-edge-usage"; +// The variable that names a file each build of an invocation appends its record to: a runner +// that starts mcpp (a test kit's step) reads the build's identifier and edges from it. +inline constexpr std::string_view kRecordEnv = "MCPP_BUILD_RECORD"; + +// What the executor measured of one compile: its output as the edge names it, its start and end +// (milliseconds since the epoch), and the compiler's processor time and largest resident set, +// absent where the system does not tell them. +struct Usage { + std::string output; + long long startedUnixMs = 0; + long long endedUnixMs = 0; + std::optional cpuMs; + std::optional peakBytes; +}; +// One line, `\n` ended, written in one append so that concurrent compiles do not interleave. +std::string usage_line(const Usage& u); +// The complete lines of `text`; a line that is not a usage line is skipped. +std::vector parse_usage(std::string_view text); + +// An edge that ran: what ninja's log and the plan say of it, and what the executor measured. +struct Edge { + std::string output; // its first output, as build.ninja spells it + std::string package; // the package the plan attributes it to; empty: the build's own + std::string program; // the test program it serves; empty when none or not known + double startedMs = 0; // after the build began + double ms = 0; + std::optional cpuMs; + std::optional peakBytes; +}; +// A package the cache served, with the units it placed. +struct Cached { + std::string package; + std::size_t units = 0; +}; +struct Record { + std::string id; // unique for an invocation's build + std::vector edges; + std::vector cached; +}; + +// A fresh identifier: the time of day and a random part. +std::string new_id(); +// Each edge's measurement from `usage`, matched by its output; the newest line of an output wins. +void attach(Record& r, const std::vector& usage); +// The record as `builds.ndjson` lines (SPEC-017 B4). +std::string to_ndjson(const Record& r); +// The records the lines of `text` hold, by identifier in the order first seen. +std::vector from_ndjson(std::string_view text); +// The `n` edges that took longest, longest first. +std::vector slowest(const Record& r, std::size_t n); +// Their sums: edge time and processor time (where measured). +struct Totals { + double ms = 0; + double cpuMs = 0; + std::size_t edges = 0, measured = 0, cachedUnits = 0; +}; +Totals totals(const Record& r); + +// The records of the builds this invocation ran, in the order they ended: what `mcpp test` keeps +// in its run (SPEC-017 B4). Safe from several threads. +void keep(Record r); +std::vector kept(); + +} // namespace mcpp::build::record + +namespace mcpp::build::record { + +namespace { +using json = nlohmann::json; + +std::string text_of(const json& j, const char* key) { + auto it = j.find(key); + return it != j.end() && it->is_string() ? it->get() : std::string{}; +} +std::optional number_of(const json& j, const char* key) { + auto it = j.find(key); + if (it == j.end() || !it->is_number()) return std::nullopt; + return it->get(); +} +std::optional bytes_of(const json& j, const char* key) { + auto it = j.find(key); + if (it == j.end() || !it->is_number()) return std::nullopt; + return it->get(); +} +// Each complete line of `text` that is a JSON object. +std::vector objects(std::string_view text) { + std::vector out; + while (!text.empty()) { + const auto nl = text.find('\n'); + if (nl == std::string_view::npos) break; + auto line = text.substr(0, nl); + text.remove_prefix(nl + 1); + auto j = json::parse(line, nullptr, false); + if (j.is_object()) out.push_back(std::move(j)); + } + return out; +} +} // namespace + +std::string usage_line(const Usage& u) { + json j = {{"output", u.output}, {"started", u.startedUnixMs}, {"ended", u.endedUnixMs}}; + if (u.cpuMs) j["cpu_ms"] = *u.cpuMs; + if (u.peakBytes) j["peak_bytes"] = *u.peakBytes; + return j.dump(-1, ' ', false, json::error_handler_t::replace) + "\n"; +} + +std::vector parse_usage(std::string_view text) { + std::vector out; + for (const auto& j : objects(text)) { + Usage u{text_of(j, "output")}; + if (u.output.empty()) continue; + u.startedUnixMs = static_cast(number_of(j, "started").value_or(0)); + u.endedUnixMs = static_cast(number_of(j, "ended").value_or(0)); + u.cpuMs = number_of(j, "cpu_ms"); + u.peakBytes = bytes_of(j, "peak_bytes"); + out.push_back(std::move(u)); + } + return out; +} + +std::string new_id() { + const auto now = std::chrono::floor(std::chrono::system_clock::now()); + return std::format("{:%Y%m%dT%H%M%S}-{:06x}", now, std::random_device{}() & 0xffffff); +} + +void attach(Record& r, const std::vector& usage) { + std::unordered_map byOutput; + for (const auto& u : usage) byOutput[u.output] = &u; + for (auto& e : r.edges) + if (auto it = byOutput.find(e.output); it != byOutput.end()) { + e.cpuMs = it->second->cpuMs; + e.peakBytes = it->second->peakBytes; + } +} + +std::string to_ndjson(const Record& r) { + std::string out; + for (const auto& e : r.edges) { + json j = {{"build", r.id}, {"output", e.output}, {"package", e.package}, + {"program", e.program}, {"started_ms", e.startedMs}, {"ms", e.ms}}; + if (e.cpuMs) j["cpu_ms"] = *e.cpuMs; + if (e.peakBytes) j["peak_bytes"] = *e.peakBytes; + out += j.dump(-1, ' ', false, json::error_handler_t::replace) + "\n"; + } + for (const auto& c : r.cached) + out += json{{"build", r.id}, {"package", c.package}, {"cached", true}, {"units", c.units}} + .dump(-1, ' ', false, json::error_handler_t::replace) + "\n"; + return out; +} + +std::vector from_ndjson(std::string_view text) { + std::vector out; + for (const auto& j : objects(text)) { + const auto id = text_of(j, "build"); + auto at = std::ranges::find(out, id, &Record::id); + if (at == out.end()) at = out.insert(out.end(), Record{id}); + if (auto c = j.find("cached"); c != j.end() && c->is_boolean() && c->get()) { + at->cached.push_back({text_of(j, "package"), + static_cast(number_of(j, "units").value_or(0))}); + continue; + } + at->edges.push_back({text_of(j, "output"), text_of(j, "package"), text_of(j, "program"), + number_of(j, "started_ms").value_or(0), number_of(j, "ms").value_or(0), + number_of(j, "cpu_ms"), bytes_of(j, "peak_bytes")}); + } + return out; +} + +std::vector slowest(const Record& r, std::size_t n) { + auto edges = r.edges; + std::ranges::stable_sort(edges, std::greater{}, &Edge::ms); + if (edges.size() > n) edges.resize(n); + return edges; +} + +Totals totals(const Record& r) { + Totals t; + for (const auto& e : r.edges) { + t.ms += e.ms; + ++t.edges; + if (e.cpuMs) { + t.cpuMs += *e.cpuMs; + ++t.measured; + } + } + for (const auto& c : r.cached) t.cachedUnits += c.units; + return t; +} + +namespace { +struct Kept { + std::mutex m; + std::vector all; +}; +Kept& kept_records() { + static Kept k; + return k; +} +} // namespace + +void keep(Record r) { + auto& k = kept_records(); + std::lock_guard lock(k.m); + k.all.push_back(std::move(r)); +} + +std::vector kept() { + auto& k = kept_records(); + std::lock_guard lock(k.m); + return k.all; +} + +} // namespace mcpp::build::record diff --git a/modules/build-record/tests/test_build_record.cpp b/modules/build-record/tests/test_build_record.cpp new file mode 100644 index 000000000..ad61747e7 --- /dev/null +++ b/modules/build-record/tests/test_build_record.cpp @@ -0,0 +1,59 @@ +import std; +import mcpplibs.mtest; +import mcpp.build.record; + +using namespace mcpplibs::mtest; + +// SUBSYSTEM-LEVEL: the line the compiler executor appends, the record a build writes from it +// (SPEC-017 B4 `builds.ndjson`), and the questions asked of the record. That a build writes it, +// and `mcpp test` keeps it in its run, is stated by the e2e suite (tests/e2e/test/streams.cpp). + +namespace rec = mcpp::build::record; + +UTest a_usage_line_reads_back{"BuildRecord/AUsageLineReadsBack", [] { + const rec::Usage u{"obj/a.o", 100, 350, 240.5, 1u << 20}; + const auto text = rec::usage_line(u) + "not json\n" + rec::usage_line({"obj/b.o", 1, 2}) + "{\"output\":"; + const auto back = rec::parse_usage(text); + expect(fatal(eq(back.size(), std::size_t{2}))); + expect(eq(back[0].output, std::string("obj/a.o"))); + expect(eq(back[0].endedUnixMs - back[0].startedUnixMs, 250LL)); + expect(back[0].cpuMs == std::optional(240.5)); + expect(back[0].peakBytes == std::optional(1u << 20)); + expect(!back[1].cpuMs && !back[1].peakBytes); +}}; + +UTest an_edge_takes_its_measurement_by_its_output{"BuildRecord/AnEdgeTakesItsMeasurementByItsOutput", [] { + rec::Record r{"b1", {{"obj/a.o", "app", "", 0, 300}, {"bin/app", "app", "", 300, 50}}, {{"dep", 12}}}; + rec::attach(r, {{"obj/a.o", 0, 1, 10.0}, {"obj/a.o", 0, 1, 280.0, 4096}}); + expect(r.edges[0].cpuMs == std::optional(280.0)) << "the newest line of an output"; + expect(r.edges[0].peakBytes == std::optional(4096)); + expect(!r.edges[1].cpuMs) << "a link the executor does not run is not measured"; + const auto t = rec::totals(r); + expect(eq(t.ms, 350.0)); + expect(eq(t.cpuMs, 280.0)); + expect(eq(t.measured, std::size_t{1})); + expect(eq(t.cachedUnits, std::size_t{12})); +}}; + +UTest a_record_reads_back_from_its_lines{"BuildRecord/ARecordReadsBackFromItsLines", [] { + rec::Record a{"b1", {{"obj/a.o", "app", "t", 5, 300, 290.0, 1024}, {"obj/c.o", "", "", 0, 900}}, {{"dep", 3}}}; + rec::Record b{"b2", {{"obj/x.o", "lib", "", 1, 2}}, {}}; + const auto back = rec::from_ndjson(rec::to_ndjson(a) + rec::to_ndjson(b)); + expect(fatal(eq(back.size(), std::size_t{2}))); + expect(eq(back[0].id, std::string("b1"))); + expect(eq(back[0].edges.size(), std::size_t{2})); + expect(eq(back[0].edges[0].program, std::string("t"))); + expect(back[0].edges[0].peakBytes == std::optional(1024)); + expect(eq(back[0].cached.size(), std::size_t{1})); + expect(eq(back[0].cached[0].units, std::size_t{3})); + expect(eq(back[1].edges[0].package, std::string("lib"))); + const auto top = rec::slowest(back[0], 1); + expect(fatal(eq(top.size(), std::size_t{1}))); + expect(eq(top[0].output, std::string("obj/c.o"))); +}}; + +UTest identifiers_differ{"BuildRecord/IdentifiersDiffer", [] { + std::set ids; + for (int i = 0; i < 64; ++i) ids.insert(rec::new_id()); + expect(ge(ids.size(), std::size_t{60})); +}}; diff --git a/src/build/execute.cppm b/src/build/execute.cppm index 62701aa21..2968b4eec 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -40,6 +40,7 @@ import mcpp.build.graph_engine; // the engine a graph names (2026-11-12 §17) import mcpplibs.os.process; // the identity of this invocation (its protocol directory) import mcpplibs.os.fs; // the budget's slot locks (SPEC-017 O11) import mcpplibs.os.machine; // the machine a run records (SPEC-017 P12.4) +import mcpp.build.record; // the run's builds (SPEC-017 B4 `builds.ndjson`) import mcpp.build.result; // the result record and the event stream (design 2026-10-26 §5) import mcpp.wire; // the mcpp.launch envelope import mcpp.build.cxx_runtime; // the island, verified after the link (design 2026-10-26 §6.3) @@ -2544,6 +2545,7 @@ export bool dep_sources_newer_than(const std::vector& depSourceRo return false; } + // mcpp#225: run ninja quietly against an already-verified-fresh build.ninja. // Shared by try_fast_build (which just reports "Finished" on success) and // try_fast_run (which goes on to locate + exec a binary). Returns nullopt @@ -2623,6 +2625,11 @@ std::optional run_ninja_fast(const std::string& ninjaProgram, std::to_string(mcpp::build::read_memory_threshold_sidecar(outputDir))); } + // The build's record, as the full path keeps it (design 2026-11-20 §5.6). + const auto logMark = mcpp::build::progress::mark_log(outputDir); + const auto usageFile = outputDir / mcpp::build::record::kUsageFile; + std::ofstream(usageFile, std::ios::binary | std::ios::trunc); + childEnv.emplace_back(std::string(mcpp::build::record::kUsageEnv), mcpplibs::os::fs::utf8(usageFile)); const auto compilerStopped=[&] {return compilerRun&&compilerRun->failed();}; auto t0 = std::chrono::steady_clock::now(); // capture_exec merges stderr into the captured output (replacing `2>&1`), @@ -2668,8 +2675,10 @@ std::optional run_ninja_fast(const std::string& ninjaProgram, reported = run.reported; // A stale graph is not this build's outcome: the full path plans // again and builds, and reports the packages then. - if (status == 0 || reported || !is_stale_ninja_failure(out)) + if (status == 0 || reported || !is_stale_ninja_failure(out)) { + mcpp::build::progress::keep_ran(outputDir, logMark, usageFile, &report); report.finish(status == 0); + } } else { // Nobody reads this ninja's progress: it reports no action start // (build progress design 2026-09-29, §6.4). @@ -2679,6 +2688,7 @@ std::optional run_ninja_fast(const std::string& ninjaProgram, if (r.exit_code == 0) r = mcpp::run::execute(argv,{.env=childEnv,.should_stop=compilerStopped}); out = std::move(r.output); status = r.exit_code; + if (status == 0 || !is_stale_ninja_failure(out)) mcpp::build::progress::keep_ran(outputDir, logMark, usageFile, nullptr); } const auto bmiRun = mcpp::buildopt::keep::run_end(outputDir); for (auto const& line : mcpp::buildopt::keep::summary(bmiRun)) @@ -4746,6 +4756,14 @@ void write_mtest_store(const std::filesystem::path& dir, const std::vectorempty()) run["jitter"] = {{"max_ms", std::atoi(jitter->c_str())}, {"seed", std::strtoull(mcpp::invocation::get("MTEST_JITTER_SEED").value_or("0").c_str(), nullptr, 10)}}; diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index c6db362e2..de785696f 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -52,6 +52,8 @@ import mcpp.modgraph.graph; import mcpplibs.os.platform; import mcpplibs.os.windows; import mcpp.invocation; +import mcpp.build.record; // what a build ran, edge by edge (design 2026-11-20 §5.6) +import mcpplibs.os.fs; // the usage file's spelling for the executor import mcpp.runtime.launch_env; import mcpp.build.command_line; import mcpp.run; @@ -5271,6 +5273,12 @@ std::expected NinjaBackend::build(const BuildPlan& plan nenv.emplace_back(ev.key, ev.value); if (compilerRun) compilerRun->environment(nenv); nenv.emplace_back("MCPP_MEMORY_THRESHOLD", std::to_string(plan.memoryThreshold)); + // THE BUILD'S RECORD (design 2026-11-20 §5.6): where ninja's log ends now, + // and the file each compile's executor appends its measurement to. + const auto logMark = mcpp::build::progress::mark_log(plan.outputDir); + const auto usageFile = plan.outputDir / mcpp::build::record::kUsageFile; + std::ofstream(usageFile, std::ios::binary | std::ios::trunc); + nenv.emplace_back(std::string(mcpp::build::record::kUsageEnv), mcpplibs::os::fs::utf8(usageFile)); const auto compilerStopped=[&] {return compilerRun&&compilerRun->failed();}; // The directory's BMI cutoff for this run (mcpp.buildopt.keep). A graph // compiled with contracts is compared exactly: a contract's violation @@ -5427,6 +5435,9 @@ std::expected NinjaBackend::build(const BuildPlan& plan for (auto const& c : advice::single_file_cascades(units, bmiRun.changed)) if (advice::first_time(plan.outputDir, c.module)) mcpp::ui::info("note", advice::line(c)); } + // The edges this build ran, kept for the command (`mcpp test` writes them + // into its run) and appended to MCPP_BUILD_RECORD's file. + mcpp::build::progress::keep_ran(plan.outputDir, logMark, usageFile, report); // Every package gets its final line before what the build has to say // after ninja: validations, advice, `Finished`. if (report) { diff --git a/src/build/progress.cppm b/src/build/progress.cppm index ca4882c0a..48e3ac9e6 100644 --- a/src/build/progress.cppm +++ b/src/build/progress.cppm @@ -39,6 +39,7 @@ import mcpp.log; import mcpp.invocation; import mcpplibs.os.fs; import mcpp.ui.terminal; +import mcpp.build.record; // what a build did, edge by edge (design 2026-11-20 §5.6) export namespace mcpp::build::progress { @@ -281,6 +282,10 @@ public: // for a later failure. std::optional failed(std::string_view outputs); void pass_end(); + // Under --verbose, the five edges of `ran` that took longest, each by its + // package and source as the lines name it, with its processor time and + // largest resident set (design 2026-11-20 §5.6). + void slowest(const mcpp::build::record::Record& ran); // The build ended. Under --verbose, the packages with nothing to do are // named, and each package that did work states its steps and span. void finish(bool success); @@ -289,6 +294,26 @@ private: std::shared_ptr impl_; }; +// THE EDGES A BUILD RAN (design 2026-11-20 §5.6). `mark_log` notes where +// ninja's log of `buildDir` ends before the build; `ran_since` reads the +// entries after it, attributes each to its package by the step record, takes +// what the compiler executor measured from its usage file and adds a line per +// package the cache served. A log ninja recompacted in between has no mark to +// read from: the edges are then those the executor measured, which leaves +// out the steps it does not run (dyndep collation, actions). +struct LogMark { + std::uintmax_t size = 0; + std::string tail; // the last bytes before the mark +}; +LogMark mark_log(const std::filesystem::path& buildDir); +mcpp::build::record::Record ran_since(const std::filesystem::path& buildDir, const LogMark& mark, + const std::filesystem::path& usageFile); +// The edges a build ran (`ran_since`), when any ran: kept for the command +// (`mcpp test` writes them into its run, SPEC-017 B4), appended to the file +// MCPP_BUILD_RECORD names, and, with a report, its slowest named. +void keep_ran(const std::filesystem::path& buildDir, const LogMark& mark, + const std::filesystem::path& usageFile, Build* report); + } // namespace mcpp::build::progress namespace mcpp::build::progress { @@ -694,6 +719,7 @@ struct Running { std::string stamp; long long since = 0; // ms since command start std::optional snapshot; + bool told = false; // its line as a long step was written }; struct Build::Impl { @@ -1023,14 +1049,33 @@ void read_starts(Report& r, Build::Impl& b, std::vector& out) { for (auto const& a : starts) { auto owner = b.record->owner.find(a.stamp); if (owner == b.record->owner.end()) continue; - auto label = b.record->actions.find(a.stamp); - b.running.push_back({owner->second, - label != b.record->actions.end() ? label->second : a.stamp, - a.stamp, now_ms() - std::max(0, nowUnix - a.unixMs)}); + // An action by its label, a compile by its source file (the executor + // writes a compile's start, design 2026-11-20 §5.6). + std::string label = a.stamp; + if (auto it = b.record->actions.find(a.stamp); it != b.record->actions.end()) label = it->second; + else if (auto sid = b.record->step.find(a.stamp); sid != b.record->step.end()) + if (auto src = b.record->sources.find(sid->second); src != b.record->sources.end()) label = src->second; + b.running.push_back({owner->second, std::move(label), a.stamp, + now_ms() - std::max(0, nowUnix - a.unixMs)}); announce(r, b, owner->second, out); } } +// A STEP THAT RUNS A MINUTE IS NAMED (design 2026-11-20 §5.6): once, with the +// time it has run. A test build's long tail (boost-m, §3.3) showed nothing for +// many minutes on a log that is not a terminal, where the status row is not +// drawn; model lock held. +void tell_long_steps(Report& r, Build::Impl& b, std::vector& out) { + if (!b.record) return; + for (auto& a : b.running) { + if (a.told || now_ms() - a.since < 60'000) continue; + a.told = true; + out.push_back(mcpp::ui::step_line( + "Running", std::format("{}: {} for {}", subject_of(r, b, b.record->packages[a.package]), a.label, + mcpp::ui::format_clock(ms(now_ms() - a.since))), 0, "")); + } +} + // Read only active graph actions. Snapshot counts never complete an action; // the ninja log removes it from running only after the command has finished. bool read_action_progress(Build::Impl& build) { @@ -1195,6 +1240,7 @@ void poll() { read_starts(r, *b, out); read_log(r, *b, out); progressChanged |= read_action_progress(*b); + tell_long_steps(r, *b, out); } } write_lines(out); @@ -1696,6 +1742,113 @@ void Build::pass_end() { write_lines(out); } +void Build::slowest(const mcpp::build::record::Record& ran) { + auto& r = report(); + std::vector out; + { + std::lock_guard lock(r.m); + auto& b = *impl_; + ensure_record(b); + if (!r.verbose || ran.edges.empty()) return; + for (const auto& e : mcpp::build::record::slowest(ran, 5)) { + // A compile by its source file, any other edge by its output. + std::string label = e.output; + if (b.record) + if (auto sid = b.record->step.find(normalise(e.output)); sid != b.record->step.end()) + if (auto src = b.record->sources.find(sid->second); src != b.record->sources.end()) + label = src->second; + if (!e.package.empty()) label = std::format("{}: {}", e.package, label); + std::string used = mcpp::ui::format_duration(ms(static_cast(e.ms))); + if (e.cpuMs) used += " · processor " + mcpp::ui::format_duration(ms(static_cast(*e.cpuMs))); + if (e.peakBytes) used += std::format(" · {} MiB", *e.peakBytes >> 20); + out.push_back(mcpp::ui::step_line("Slowest", std::format("{} · {}", label, used), 0, "")); + } + } + write_lines(out); +} + +void keep_ran(const std::filesystem::path& buildDir, const LogMark& mark, + const std::filesystem::path& usageFile, Build* report) { + namespace rec = mcpp::build::record; + auto ran = ran_since(buildDir, mark, usageFile); + if (ran.edges.empty()) return; + if (const auto file = mcpp::invocation::get(rec::kRecordEnv); file && !file->empty()) + (void)mcpplibs::os::fs::append(mcpplibs::os::fs::from_utf8(*file), rec::to_ndjson(ran)); + if (report) report->slowest(ran); + rec::keep(std::move(ran)); +} + +LogMark mark_log(const std::filesystem::path& buildDir) { + LogMark m; + const auto log = buildDir / ".ninja_log"; + std::error_code ec; + m.size = std::filesystem::exists(log, ec) ? std::filesystem::file_size(log, ec) : 0; + if (ec) m.size = 0; + if (m.size > 0) { + std::ifstream f(log, std::ios::binary); + const auto n = std::min(m.size, 256); + f.seekg(static_cast(m.size - n)); + m.tail.resize(static_cast(n)); + f.read(m.tail.data(), static_cast(n)); + if (!f) m.tail.clear(); + } + return m; +} + +mcpp::build::record::Record ran_since(const std::filesystem::path& buildDir, const LogMark& mark, + const std::filesystem::path& usageFile) { + namespace rec = mcpp::build::record; + rec::Record out{rec::new_id()}; + std::vector usage; + if (auto text = mcpplibs::os::fs::read(usageFile)) usage = rec::parse_usage(*text); + for (auto& u : usage) u.output = normalise(u.output); // as the log spells it + const auto stepRecord = read_record(buildDir); + std::string text; + if (auto all = mcpplibs::os::fs::read(buildDir / ".ninja_log")) text = std::move(*all); + // The log as it was at the mark is a prefix of it now, unless ninja + // recompacted it. + const bool kept = text.size() >= mark.size + && std::string_view(text).substr(static_cast(mark.size - mark.tail.size()), mark.tail.size()) + == mark.tail; + std::set measured; + for (const auto& u : usage) measured.insert(normalise(u.output)); + std::size_t consumed = 0; + auto steps = parse_log(kept ? std::string_view(text).substr(static_cast(mark.size)) + : std::string_view(text), &consumed); + // The log's times are after each ninja run began, and a build runs ninja + // up to three times (placement, scan, work). An edge's start after the + // build began is the executor's where it measured the edge; another edge + // is placed by the nearest measured edge before it in the log, whose run + // it shares. + const long long began = usage.empty() ? 0 + : std::ranges::min(usage, {}, &rec::Usage::startedUnixMs).startedUnixMs; + std::unordered_map byOutput; + for (const auto& u : usage) byOutput[u.output] = &u; + long long offset = 0; // the build's time minus the log's, from the last measured edge + for (const auto& st : steps) { + if (st.outputs.empty()) continue; + const auto& first = st.outputs.front(); + if (!kept && !measured.contains(first)) continue; + rec::Edge e{first}; + if (stepRecord) + for (const auto& o : st.outputs) + if (auto it = stepRecord->owner.find(o); it != stepRecord->owner.end()) { + e.package = stepRecord->packages[it->second].name; + break; + } + e.ms = static_cast(st.end - st.start); + if (auto u = byOutput.find(first); u != byOutput.end()) + offset = u->second->startedUnixMs - began - st.start; + e.startedMs = static_cast(std::max(0LL, st.start + offset)); + out.edges.push_back(std::move(e)); + } + rec::attach(out, usage); + if (stepRecord) + for (const auto& p : stepRecord->packages) + if (p.cachedUnits > 0) out.cached.push_back({p.name, p.cachedUnits}); + return out; +} + void Build::finish(bool success) { auto& r = report(); std::vector out; diff --git a/src/cli.cppm b/src/cli.cppm index 95ce62904..ccec26e28 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -31,6 +31,8 @@ import mcpp.wire; import mcpp.cli.cmd_sbom; import mcpplibs.os.platform; import mcpplibs.os.fs; +import mcpplibs.os.process; // the compiler's use, measured by the executor +import mcpp.build.record; // the line the executor appends (design 2026-11-20 §5.6) import mcpplibs.os.text; import mcpp.invocation; import mcpp.effects; @@ -320,8 +322,27 @@ int run(int argc, char** argv) { if (semaphore.empty() || cap < 0 || i + 1 >= argc) return 2; std::vector command; for (++i; i < argc; ++i) command.emplace_back(argv[i]); - return mcpp::build::schedule::detach::run_capped(command, semaphore, cap, output, depfile, bmi, object, relocate, - reading); + // THE EDGE, MEASURED (design 2026-11-20 §5.6): its start, for the + // progress line of a long compile, and, once the compiler is reaped, + // its processor time and largest resident set -- this process starts + // no other program, so its children's use is the compiler's. + mcpp::build::progress::record_action_start(output.generic_string()); + const auto unixMs = [] { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + }; + const auto started = unixMs(); + const int status = mcpp::build::schedule::detach::run_capped(command, semaphore, cap, output, depfile, bmi, + object, relocate, reading); + if (const auto file = mcpp::invocation::get(mcpp::build::record::kUsageEnv); file && !file->empty()) { + mcpp::build::record::Usage u{output.generic_string(), started, unixMs()}; + if (const auto used = mcpplibs::os::process::resources(mcpplibs::os::process::of::children)) { + u.cpuMs = std::chrono::duration(used->user + used->system).count(); + u.peakBytes = used->peak_memory; + } + (void)mcpplibs::os::fs::append(mcpp::home::from_utf8(*file), mcpp::build::record::usage_line(u)); + } + return status; } ReportUnnarrowablePaths reportUnnarrowable_; diff --git a/tests/e2e/test/streams.cpp b/tests/e2e/test/streams.cpp index e58e5a84a..9c4c8951b 100644 --- a/tests/e2e/test/streams.cpp +++ b/tests/e2e/test/streams.cpp @@ -87,6 +87,37 @@ main = "src/main.cpp" auto b = p.mcpp({"build"}, {.env = {{"MTEST_BUDGET", budget.string()}, {"MTEST_BUDGET_HELD", (budget / "slot-0").string()}}}); expect(e2e::succeeded(b)) << "B: a build within a budget of one slot: " << s::text(b); }; + "927 a build records each edge it ran, and a test run keeps the builds it ran"_test[s::gcc] = [] { + // Design 2026-11-20 §5.6, SPEC-017 B4. A build appends its record to the file + // MCPP_BUILD_RECORD names: each edge that ran, with its package, its time and, from the + // compiler executor, its processor time and largest resident set (A). `--verbose` names the + // slowest edges (A). `mcpp test` keeps the builds it ran in its run as `builds.ndjson` (B). + s::project p{{"mcpp.toml", "[package]\nname = \"p\"\nversion = \"0.1.0\"\n\n[targets.p]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}, + {"src/main.cpp", "import p.a;\nint main() { return a() == 1 ? 0 : 1; }\n"}, + {"src/a.cppm", "export module p.a;\nexport int a() { return 1; }\n"}, + {"tests/t.cpp", "int main() { return 0; }\n"}}; + const auto file = s::case_dir() / "record.ndjson"; + auto a = p.mcpp({"build", "--verbose"}, {.env = {{"MCPP_BUILD_RECORD", file.string()}}}); + expect(fatal(e2e::succeeded(a))) << s::text(a); + expect(e2e::contains(s::text(a), "Slowest")) << "A: --verbose names no slowest edge"; + const auto lines = s::json_lines(s::read_file(file)); + expect(fatal(!lines.empty())) << "A: the build appended no record"; + bool compiled = false; + for (const auto& l : lines) { + expect(!l.value("build", "").empty()) << l.dump(); + // The interface unit's object (`obj/a.m.o`), not its scan. + const auto out = l.value("output", ""); + if (out.contains("/a.") && out.ends_with(".o")) + compiled = l.value("package", "") == "p" && l.value("ms", -1.0) >= 0 && l.value("cpu_ms", 0.0) > 0; + } + expect(compiled) << "A: the compile of a.cppm, its package and its processor time: " << s::read_file(file); + auto b = p.mcpp({"test"}); + expect(fatal(e2e::succeeded(b))) << s::text(b); + const auto runs = s::json_lines(p.read("target/mtest/index.ndjson")); + expect(fatal(!runs.empty())); + const auto builds = s::json_lines(s::read_file(p.at("target/mtest/runs") / runs.back()["id"].get() / "builds.ndjson")); + expect(!builds.empty()) << "B: the run kept no build"; + }; "178 test observability"_test[s::unix_shell] = [] { // `mcpp test` is streamed, timed and bounded. POSIX only: the deadline runner has no // kill-by-handle path on Windows, so --timeout and --build-timeout are documented POSIX-only. diff --git a/tests/unit/test_build_progress.cpp b/tests/unit/test_build_progress.cpp index 6d0e3a373..471576d29 100644 --- a/tests/unit/test_build_progress.cpp +++ b/tests/unit/test_build_progress.cpp @@ -5,6 +5,7 @@ import mcpplibs.os.stdio; import mcpp.ui; import mcpp.build.progress; import mcpp.build.action_progress; +import mcpp.build.record; using namespace mcpplibs::mtest; @@ -240,6 +241,43 @@ UTest a_compile_is_named_by_its_source_and_the_header_carries_the_descriptor{"Pr expect(eq(read_descriptor(t.path / "absent"), "")); }}; +UTest the_edges_after_the_mark_are_the_builds{"ProgressRecord/TheEdgesAfterTheMarkAreTheBuilds", [] { + // Design 2026-11-20 §5.6: the entries ninja appended after the mark, each + // by its package, with what the executor measured; a cached package is a + // line of its own. A recompacted log has no mark: the edges are those the + // executor measured. + Tmp t; + Attribution a; + a.owner("app"); + a.statement("build obj/a.o: cxx_obj a.cpp\n"); + a.statement("build obj/a.cpp.ddi.dd: dd obj/a.cpp.ddi\n"); + auto r = a.record({{"app", true, "app", 0, 0}, {"dep", false, "dep", 7, 0}}); + write_record(t.path, r); + const auto log = t.path / ".ninja_log"; + append(log, "# ninja log v6\n0\t5\t1\tobj/old.o\tff\n"); + const auto mark = mark_log(t.path); + append(log, "0\t2\t1\tobj/a.cpp.ddi.dd\taa\n10\t310\t1\tobj/a.o\tbb\n"); + const auto usage = t.path / "usage"; + append(usage, mcpp::build::record::usage_line({"obj/a.o", 5000, 5300, 290.0, 1u << 20})); + auto ran = ran_since(t.path, mark, usage); + expect(fatal(eq(ran.edges.size(), std::size_t{2}))); + expect(eq(ran.edges[0].output, std::string("obj/a.cpp.ddi.dd"))); + expect(eq(ran.edges[1].output, std::string("obj/a.o"))); + expect(eq(ran.edges[1].package, std::string("app"))); + expect(eq(ran.edges[1].ms, 300.0)); + expect(ran.edges[1].cpuMs == std::optional(290.0)); + expect(fatal(eq(ran.cached.size(), std::size_t{1}))); + expect(eq(ran.cached[0].package, std::string("dep"))); + expect(eq(ran.cached[0].units, std::size_t{7})); + // Recompacted: shorter than at the mark. + std::ofstream(log, std::ios::binary | std::ios::trunc) << "# ninja log v6\n0\t2\t1\tobj/a.cpp.ddi.dd\taa\n10\t310\t1\tobj/a.o\tbb\n"; + auto big = mark; + big.size = 4096; + ran = ran_since(t.path, big, usage); + expect(fatal(eq(ran.edges.size(), std::size_t{1}))); + expect(eq(ran.edges[0].output, std::string("obj/a.o"))); +}}; + UTest paths_are_compared_normalised{"ProgressRecord/PathsAreComparedNormalised", [] { expect(eq(normalise("./obj\\a.o"), "obj/a.o")); expect(eq(normalise("obj/../obj/b.o"), "obj/b.o")); From 8bf0bb92a9298de80c1ad209b6bd9554923a74aa Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 02:52:04 +0900 Subject: [PATCH 09/38] chore: mcpplibs.mtest 0.11.0 and mcpplibs.os 0.4.7 throughout, the skeleton's mtest and the launchers' os with them --- bench/mcpp.toml | 2 +- docs/05-dependencies.md | 4 ++-- docs/zh/05-dependencies.md | 4 ++-- mcpp.toml | 2 +- modules/build-record/mcpp.toml | 2 +- modules/buildmcpp/mcpp.toml | 2 +- modules/buildopt/mcpp.toml | 2 +- modules/graph/mcpp.toml | 2 +- modules/manifest/mcpp.toml | 2 +- modules/toolchain-model/mcpp.toml | 2 +- modules/versioning/mcpp.toml | 2 +- src/build/test_protocol.cppm | 2 +- src/pack/distribution.cppm | 2 +- tests/canary/cxx/mcpp.toml | 2 +- tests/e2e/deps/graph.cpp | 3 ++- tests/support/mcpp.toml | 2 +- 16 files changed, 19 insertions(+), 18 deletions(-) diff --git a/bench/mcpp.toml b/bench/mcpp.toml index 20021a9bb..10dbaa374 100644 --- a/bench/mcpp.toml +++ b/bench/mcpp.toml @@ -21,4 +21,4 @@ windows = "llvm@23.1.3" # small fixture. `mcpp build` ignores [dev-dependencies], so the harness itself # still takes nothing beyond `import std;`. [dev-dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" diff --git a/docs/05-dependencies.md b/docs/05-dependencies.md index f42f4ef6c..dca3fe935 100644 --- a/docs/05-dependencies.md +++ b/docs/05-dependencies.md @@ -67,8 +67,8 @@ not deprecated: ```toml [dev-dependencies] -mcpplibs.mtest = "0.10.0" -"mcpplibs.mtest.gtest" = "0.10.0" +mcpplibs.mtest = "0.11.0" +"mcpplibs.mtest.gtest" = "0.11.0" ``` ```toml diff --git a/docs/zh/05-dependencies.md b/docs/zh/05-dependencies.md index d3ce92856..694b90cfb 100644 --- a/docs/zh/05-dependencies.md +++ b/docs/zh/05-dependencies.md @@ -62,8 +62,8 @@ mcpp 把它读作这些点所拼写的选择器,警告该拼法已弃用(自 ```toml [dev-dependencies] -mcpplibs.mtest = "0.10.0" -"mcpplibs.mtest.gtest" = "0.10.0" +mcpplibs.mtest = "0.11.0" +"mcpplibs.mtest.gtest" = "0.11.0" ``` ```toml diff --git a/mcpp.toml b/mcpp.toml index abc349035..16e1c421b 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -182,7 +182,7 @@ members = [ # `mcpp build` ignores [dev-dependencies]; only `mcpp test` resolves them. [dev-dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" # The support of the end-to-end cases (tests/support, design 2026-11-12 §5.9). mcpp.test-support = { path = "tests/support" } diff --git a/modules/build-record/mcpp.toml b/modules/build-record/mcpp.toml index 25cb503eb..27754ce01 100644 --- a/modules/build-record/mcpp.toml +++ b/modules/build-record/mcpp.toml @@ -17,4 +17,4 @@ license = "Apache-2.0" nlohmann.json = "3.12.0" [dev-dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" diff --git a/modules/buildmcpp/mcpp.toml b/modules/buildmcpp/mcpp.toml index e02b2515f..ac7fd6b02 100644 --- a/modules/buildmcpp/mcpp.toml +++ b/modules/buildmcpp/mcpp.toml @@ -41,4 +41,4 @@ mcpp.source-kind = { path = "../source-kind" } mcpp.toolchain-model = { path = "../toolchain-model" } [dev-dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" diff --git a/modules/buildopt/mcpp.toml b/modules/buildopt/mcpp.toml index 083c09659..ab3a03ccb 100644 --- a/modules/buildopt/mcpp.toml +++ b/modules/buildopt/mcpp.toml @@ -21,4 +21,4 @@ mcpp.manifest = { path = "../manifest" } mcpp.toolchain-model = { path = "../toolchain-model" } [dev-dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" diff --git a/modules/graph/mcpp.toml b/modules/graph/mcpp.toml index dd56ac94d..fa53d0444 100644 --- a/modules/graph/mcpp.toml +++ b/modules/graph/mcpp.toml @@ -17,4 +17,4 @@ description = "Algorithms over index-addressed directed graphs: topological orde license = "Apache-2.0" [dev-dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" diff --git a/modules/manifest/mcpp.toml b/modules/manifest/mcpp.toml index d1dc930c9..7f4b4f6e3 100644 --- a/modules/manifest/mcpp.toml +++ b/modules/manifest/mcpp.toml @@ -30,4 +30,4 @@ mcpplibs.os = "0.4.7" # state its contract in isolation; the root `tests/unit/` states the facts two # layers must agree on. [dev-dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" diff --git a/modules/toolchain-model/mcpp.toml b/modules/toolchain-model/mcpp.toml index 155505710..7b07c45a2 100644 --- a/modules/toolchain-model/mcpp.toml +++ b/modules/toolchain-model/mcpp.toml @@ -29,7 +29,7 @@ mcpplibs.os = "0.4.7" # state its contract in isolation; the root `tests/unit/` states the facts two # layers must agree on. [dev-dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" # build.mcpp compiles data/builtin.toml (the built-in target rows and providers) # into a module of static tables, with mcpp's own TOML parser. diff --git a/modules/versioning/mcpp.toml b/modules/versioning/mcpp.toml index 7c5d19a47..0e4ff3542 100644 --- a/modules/versioning/mcpp.toml +++ b/modules/versioning/mcpp.toml @@ -25,4 +25,4 @@ license = "Apache-2.0" # state its contract in isolation; the root `tests/unit/` states the same facts # as the engine consumes them, and the pair is the cross-check. [dev-dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" diff --git a/src/build/test_protocol.cppm b/src/build/test_protocol.cppm index cd31ccf90..8e71f1ad8 100644 --- a/src/build/test_protocol.cppm +++ b/src/build/test_protocol.cppm @@ -19,7 +19,7 @@ inline constexpr int kVersion = 1; // The framework mcpp scaffolds tests with and suggests where a package has // none (docs/01, docs/08): mtest, by this release, from its repository. -inline constexpr std::string_view kMtestRelease = "0.10.0"; +inline constexpr std::string_view kMtestRelease = "0.11.0"; inline constexpr std::string_view kMtestRepo = "https://github.com/mcpp-community/mtest"; // The protocol's variables, removed from the environment a program inherits diff --git a/src/pack/distribution.cppm b/src/pack/distribution.cppm index f5f4c33c4..ab6877bba 100644 --- a/src/pack/distribution.cppm +++ b/src/pack/distribution.cppm @@ -96,7 +96,7 @@ std::string exe_launcher_main(const Launch& l); // When the two shared one resolution, this had to be the release the plugin // linked (0.4.5, through mcpp.plugins 0.26.1): 0.4.4 and 0.4.6 were refused as // irreconcilable (e2e 988, the canaries' pack cells). -inline constexpr std::string_view kLauncherOs = "0.4.6"; +inline constexpr std::string_view kLauncherOs = "0.4.7"; // ─── mcpp-package.json (SPEC-014 §4.5) ──────────────────────────────────── diff --git a/tests/canary/cxx/mcpp.toml b/tests/canary/cxx/mcpp.toml index a70b33875..269c4e9fe 100644 --- a/tests/canary/cxx/mcpp.toml +++ b/tests/canary/cxx/mcpp.toml @@ -9,4 +9,4 @@ kind = "bin" main = "src/main.cpp" [dev-dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" diff --git a/tests/e2e/deps/graph.cpp b/tests/e2e/deps/graph.cpp index 225a25526..72fa3d067 100644 --- a/tests/e2e/deps/graph.cpp +++ b/tests/e2e/deps/graph.cpp @@ -284,7 +284,8 @@ int main() { const auto marker = s::read_file(g.parent_path() / ".mcpp-build-programs"); if (std::regex_search(marker, v, std::regex{R"(pin=mcpplibs\.os=([0-9.]+))"})) pinned.insert(v[1].str()); } - expect(fatal(pinned == std::set{"0.4.1", "0.4.6"})) << "A: the graphs do not take one version each\n" << s::text(r); + // `^0.4.4` is the newest 0.4 release the index serves: the test names the one it pins. + expect(fatal(pinned.size() == 2u && pinned.contains("0.4.1"))) << "A: the graphs do not take one version each\n" << s::text(r); const auto lock = app.read("mcpp.lock"); expect(fatal(s::has_line(lock, R"re(^\[package\."(mcpplibs\.)?os"\])re"))) << "A: the lock has no entry for the first version\n" << lock; expect(fatal(s::has_line(lock, R"re(^\[package\."(mcpplibs\.)?os\+b[0-9a-f]{12}"\])re"))) << "A: the lock has no entry for the second version\n" << lock; diff --git a/tests/support/mcpp.toml b/tests/support/mcpp.toml index 1bf503f01..778ed1c4b 100644 --- a/tests/support/mcpp.toml +++ b/tests/support/mcpp.toml @@ -13,5 +13,5 @@ license = "Apache-2.0" kind = "lib" [dependencies] -mcpplibs.mtest = "0.10.0" +mcpplibs.mtest = "0.11.0" nlohmann.json = "3.12.0" From f7d3a16d2b9d9bb8b13c1c0d9038744be3d351fe Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 02:52:04 +0900 Subject: [PATCH 10/38] feat: the dependencies from the store and git compile with the line tables under dev ([profile.] dependency_debug; #927) --- modules/manifest/src/toml.cppm | 7 ++++ modules/manifest/src/types.cppm | 10 ++++++ modules/toolchain-model/src/dialect.cppm | 6 ++++ src/build/prepare/plan.cpp | 45 ++++++++++++++++++++++++ src/build/prepare/toolchain.cpp | 18 ++++++++++ src/build/prepare_inputs.cppm | 1 + src/project.cppm | 5 +++ tests/e2e/build/flags.cpp | 33 +++++++++++++++++ 8 files changed, 125 insertions(+) diff --git a/modules/manifest/src/toml.cppm b/modules/manifest/src/toml.cppm index 117b7d3d8..0c25522f6 100644 --- a/modules/manifest/src/toml.cppm +++ b/modules/manifest/src/toml.cppm @@ -1347,6 +1347,13 @@ std::expected parse_document(const t::Document& documen pr.dependencyLinkage = it->second.as_string(); pr.dependencyLinkageDeclared = true; } + if (auto it = tt.find("dependency_debug"); it != tt.end() && it->second.is_bool()) { + pr.dependencyDebug = it->second.as_bool() ? "true" : "false"; + pr.dependencyDebugDeclared = true; + } else if (it != tt.end() && it->second.is_string()) { + pr.dependencyDebug = it->second.as_string(); + pr.dependencyDebugDeclared = true; + } read_list("cflags", pr.cflags); read_list("cxxflags", pr.cxxflags); read_list("ldflags", pr.ldflags); diff --git a/modules/manifest/src/types.cppm b/modules/manifest/src/types.cppm index 13b6dcc4e..1ad60f63c 100644 --- a/modules/manifest/src/types.cppm +++ b/modules/manifest/src/types.cppm @@ -1294,6 +1294,10 @@ struct BuildConfig : BuildInputs { std::string optLevel = "2"; // -O level bool debug = false; // -g bool debugLines = false; // -g1: the line tables alone + // The debug information of the dependencies from the store and git, as + // the profile's `dependency_debug` resolves it (design 2026-11-20 §5.15): + // "true", "false" or "lines"; empty when it is the graph's own. + std::string dependencyDebug; bool lto = false; // -flto, /GL + /LTCG on cl.exe // LTO was requested and some objects of the plan are compiled without it: // those that feed a PE shared library whose exports are discovered, or a @@ -2323,6 +2327,12 @@ struct Profile { // about the member that caused it. std::string dependencyLinkage; bool dependencyLinkageDeclared = false; + // `dependency_debug` (design 2026-11-20 §5.15): the debug information of + // the dependencies from the store and git, `true`, `false` or `"lines"`, + // as written ("true"/"false" for a boolean). Not written: "lines" for + // `dev` and `debug`, else the profile's `debug`. + std::string dependencyDebug; + bool dependencyDebugDeclared = false; // Passthrough escape hatch (fixed keys, open values — I6 completeness): std::vector cflags; std::vector cxxflags; diff --git a/modules/toolchain-model/src/dialect.cppm b/modules/toolchain-model/src/dialect.cppm index 4c964e0fd..0ac6e4c58 100644 --- a/modules/toolchain-model/src/dialect.cppm +++ b/modules/toolchain-model/src/dialect.cppm @@ -35,6 +35,10 @@ struct CommandDialect { // GCC's minimal level and clang's `-gline-tables-only`; cl.exe has no // level below /Z7. std::string_view lineDebugFlags; // "-g1" | "/Z7" + // No debug information, after a flag that asked for it: a dependency + // whose profile states `dependency_debug = false` (design 2026-11-20 + // §5.15). cl.exe has no flag that takes /Z7 back: empty. + std::string_view noDebugFlags; // "-g0" | "" std::string_view alwaysFlags; // "" | "/nologo /EHsc /utf-8" // Same rationale as forceCxxLangArgv: the argv consumer gets tokens, not // a string it has to split. Both spans point at static arrays below. @@ -272,6 +276,7 @@ constexpr CommandDialect kGnuDialect{ .optPrefix = "-O", .debugFlags = "-g", .lineDebugFlags = "-g1", + .noDebugFlags = "-g0", .alwaysFlags = "", .alwaysFlagsArgv = {}, .libFlag = "-l{}", @@ -313,6 +318,7 @@ constexpr CommandDialect kMsvcDialect{ // own, which the object cache and a taken-over build directory assume. .debugFlags = "/Z7", .lineDebugFlags = "/Z7", + .noDebugFlags = "", .alwaysFlags = "/nologo /EHsc /utf-8", .alwaysFlagsArgv = kMsvcAlwaysArgv, .libFlag = "{}.lib", diff --git a/src/build/prepare/plan.cpp b/src/build/prepare/plan.cpp index 34517f85d..9b2114c91 100644 --- a/src/build/prepare/plan.cpp +++ b/src/build/prepare/plan.cpp @@ -436,7 +436,52 @@ static std::expected step13_link_forms(PrepareState& state, B return {}; } +// THE DEPENDENCIES' DEBUG INFORMATION (design 2026-11-20 §5.15, SPEC-004 §9 +// item 18): the packages from the store and git compile with the level the +// profile's `dependency_debug` resolves to, its flag after the graph's. It is +// one of each package's own flags, so it is part of the package's +// configuration: its instance digest and its cache key change with it. +static void step13_dependency_debug(PrepareState& state) { + const auto& level = state.m->buildConfig.dependencyDebug; + if (level.empty() || !state.tc) return; + const auto& d = mcpp::toolchain::dialect_for(*state.tc); + const std::string_view flag = level == "lines" ? d.lineDebugFlags : level == "true" ? d.debugFlags : d.noDebugFlags; + if (flag.empty()) { + if (level == "false") + mcpp::diag::degraded("profile/dependency-debug", + "dependency_debug = false: this compiler has no flag that takes back the debug " + "information the profile asks for, so the dependencies keep it", + "the dependencies' objects carry their debug information"); + return; + } + const auto element = mcpp::manifest::flag_element(std::string(flag)); + std::set named; + // The resolution's record of each package, as the report names its source. + auto kind_of = [&](std::size_t i) -> std::string { + for (auto const& [key, r] : state.resolved) + if (r.depIndex + 1 == i) return r.source; + return i - 1 < state.dep_cache_identities.size() ? state.dep_cache_identities[i - 1].sourceKind : std::string{}; + }; + for (std::size_t i = 1; i < state.packages.size(); ++i) { + const auto kind = kind_of(i); + if ((kind != "version" && kind != "git") || state.packages[i].selectedMember) continue; + auto& pkg = state.packages[i]; + for (auto* flags : {&pkg.privateBuild.cflags, &pkg.privateBuild.cxxflags, &pkg.manifest.buildConfig.cflags, + &pkg.manifest.buildConfig.cxxflags}) + flags->push_back(element); + named.insert(mcpp::build::qualified_package_name(pkg.manifest)); + } + // The scan copied each unit's flags from its package before this step. + for (auto& u : state.scan.graph.units) + if (named.contains(u.packageName)) { + u.packageCflags.push_back(element); + u.packageCxxflags.push_back(element); + } + mcpp::log::verbose("profile", std::format("dependency_debug = {}: {} on {} package(s)", level, flag, named.size())); +} + static std::expected step13_make_plan(PrepareState& state, BuildContext& ctx) { + step13_dependency_debug(state); // An image that is not the root's links with its owner's closure's own // flags (workspace design 2026-09-29 §15; for a dependency's shared // library in every plan since 2026.10.5.2): each package's link flags, diff --git a/src/build/prepare/toolchain.cpp b/src/build/prepare/toolchain.cpp index 49df6bead..d77d325f0 100644 --- a/src/build/prepare/toolchain.cpp +++ b/src/build/prepare/toolchain.cpp @@ -271,6 +271,24 @@ static std::expected step1_define_early_toolchain_closures(Pr "[profile.{}] debug = \"{}\": expected true, false or \"lines\"", pname, pr.debugWritten)); state.m->buildConfig.debug = pr.debug; state.m->buildConfig.debugLines = pr.debugLines; + // THE DEPENDENCIES' DEBUG INFORMATION (design 2026-11-20 §5.15): the + // packages from the store and git get the line tables under the + // profiles a developer debugs, unless the profile states otherwise. + // Their debug information made 8.5 of a test program's 11.5 MB, and + // 15.4 of a workspace's 22 GB of test programs (mcpp#927); a backtrace + // through them keeps its files and lines. + { + if (pr.dependencyDebugDeclared && pr.dependencyDebug != "true" && pr.dependencyDebug != "false" + && pr.dependencyDebug != "lines") + return std::unexpected(std::format( + "[profile.{}] dependency_debug = \"{}\": expected true, false or \"lines\"", pname, + pr.dependencyDebug)); + const std::string graph = pr.debug ? (pr.debugLines ? "lines" : "true") : "false"; + const std::string wanted = pr.dependencyDebugDeclared ? pr.dependencyDebug + : (pname == "dev" || pname == "debug") && pr.debug ? std::string("lines") + : graph; + state.m->buildConfig.dependencyDebug = wanted == graph ? std::string{} : wanted; + } state.m->buildConfig.lto = pr.lto; state.m->buildConfig.strip = pr.strip; // `ndebug` and `trim_paths` follow the industry's convention for a diff --git a/src/build/prepare_inputs.cppm b/src/build/prepare_inputs.cppm index 35279ebac..99b432e53 100644 --- a/src/build/prepare_inputs.cppm +++ b/src/build/prepare_inputs.cppm @@ -607,6 +607,7 @@ std::string canonical_configuration_flags(const mcpp::manifest::Manifest& root, if (root.buildConfig.ndebug) s += " ndebug=1"; if (root.buildConfig.trimPaths) s += " trim_paths=1"; if (root.buildConfig.splitDebuginfo) s += " split_debuginfo=1"; + if (!root.buildConfig.dependencyDebug.empty()) s += " dependency_debug=" + root.buildConfig.dependencyDebug; if (!root.buildConfig.dependencyLinkage.empty()) { s += " deplinkage="; s += root.buildConfig.dependencyLinkage; diff --git a/src/project.cppm b/src/project.cppm index 0518388f4..c504f4cb2 100644 --- a/src/project.cppm +++ b/src/project.cppm @@ -379,6 +379,10 @@ void merge_layer_root_position(mcpp::manifest::Manifest& member, to.dependencyLinkage = from.dependencyLinkage; to.dependencyLinkageDeclared = true; } + if (!to.dependencyDebugDeclared && from.dependencyDebugDeclared) { + to.dependencyDebug = from.dependencyDebug; + to.dependencyDebugDeclared = true; + } to.cflags.insert(to.cflags.begin(), from.cflags.begin(), from.cflags.end()); to.cxxflags.insert(to.cxxflags.begin(), from.cxxflags.begin(), from.cxxflags.end()); to.ldflags.insert(to.ldflags.begin(), from.ldflags.begin(), from.ldflags.end()); @@ -1149,6 +1153,7 @@ export std::string root_position_key( field("profile." + name, std::format("{}/{}/{}/{}/{}", pr.optLevel, pr.debug ? (pr.debugLines ? "lines" : "1") : "0", pr.lto ? 1 : 0, pr.strip ? 1 : 0, pr.dependencyLinkageDeclared ? pr.dependencyLinkage : std::string("-"))); + if (pr.dependencyDebugDeclared) field("profile." + name + ".dependency_debug", pr.dependencyDebug); list("profile." + name + ".cflags", pr.cflags); list("profile." + name + ".cxxflags", pr.cxxflags); list("profile." + name + ".ldflags", pr.ldflags); diff --git a/tests/e2e/build/flags.cpp b/tests/e2e/build/flags.cpp index cdd12a0c4..6b96e2357 100644 --- a/tests/e2e/build/flags.cpp +++ b/tests/e2e/build/flags.cpp @@ -780,6 +780,39 @@ int helper(void) { return FLAVOUR * 10; } expect(fatal(!stanza.empty())) << "no shared entry object in build.ninja"; expect(s::lacks(stanza, "FLAVOUR")) << "the shared entry object carries a program's define\n" << stanza; }; + "927 the dependencies from the store compile with the line tables under dev"_test[s::gcc] = [] { + // Design 2026-11-20 §5.15, SPEC-004 §9 item 18. Under `dev`, a package from the index + // (mcpplibs.mtest, the scaffold's dev-dependency) compiles with `-g1` after the graph's + // `-g`, and the project's own units keep `-g` alone (A); `dependency_debug = true` gives the + // dependencies the graph's level, and they are another instance (B). + auto p = s::project::scaffold("depdebug"); + auto test_graph = [&] { + for (const auto& f : s::find_files(p.at("target"), R"(test\.ninja)")) + if (f.generic_string().find("/.build-mcpp/") == std::string::npos) return s::read_file(f); + return std::string{}; + }; + auto mtest_flags = [](const std::string& graph) { + std::string lines; + std::istringstream in(graph); + bool inMtest = false; + for (std::string line; std::getline(in, line);) { + if (line.starts_with("build ")) inMtest = line.contains("obj/mcpplibs_mtest/"); + if (inMtest && line.contains("unit_cxxflags")) lines += line + "\n"; + } + return lines; + }; + auto a = p.mcpp({"test", "--no-run"}); + expect(fatal(e2e::succeeded(a))) << s::text(a); + const auto lines = mtest_flags(test_graph()); + expect(fatal(!lines.empty())) << "A: no unit of mcpplibs.mtest in test.ninja"; + expect(e2e::contains(lines, " -g1")) << "A: the dependency's units have no line tables flag\n" << lines; + p.write("mcpp.toml", p.read("mcpp.toml") + "\n[profile.dev]\ndependency_debug = true\n"); + auto b = p.mcpp({"test", "--no-run"}); + expect(fatal(e2e::succeeded(b))) << s::text(b); + const auto full = mtest_flags(test_graph()); + expect(fatal(!full.empty())); + expect(s::lacks(full, " -g1")) << "B: dependency_debug = true kept the line tables flag\n" << full; + }; }}; } // namespace From 44f82756fba98b2f2c439e6df97ef8fcf64340fc Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 03:10:49 +0900 Subject: [PATCH 11/38] feat: a package whose sources are named runs its build program in the home, keyed by its contract; key paths are named by role (#986, #927) --- src/bmi_cache/maintenance.cppm | 29 +++++++++++++++++++++ src/build/build_program.cppm | 21 +++++++++++++++ src/build/cache_key.cppm | 35 ++++++++++++++++++++----- src/build/prepare/dependency_cache.cpp | 4 +-- src/build/prepare/features.cpp | 26 +++++++++++++++++++ src/build/prepare/state.cppm | 3 +++ src/build/prepare/target_side.cpp | 9 ++++--- tests/e2e/build-program/deps.cpp | 9 ++++--- tests/e2e/deps/git.cpp | 36 ++++++++++++++++++++++++++ 9 files changed, 156 insertions(+), 16 deletions(-) diff --git a/src/bmi_cache/maintenance.cppm b/src/bmi_cache/maintenance.cppm index 957c149e0..ca5b0a276 100644 --- a/src/bmi_cache/maintenance.cppm +++ b/src/bmi_cache/maintenance.cppm @@ -353,10 +353,39 @@ std::vector walk_variant_entries() { return out; } +// THE OUTPUTS OF BUILD PROGRAMS IN THE HOME (design 2026-11-20 §5.2): one per +// package, version and contract under `build-cache/v1/out`, each last used when +// a build last ran or replayed the program (its `.mcpp-used` is touched then). +// A directory a build still names is made again by the program's next run. +std::vector walk_program_out_entries() { + std::vector out; + std::error_code ec; + const auto root = mcpp::home::cache_root() / "out"; + if (!std::filesystem::exists(root, ec)) return out; + for (auto& pkg : std::filesystem::directory_iterator(root, ec)) { + if (!pkg.is_directory()) continue; + for (auto& v : std::filesystem::directory_iterator(pkg.path(), ec)) { + if (!v.is_directory()) continue; + Entry e; + e.dir = v.path(); + e.kind = "program-out"; + e.key = v.path().filename().string(); + e.label = std::format("{} (build program's output, {})", pkg.path().filename().string(), e.key); + std::error_code fe; + e.accessed = std::filesystem::is_regular_file(e.dir / ".mcpp-used", fe) + ? dir_mtime_seconds(e.dir / ".mcpp-used") : dir_mtime_seconds(e.dir); + measure(e); + out.push_back(std::move(e)); + } + } + return out; +} + std::vector walk_collectable() { auto out = walk_pkg_entries(); for (auto& e : walk_tool_entries()) out.push_back(std::move(e)); for (auto& e : walk_variant_entries()) out.push_back(std::move(e)); + for (auto& e : walk_program_out_entries()) out.push_back(std::move(e)); return out; } diff --git a/src/build/build_program.cppm b/src/build/build_program.cppm index 387916ad3..6651b2e9e 100644 --- a/src/build/build_program.cppm +++ b/src/build/build_program.cppm @@ -552,6 +552,18 @@ install_hook_env(const BuildProgramEnv& env); // `compile()`. Written by the build programs' graph (mcpp#797). std::string synthesised_rule_program(const std::vector& modules); +// THE OUTPUTS OF A PACKAGE WHOSE SOURCES ARE NAMED, IN THE HOME (design +// 2026-11-20 §5.2). A program's OUT_DIR reaches its package's cache key through +// the include directories it states; under each project's `target/` it gave +// every project its own key for openkal-musl and openkal-windows, and the +// runtime above them was compiled again in each (478 s of a 7 s step, §3.1). +// `/@//` holds `out/` (OUT_DIR) +// and `run/` (the program's run record), the digest being the contract's with +// OUT_DIR aside: one contract, one directory, so a second project replays the +// directives the first recorded and nothing writes the files another reads. +std::filesystem::path program_out_root(); +std::string program_contract_digest(const std::filesystem::path& root, const BuildProgramEnv& env); + // Runs the build program of the package at `root` -- `env.programBinary`, as // the build programs' graph compiled it -- or replays its cached directives // when nothing it depends on changed, and applies them to `m`. @@ -1561,6 +1573,15 @@ std::string synthesised_rule_program(const std::vector& modules) { return synthesised_rule_program_impl(modules); } +fs::path program_out_root() { return mcpp::home::cache_root() / "out"; } + +std::string program_contract_digest(const fs::path& root, const BuildProgramEnv& env) { + auto e = contract_env(root, "", env, /*warn=*/false); + e.emplace_back("(program)", env.programIdentity); + e.emplace_back("(graph document)", env.graphDigest); + return contract_hash(e).substr(0, 16); +} + std::expected run_build_program( mcpp::manifest::Manifest& m, const fs::path& root, diff --git a/src/build/cache_key.cppm b/src/build/cache_key.cppm index 48505366d..c995d4355 100644 --- a/src/build/cache_key.cppm +++ b/src/build/cache_key.cppm @@ -307,6 +307,16 @@ BuildAxes build_axes(const mcpp::toolchain::Toolchain& tc, void fill_package_config(PackageAxes& out, const mcpp::modgraph::PackageRoot& pkg, const std::filesystem::path& storeRoot); +// THE PATHS OF A KEY BY THEIR ROLE (design 2026-11-20 §5.2): a directory under +// any of `storeRoots` is `/...`, under `outRoot` (the home's build-program +// outputs, `/build-cache/v1/out`) `/...`, under the package's root +// `/...`. Only the first store root was replaced before, so a dependency's +// include directory from a second store (nlohmann.json's for mcpplibs.mtest) +// kept the user's home in the key. +void fill_package_config(PackageAxes& out, + const mcpp::modgraph::PackageRoot& pkg, + const std::vector& storeRoots, + const std::filesystem::path& outRoot); } // namespace mcpp::build::cache_key @@ -610,6 +620,16 @@ BuildAxes build_axes(const mcpp::toolchain::Toolchain& tc, void fill_package_config(PackageAxes& out, const mcpp::modgraph::PackageRoot& pkg, const std::filesystem::path& storeRoot) +{ + std::vector roots; + if (!storeRoot.empty()) roots.push_back(storeRoot); + fill_package_config(out, pkg, roots, {}); +} + +void fill_package_config(PackageAxes& out, + const mcpp::modgraph::PackageRoot& pkg, + const std::vector& storeRoots, + const std::filesystem::path& outRoot) { const auto& bc = pkg.manifest.buildConfig; @@ -679,14 +699,17 @@ void fill_package_config(PackageAxes& out, // Include dirs are absolute payload paths under the xpkgs store; make them // store-relative so the key does not encode this machine's MCPP_HOME. - auto storeStr = storeRoot.generic_string(); auto relativize = [&](const std::filesystem::path& p) { auto s = p.generic_string(); - if (!storeStr.empty() && s.starts_with(storeStr)) - return "" + s.substr(storeStr.size()); - auto pkgStr = pkg.root.generic_string(); - if (!pkgStr.empty() && s.starts_with(pkgStr)) - return "" + s.substr(pkgStr.size()); + auto under = [&](const std::filesystem::path& root) { + const auto r = root.generic_string(); + return !r.empty() && s.starts_with(r) && (s.size() == r.size() || s[r.size()] == '/') + ? std::optional(s.substr(r.size())) : std::nullopt; + }; + if (auto rest = under(outRoot)) return "" + *rest; + for (auto const& store : storeRoots) + if (auto rest = under(store)) return "" + *rest; + if (auto rest = under(pkg.root)) return "" + *rest; return s; }; auto add_dirs = [&](const std::vector& dirs, diff --git a/src/build/prepare/dependency_cache.cpp b/src/build/prepare/dependency_cache.cpp index 115494845..02f9aebbc 100644 --- a/src/build/prepare/dependency_cache.cpp +++ b/src/build/prepare/dependency_cache.cpp @@ -249,9 +249,7 @@ static std::expected package_keys(PrepareState& state, // dirs are relativized against it so a key survives a different // MCPP_HOME; a project-local payload falls back to the `` // prefix inside fill_package_config and is equally stable. - ck::fill_package_config(pa, state.packages[idx], - state.storeRoots.empty() ? std::filesystem::path{} - : state.storeRoots.front()); + ck::fill_package_config(pa, state.packages[idx], state.storeRoots, mcpp::build::program_out_root()); pa.sources = pkgSources[idx]; const auto qname = mcpp::build::qualified_package_name(state.packages[idx].manifest); // A GCC BMI records its imports' paths as the module map spells diff --git a/src/build/prepare/features.cpp b/src/build/prepare/features.cpp index f470afd0b..d34367bea 100644 --- a/src/build/prepare/features.cpp +++ b/src/build/prepare/features.cpp @@ -2045,6 +2045,32 @@ static std::expected step6_dependency_build_programs(PrepareS } else { bpEnv.programError = program.error(); } + // A PACKAGE WHOSE SOURCES ARE NAMED RUNS ITS PROGRAM IN THE HOME + // (design 2026-11-20 §5.2): its OUT_DIR and run record are keyed by + // its contract, so every project that builds it in one + // configuration states the same include directories and shares + // its cache entry. Held while the program runs: a second project + // waits, then replays the directives the first recorded. + std::optional outLock; + if (program && sources_named(state, i) && !state.overrides.describes()) { + auto label = std::format("{}@{}", pkg.manifest.package.name, pkg.manifest.package.version); + for (auto& c : label) + if (c == '/' || c == '\\' || c == ':') c = '_'; + const auto dir = mcpp::build::program_out_root() / label + / mcpp::build::program_contract_digest(pkg.root, bpEnv); + bpEnv.outDir = dir / "out"; + bpEnv.genBase = bpEnv.outDir; + bpEnv.artifactsDir = dir / "run"; + outLock = mcpp::home::FileLock::acquire( + dir, "run.lock", std::format("{}'s build program", label), std::chrono::seconds{2}, + [&](std::string_view holder) { + mcpp::ui::info("note", std::format("waiting for {} ({})", holder.empty() ? label : holder, + mcpp::home::spelling(dir))); + }); + // Its last use, which `mcpp cache gc` collects by. + std::ofstream(dir / ".mcpp-used", std::ios::binary | std::ios::trunc); + state.programOutDirs[i] = bpEnv.outDir; + } auto& bcDep = pkg.manifest.buildConfig; const auto cxxN = bcDep.cxxflags.size(); const auto cN = bcDep.cflags.size(); diff --git a/src/build/prepare/state.cppm b/src/build/prepare/state.cppm index da8ec4368..dabe2b6e0 100644 --- a/src/build/prepare/state.cppm +++ b/src/build/prepare/state.cppm @@ -810,6 +810,9 @@ struct PrepareState { std::function&)> programOutDir; + // The OUT_DIR of each package whose program ran in the home (design + // 2026-11-20 §5.2), by its index: what the graph document states for it. + std::map programOutDirs; // What this configuration uses outside its own directory, written into it // as `uses.list` so that `mcpp clean --stale` keeps these and collects the // rest (mcpp#874): the `.build-mcpp/out/` of each program it ran and diff --git a/src/build/prepare/target_side.cpp b/src/build/prepare/target_side.cpp index f64362682..c0e99b407 100644 --- a/src/build/prepare/target_side.cpp +++ b/src/build/prepare/target_side.cpp @@ -1828,9 +1828,12 @@ static void step9_define_graph_package_entry_closure(PrepareState& state) { // A member's program keeps its cache in the project (step 9). const bool member = state.m->package.virtualRoot && i != 0 && state.isWorkspaceMemberPackage(i); - entry["outDir"] = state.programOutDir(member ? *state.root : state.workRoot, pm, - i < state.activeFeaturesByPackage.size() - ? state.activeFeaturesByPackage[i] : none) + const auto home = state.programOutDirs.find(i); + entry["outDir"] = (home != state.programOutDirs.end() + ? home->second + : state.programOutDir(member ? *state.root : state.workRoot, pm, + i < state.activeFeaturesByPackage.size() + ? state.activeFeaturesByPackage[i] : none)) .generic_string(); return entry; } diff --git a/tests/e2e/build-program/deps.cpp b/tests/e2e/build-program/deps.cpp index b7758bb9e..868196fe3 100644 --- a/tests/e2e/build-program/deps.cpp +++ b/tests/e2e/build-program/deps.cpp @@ -75,11 +75,12 @@ int main() { expect(fatal(e2e::succeeded(app.mcpp({"build"})))) << "build failed"; expect(eq(s::last_line(app.mcpp({"run"}).out), std::string("dep = 77"))); expect(eq(s::listing(dep.dir()), before)) << "dependency root was written to"; - // The program lives in .build-mcpp/deps, its output in this configuration's directory. + // A path dependency outside the project is named by its content (#974): its program's + // output is in the home, by package and contract (design 2026-11-20 §5.2). bool found = false; - for (const auto& f : s::find_files(app.at("target/.build-mcpp/out"), R"(gen_dep\.cpp)")) - found = found || f.generic_string().ends_with("deplib/gen_dep.cpp"); - expect(found) << "dep OUT_DIR generated file not in project tree"; + for (const auto& f : s::find_files(s::shared_home() / "build-cache/v1/out", R"(gen_dep\.cpp)")) + found = found || f.generic_string().contains("/deplib@0.1.0/"); + expect(found) << "dep OUT_DIR generated file not in the home's build-program outputs"; }; "125 build mcpp dep dir"_test[s::elf][s::gcc] = [] { diff --git a/tests/e2e/deps/git.cpp b/tests/e2e/deps/git.cpp index 5dc253961..dabe16add 100644 --- a/tests/e2e/deps/git.cpp +++ b/tests/e2e/deps/git.cpp @@ -306,6 +306,42 @@ int main() { expect(s::lacks(s::text(b), "\x1b")) << "B: the log carries an ESC byte"; }; + "955 a dependency's build program writes its outputs in the home, and a second project takes them"_test[s::gcc] = [] { + // Design 2026-11-20 §5.2. A package whose sources are named (a git commit) runs its build + // program with OUT_DIR in the home, keyed by the program's contract: a second project + // replays the program's directives (A), states the same include directory, and is served + // the package from the cache instead of compiling it (B). + const auto tmp = s::case_dir(); + s::project lib{tmp / "genlib", {{"mcpp.toml", "[package]\nname = \"genlib\"\nversion = \"0.1.0\"\n[targets.genlib]\nkind = \"lib\"\n"}, + {"build.mcpp", R"cpp(import std; +int main() { + const std::filesystem::path out = std::getenv("MCPP_OUT_DIR"); + std::filesystem::create_directories(out / "gen"); + std::ofstream(out / "gen" / "answer.h") << "#define GEN_ANSWER 42\n"; + std::println("mcpp:include-dir={}", (out / "gen").string()); + return 0; +} +)cpp"}, + {"src/genlib.cppm", "module;\n#include \"answer.h\"\nexport module genlib;\nexport int answer() { return GEN_ANSWER; }\n"}}}; + git(lib.dir(), {"init", "--quiet"}); + git(lib.dir(), {"add", "-A"}); + git(lib.dir(), {"commit", "--quiet", "-m", "init"}); + const auto rev = git(lib.dir(), {"rev-parse", "HEAD"}); + auto app = [&](std::string_view name) { + return s::project{tmp / name, {{"mcpp.toml", std::format("[package]\nname = \"{}\"\nversion = \"0.1.0\"\n[dependencies]\ngenlib = {{ git = \"file://{}\", rev = \"{}\" }}\n", name, lib.dir().generic_string(), rev)}, + {"src/main.cpp", "import genlib;\nint main() { return answer() == 42 ? 0 : 1; }\n"}}}; + }; + auto one = app("one"); + auto a = one.mcpp({"build"}); + expect(fatal(e2e::succeeded(a))) << s::text(a); + auto two = app("two"); + auto b = two.mcpp({"build", "--verbose"}); + expect(fatal(e2e::succeeded(b))) << s::text(b); + expect(s::has_line(s::text(b), R"(build\.mcpp .*genlib +cached)")) << "A: the second project ran the program again\n" << s::text(b); + expect(s::has_line(s::text(b), R"(Cached genlib v0\.1\.0 )")) << "B: the second project compiled the package\n" << s::text(b); + expect(e2e::succeeded(two.mcpp({"run"}))) << "B: the program the cached package was linked into"; + }; + "943 a transitive git branch is locked"_test[s::gcc] = [] { // A git branch reached through another package is locked as the root's own are: a later // commit on it is not taken until `mcpp update`. From aa9af7acbb73558b0227174f4f91b473d0509b2c Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 03:10:49 +0900 Subject: [PATCH 12/38] test: a program run through wine returns when wine's services have ended --- tests/e2e/build-program/cross.cpp | 2 +- tests/e2e/openkal/trees.cpp | 2 +- tests/e2e/windows/images.cpp | 7 ++++++- tests/support/src/support.cpp | 11 +++++++++++ tests/support/src/support.cppm | 7 +++++++ 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/e2e/build-program/cross.cpp b/tests/e2e/build-program/cross.cpp index eb05163c7..5bfa7089d 100644 --- a/tests/e2e/build-program/cross.cpp +++ b/tests/e2e/build-program/cross.cpp @@ -38,7 +38,7 @@ int main() { expect(fatal(exe.has_value())) << "no PE artifact produced"; // Under wine, when the machine has it, the loop is whole. if (s::wine) { - auto w = e2e::run({.argv = {"wine", exe->string()}, .env = {{"WINEDEBUG", "-all"}}}); + auto w = e2e::run({.argv = s::through_wine(*s::on_path("wine"), *exe), .env = {{"WINEDEBUG", "-all"}}}); auto out = s::last_line(w.out); if (out.ends_with('\r')) out.pop_back(); expect(eq(out, std::string("bp target = x86_64-windows-gnu"))) << "unexpected wine output"; diff --git a/tests/e2e/openkal/trees.cpp b/tests/e2e/openkal/trees.cpp index b18a13ab9..624b9dbb1 100644 --- a/tests/e2e/openkal/trees.cpp +++ b/tests/e2e/openkal/trees.cpp @@ -627,7 +627,7 @@ E2E trees{"trees", timeout{std::chrono::minutes{40}}, [] { expect(fatal(e2e::contains(s::text(r), "is a polyglot file"))) << "G: emit xpkg does not name the polyglot file: " << s::text(r); // H if (const auto wine = s::on_path("wine"); wine && os == "linux") { - auto w = e2e::run({.argv = {wine->string(), f.string(), "onwine"}, .env = {{"WINEDEBUG", "-all"}, {"XDG_CACHE_HOME", std::nullopt}}, .timeout = std::chrono::seconds{300}}); + auto w = e2e::run({.argv = s::through_wine(*wine, f, {"onwine"}), .env = {{"WINEDEBUG", "-all"}, {"XDG_CACHE_HOME", std::nullopt}}, .timeout = std::chrono::seconds{300}}); expect(fatal(e2e::succeeded(w))) << "H: wine: " << s::text(w); expect(fatal(e2e::contains(s::text(w), "first [onwine]"))) << "H: the Windows program's output: " << s::text(w); } diff --git a/tests/e2e/windows/images.cpp b/tests/e2e/windows/images.cpp index a277f6b08..277bef2fc 100644 --- a/tests/e2e/windows/images.cpp +++ b/tests/e2e/windows/images.cpp @@ -57,7 +57,12 @@ void subsystems(const s::project& p, std::vector build_args, std::o } if (!runner && !s::on_windows()) return; // criterion 3 is not measured on a host that cannot execute a PE image const auto run = [&](const fs::path& exe) { - if (runner) return s::run(*runner, {exe.string()}, p.dir(), {{"WINEDEBUG", "-all"}}); + if (runner) { + auto argv = s::through_wine(*runner, exe); + const fs::path shell = s::on_path("sh").value_or("/bin/sh"); + argv.erase(argv.begin()); + return s::run(shell, argv, p.dir(), {{"WINEDEBUG", "-all"}}); + } return s::run(exe, {}, p.dir()); }; p.remove("order.txt"); diff --git a/tests/support/src/support.cpp b/tests/support/src/support.cpp index a61ad1156..723c35c33 100644 --- a/tests/support/src/support.cpp +++ b/tests/support/src/support.cpp @@ -312,6 +312,17 @@ std::string text(const e2e::outcome& r) { return r.out + r.err; } fs::path shared_home() { return home_dir(); } +std::vector through_wine(const fs::path& wine, const fs::path& program, + const std::vector& args) { + const auto server = wine.parent_path() / "wineserver"; + std::vector argv = {"sh", "-c", + std::format("'{}' \"$@\"; status=$?; '{}' -w; exit $status", wine.string(), + fs::exists(server) ? server.string() : std::string("wineserver")), + "wine", program.string()}; + argv.insert(argv.end(), args.begin(), args.end()); + return argv; +} + namespace { void link_or_copy(const fs::path& from, const fs::path& to) { std::error_code ec; diff --git a/tests/support/src/support.cppm b/tests/support/src/support.cppm index 172752ede..28cec1a6b 100644 --- a/tests/support/src/support.cppm +++ b/tests/support/src/support.cppm @@ -159,6 +159,13 @@ fs::path another_home(home kind = home::fresh); // The shared home: `MCPP_HOME`, else `~/.mcpp`. fs::path shared_home(); +// The command that runs `program` with `args` through `wine` and returns when every process wine +// started has ended (`wineserver -w`), with the program's status. Wine starts its services with +// the program's streams; they outlive the program by seconds, and the case's end check fails a +// run whose output a process still holds (mtest 0.11.0, design 2026-11-20 §5.18). +std::vector through_wine(const fs::path& wine, const fs::path& program, + const std::vector& args = {}); + struct options { home where = home::shared; // Set (a value) or removed (nothing) for the run, after the home. From 31dbf2b22d051df2e7e75803ea1ab9aa275e5aa9 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 03:32:34 +0900 Subject: [PATCH 13/38] feat: a compile whose command and read files hold an earlier compile's contents is placed from the home's unit cache, and no compiler runs (#927) --- docs/specs/test.md | 2 +- mcpp.toml | 2 + modules/build-record/src/build_record.cppm | 9 +- modules/unit-cache/mcpp.toml | 21 ++ modules/unit-cache/src/unit_cache.cppm | 326 +++++++++++++++++++ modules/unit-cache/tests/test_unit_cache.cpp | 79 +++++ src/bmi_cache/maintenance.cppm | 30 ++ src/build/execute.cppm | 1 + src/build/ninja_backend.cppm | 30 ++ src/build/plan.cppm | 5 + src/build/prepare/plan.cpp | 4 + src/cli.cppm | 21 ++ tests/e2e/cache/entries.cpp | 34 ++ 13 files changed, 562 insertions(+), 2 deletions(-) create mode 100644 modules/unit-cache/mcpp.toml create mode 100644 modules/unit-cache/src/unit_cache.cppm create mode 100644 modules/unit-cache/tests/test_unit_cache.cpp diff --git a/docs/specs/test.md b/docs/specs/test.md index fd9df9689..1d4915ca1 100644 --- a/docs/specs/test.md +++ b/docs/specs/test.md @@ -174,7 +174,7 @@ | `runs//run.json` | 运行记录:`schema`(`mtest.run/1`)、`id`、`started`、`finished`、`status`、`system`、`arch`、`machine`、`commit`、`set`、`selection`、`group`、`monitor`、`ms`、`jobs`、`counts`;`processors`、`memory`、`budget`、`jitter`(@NEXT@) | | `runs//programs.ndjson` | 每个程序进程一行:`name`、`argv`、`status`、`exit`、`ms`、`started_ms`、`worker`、`select`、`resources`、`fixtures`,运行器构建它时的 `build_started_ms` 与 `build_ms`,`usage`(带 `source`:`program` 或 `engine`) | | `runs//cases.ndjson` | 每个用例一行:`program`、`id`、`kind`、`status`、`ms`、`started_ms`、`reason`、`failures`、`metrics`、`bounds`、`tags`、`clauses`;`cpu_ms`、`steps`(`step` 事件的字段,`id` 除外)、`exercised`、`limit_ms`(用例所受的时限,O5;未知时省略)(@NEXT@) | -| `runs//builds.ndjson` | 运行器为这次运行所做的构建(@NEXT@):每条运行过的边一行,`build`(构建记录的标识)、`output`、`package`、`program`(该边所服务的测试程序,可为空)、`started_ms`、`ms`、`cpu_ms`、`peak_bytes`;由缓存提供的包一行,`build`、`package`、`cached`(`true`)、`units` | +| `runs//builds.ndjson` | 运行器为这次运行所做的构建(@NEXT@):每条运行过的边一行,`build`(构建记录的标识)、`output`、`package`、`program`(该边所服务的测试程序,可为空)、`started_ms`、`ms`、`cpu_ms`、`peak_bytes`,单元缓存放置的边另有 `served`(`true`,没有编译器运行);由缓存提供的包一行,`build`、`package`、`cached`(`true`)、`units` | | `runs//fixtures.ndjson` | 每个运行夹具一行:`name`、`program`、`started_ms`、`ready_ms`、`ended_ms`、`status` | - **P12.4 机器(@NEXT@)。** `machine` 是机器的类别:处理器的型号,系统不给出型号时为 `/<逻辑处理器数>`;`processors` 为逻辑处理器数,`memory` 为物理内存的字节数。比较不同运行的用时(O10 的估计、界限、趋势)时,读者**应当**按 `machine` 分组;没有同类的历史时取全部。处理器时间(`cpu_ms`)在同类机器之间可比,墙钟只在同一次运行之内可比。 diff --git a/mcpp.toml b/mcpp.toml index 16e1c421b..91548ce89 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -154,6 +154,7 @@ mcpp.manifest = { path = "modules/manifest" } mcpp.toolchain-model = { path = "modules/toolchain-model" } mcpp.buildmcpp = { path = "modules/buildmcpp" } mcpp.build-record = { path = "modules/build-record" } +mcpp.unit-cache = { path = "modules/unit-cache" } [target.linux.build] sources = ["!src/native/windows/**"] @@ -178,6 +179,7 @@ members = [ "modules/toolchain-model", "modules/buildmcpp", "modules/build-record", + "modules/unit-cache", ] # `mcpp build` ignores [dev-dependencies]; only `mcpp test` resolves them. diff --git a/modules/build-record/src/build_record.cppm b/modules/build-record/src/build_record.cppm index 899a09717..9f262acd7 100644 --- a/modules/build-record/src/build_record.cppm +++ b/modules/build-record/src/build_record.cppm @@ -27,6 +27,7 @@ struct Usage { long long endedUnixMs = 0; std::optional cpuMs; std::optional peakBytes; + bool served = false; // the unit cache placed its outputs; no compiler ran }; // One line, `\n` ended, written in one append so that concurrent compiles do not interleave. std::string usage_line(const Usage& u); @@ -42,6 +43,7 @@ struct Edge { double ms = 0; std::optional cpuMs; std::optional peakBytes; + bool served = false; // placed by the unit cache (design 2026-11-20 §5.3) }; // A package the cache served, with the units it placed. struct Cached { @@ -117,6 +119,7 @@ std::string usage_line(const Usage& u) { json j = {{"output", u.output}, {"started", u.startedUnixMs}, {"ended", u.endedUnixMs}}; if (u.cpuMs) j["cpu_ms"] = *u.cpuMs; if (u.peakBytes) j["peak_bytes"] = *u.peakBytes; + if (u.served) j["served"] = true; return j.dump(-1, ' ', false, json::error_handler_t::replace) + "\n"; } @@ -129,6 +132,7 @@ std::vector parse_usage(std::string_view text) { u.endedUnixMs = static_cast(number_of(j, "ended").value_or(0)); u.cpuMs = number_of(j, "cpu_ms"); u.peakBytes = bytes_of(j, "peak_bytes"); + if (auto s = j.find("served"); s != j.end() && s->is_boolean()) u.served = s->get(); out.push_back(std::move(u)); } return out; @@ -146,6 +150,7 @@ void attach(Record& r, const std::vector& usage) { if (auto it = byOutput.find(e.output); it != byOutput.end()) { e.cpuMs = it->second->cpuMs; e.peakBytes = it->second->peakBytes; + e.served = it->second->served; } } @@ -156,6 +161,7 @@ std::string to_ndjson(const Record& r) { {"program", e.program}, {"started_ms", e.startedMs}, {"ms", e.ms}}; if (e.cpuMs) j["cpu_ms"] = *e.cpuMs; if (e.peakBytes) j["peak_bytes"] = *e.peakBytes; + if (e.served) j["served"] = true; out += j.dump(-1, ' ', false, json::error_handler_t::replace) + "\n"; } for (const auto& c : r.cached) @@ -177,7 +183,8 @@ std::vector from_ndjson(std::string_view text) { } at->edges.push_back({text_of(j, "output"), text_of(j, "package"), text_of(j, "program"), number_of(j, "started_ms").value_or(0), number_of(j, "ms").value_or(0), - number_of(j, "cpu_ms"), bytes_of(j, "peak_bytes")}); + number_of(j, "cpu_ms"), bytes_of(j, "peak_bytes"), + j.value("served", false)}); } return out; } diff --git a/modules/unit-cache/mcpp.toml b/modules/unit-cache/mcpp.toml new file mode 100644 index 000000000..cb15d905c --- /dev/null +++ b/modules/unit-cache/mcpp.toml @@ -0,0 +1,21 @@ +# mcpp.unit-cache -- a compile's outputs addressed by what it read (design 2026-11-20 §5.3). +# +# The compiler executor asks it before it starts a compiler: a key of the command, the compiler +# and the source's content names the manifests earlier compiles of that key recorded, each the +# files the compile read and their contents; a manifest whose files all still hold those contents +# names the outputs, which are placed and the compiler is not started. After a compile it records +# the manifest from the compiler's depfile and keeps the outputs. It knows nothing of ninja, the +# plan or the executor's slots: the executor hands it the command and the paths. + +[package] +name = "unit-cache" +namespace = "mcpp" +version = "0.1.0" +description = "A compile's outputs addressed by the command and the contents of the files it read" +license = "Apache-2.0" + +[dependencies] +mcpp.toolchain-model = { path = "../toolchain-model" } + +[dev-dependencies] +mcpplibs.mtest = "0.11.0" diff --git a/modules/unit-cache/src/unit_cache.cppm b/modules/unit-cache/src/unit_cache.cppm new file mode 100644 index 000000000..98b5c21fe --- /dev/null +++ b/modules/unit-cache/src/unit_cache.cppm @@ -0,0 +1,326 @@ +// mcpp.build.unit_cache -- a compile's outputs addressed by what it read (design 2026-11-20 §5.3). +// +// A FRESH CHECKOUT COMPILED EVERY UNIT AGAIN. ninja decides by modification time, and a checkout +// writes every file new: CI rebuilt mcpp's 830 edges in each job (§3.5), and a toolchain upgrade +// that moved the fingerprint directory compiled a project again with nothing changed. The key here +// is the content of what a compile reads, so a unit whose inputs hold the bytes they held is +// placed, not compiled. +// +// TWO LEVELS. What a compile reads is known only after it ran (its depfile), so the first key +// takes what is known before: the command, the directory it runs in, the source's content, the +// response files' and the module mapper's. It names the manifests earlier compiles of that key +// recorded, each a list of the files the compile read with their contents' digests; the first +// whose files all hold those contents names the outputs. +// +// WHAT IS NOT LOOKED AT, AND WHY THAT IS SAFE. A file under an immutable root (a store of +// versioned payloads: the toolchain, the packages of the index) does not change in place; its +// path is its identity, and a different version is another path in the command. The compiler must +// be under one: its system headers do not appear in a `-MMD` depfile, and only a toolchain whose +// files never change makes their absence harmless. A `-isystem` directory outside them is the same +// absence, and refuses the unit. +export module mcpp.build.unit_cache; + +import std; +import mcpp.toolchain.fingerprint; + +export namespace mcpp::build::unit_cache { + +// The cache's directory, and the roots whose files never change in place, joined by the system's +// path list separator: what the build states to the compiler executor. +inline constexpr std::string_view kDirEnv = "MCPP_UNIT_CACHE"; +inline constexpr std::string_view kRootsEnv = "MCPP_IMMUTABLE_ROOTS"; +// A build directory's memory of file digests, by path, size and modification time: a header is +// read once per build, not once per unit that includes it. +inline constexpr std::string_view kDigestsFile = ".mcpp-digests"; + +// One compile, as the executor runs it: the compiler's command, the directory it runs in, and the +// outputs the executor publishes. +struct Unit { + std::vector argv; + std::filesystem::path cwd; + std::filesystem::path object; + std::filesystem::path bmi; // empty: the unit provides no module +}; + +// What the command names, read from its words: the source (`-c `), the depfile (`-MF`), the +// module mapper (`-fmodule-mapper=`), the response files (`@file`), the directories searched as +// the system's (`-isystem`, `-idirafter`), and whether the debug information is split. +struct Shape { + std::filesystem::path source, depfile, mapper; + std::vector responses, systemDirs; + bool splitDebug = false; +}; +std::optional shape_of(const std::vector& argv); + +// The prerequisites of every rule of a Makefile-syntax depfile, each once, unescaped. +std::vector depfile_prerequisites(std::string_view text); + +class Cache { +public: + Cache(std::filesystem::path dir, std::vector immutableRoots, + std::filesystem::path buildDir); + // Why `u` is not cached, or empty when it is. + std::string refusal(const Unit& u) const; + // Places the outputs of an earlier compile that read what this one would: true when it did. + bool serve(const Unit& u); + // Records what a compile that succeeded read, and keeps its outputs. + void keep(const Unit& u); + +private: + std::string first_key(const Unit& u, const Shape& s); + std::string digest(const std::filesystem::path& file); + bool immutable(const std::filesystem::path& file) const; + std::filesystem::path dir_; + std::vector roots_; + std::filesystem::path buildDir_; + std::map> memo_; // "path\tsize\tmtime" -> digest + bool memoLoaded_ = false; +}; + +} // namespace mcpp::build::unit_cache + +namespace mcpp::build::unit_cache { + +namespace fs = std::filesystem; + +namespace { + +std::string read_all(const fs::path& p) { + std::ifstream in(p, std::ios::binary); + return {std::istreambuf_iterator(in), {}}; +} + +// Written beside `to` and renamed over it, so a reader never sees part of it. +bool place(const fs::path& from, const fs::path& to) { + std::error_code ec; + if (to.has_parent_path()) fs::create_directories(to.parent_path(), ec); + auto tmp = to; + tmp += std::format(".mcpp-unit-{:x}", std::random_device{}()); + fs::copy_file(from, tmp, fs::copy_options::overwrite_existing, ec); + if (ec) return false; + fs::rename(tmp, to, ec); + if (ec) fs::remove(tmp, ec); + return !ec; +} + +std::string normal(const fs::path& base, const fs::path& p) { + return (p.is_absolute() ? p : base / p).lexically_normal().generic_string(); +} + +} // namespace + +std::optional shape_of(const std::vector& argv) { + Shape s; + for (std::size_t i = 1; i < argv.size(); ++i) { + const std::string_view w = argv[i]; + auto next = [&]() -> std::string_view { return i + 1 < argv.size() ? std::string_view(argv[++i]) : std::string_view{}; }; + if (w == "-c") s.source = next(); + else if (w == "-MF") s.depfile = next(); + else if (w.starts_with("-fmodule-mapper=")) s.mapper = w.substr(16); + else if (w.starts_with("@") && w.size() > 1) s.responses.emplace_back(w.substr(1)); + else if (w == "-isystem" || w == "-idirafter") s.systemDirs.emplace_back(next()); + else if ((w.starts_with("-isystem") || w.starts_with("-idirafter")) && w.size() > 9) + s.systemDirs.emplace_back(w.starts_with("-isystem") ? w.substr(8) : w.substr(10)); + else if (w == "-gsplit-dwarf") s.splitDebug = true; + } + if (s.source.empty() || s.depfile.empty()) return std::nullopt; + // A module mapper that is a program, or a server, answers what a file cannot record. + if (!s.mapper.empty() && (s.mapper.native().find('|') != fs::path::string_type::npos + || s.mapper.native().find('=') != fs::path::string_type::npos + || s.mapper.native().find(':') == 0)) + return std::nullopt; + return s; +} + +std::vector depfile_prerequisites(std::string_view text) { + std::vector out; + std::set seen; + std::string word; + bool afterColon = false; + auto end_word = [&] { + if (afterColon && !word.empty() && seen.insert(word).second) out.push_back(word); + word.clear(); + }; + for (std::size_t i = 0; i < text.size(); ++i) { + const char c = text[i]; + if (c == '\\' && i + 1 < text.size()) { + const char n = text[i + 1]; + if (n == '\n') { end_word(); ++i; continue; } + if (n == '\r' && i + 2 < text.size() && text[i + 2] == '\n') { end_word(); i += 2; continue; } + if (n == ' ' || n == '#' || n == '\\') { word += n; ++i; continue; } + } + if (c == '$' && i + 1 < text.size() && text[i + 1] == '$') { word += '$'; ++i; continue; } + if (c == '\n') { end_word(); afterColon = false; continue; } + if (c == ' ' || c == '\t' || c == '\r') { end_word(); continue; } + // The colon that ends a rule's targets: followed by space or the line's end, so a + // drive letter (`C:/x`) stays in its word. + if (c == ':' && !afterColon && (i + 1 >= text.size() || text[i + 1] == ' ' || text[i + 1] == '\n' + || text[i + 1] == '\r' || text[i + 1] == '\t')) { + word.clear(); + afterColon = true; + continue; + } + word += c; + } + end_word(); + return out; +} + +Cache::Cache(fs::path dir, std::vector immutableRoots, fs::path buildDir) + : dir_(std::move(dir)), roots_(std::move(immutableRoots)), buildDir_(std::move(buildDir)) { + for (auto& r : roots_) r = r.lexically_normal(); +} + +bool Cache::immutable(const fs::path& file) const { + const auto f = file.lexically_normal().generic_string(); + for (const auto& r : roots_) { + const auto root = r.generic_string(); + if (!root.empty() && f.starts_with(root) && (f.size() == root.size() || f[root.size()] == '/' + || root.ends_with('/'))) + return true; + } + return false; +} + +std::string Cache::digest(const fs::path& file) { + if (!memoLoaded_) { + memoLoaded_ = true; + std::istringstream in(read_all(buildDir_ / kDigestsFile)); + for (std::string line; std::getline(in, line);) + if (const auto tab = line.rfind('\t'); tab != std::string::npos) + memo_.emplace(line.substr(0, tab), line.substr(tab + 1)); + } + std::error_code ec; + const auto size = fs::file_size(file, ec); + if (ec) return {}; + const auto mtime = fs::last_write_time(file, ec).time_since_epoch().count(); + if (ec) return {}; + const auto key = std::format("{}\t{}\t{}", file.generic_string(), size, mtime); + if (auto it = memo_.find(key); it != memo_.end()) return it->second; + auto d = mcpp::toolchain::hash_file(file); + memo_.emplace(key, d); + // A FILE WRITTEN IN THE LAST TWO SECONDS IS NOT REMEMBERED for other compiles: a second write + // of the same size within one tick of the file system's clock keeps the time, and the memory + // would then hold the first content's digest (git's "racily clean" entries). + if (fs::file_time_type::clock::now() - fs::last_write_time(file, ec) < std::chrono::seconds{2} || ec) return d; + // One line, one append: the units of a build write it concurrently. + std::ofstream(buildDir_ / kDigestsFile, std::ios::binary | std::ios::app) << key + "\t" + d + "\n"; + return d; +} + +std::string Cache::refusal(const Unit& u) const { + if (u.argv.empty()) return "no command"; + const auto shape = shape_of(u.argv); + if (!shape) return "the command names no source and depfile"; + if (!immutable(fs::path(u.argv.front()))) + return "the compiler is not in a store whose files never change"; + for (const auto& d : shape->systemDirs) + if (!immutable(d.is_absolute() ? d : u.cwd / d)) + return std::format("a system include directory, {}, is not in a store", d.generic_string()); + return {}; +} + +std::string Cache::first_key(const Unit& u, const Shape& s) { + std::string text = "unit-cache-v1\n"; + text += "cwd=" + u.cwd.lexically_normal().generic_string() + "\n"; + for (const auto& w : u.argv) { + text += w; + text += '\0'; + } + text += "\nsource=" + digest(s.source.is_absolute() ? s.source : u.cwd / s.source) + "\n"; + for (const auto& r : s.responses) text += "response=" + digest(r.is_absolute() ? r : u.cwd / r) + "\n"; + if (!s.mapper.empty()) text += "mapper=" + digest(s.mapper.is_absolute() ? s.mapper : u.cwd / s.mapper) + "\n"; + return mcpp::toolchain::hash_string(text) + mcpp::toolchain::hash_string(text + "\x01"); +} + +bool Cache::serve(const Unit& u) { + if (!refusal(u).empty()) return false; + const auto shape = shape_of(u.argv); + const auto key = first_key(u, *shape); + const auto entry = dir_ / key.substr(0, 2) / key; + std::error_code ec; + if (!fs::is_directory(entry, ec)) return false; + for (const auto& m : fs::directory_iterator(entry, ec)) { + if (!m.is_directory() || m.path().filename().string().starts_with(".")) continue; + std::istringstream in(read_all(m.path() / "manifest.tsv")); + bool holds = true; + bool any = false; + for (std::string line; holds && std::getline(in, line);) { + const auto tab = line.find('\t'); + if (tab == std::string::npos) { holds = false; break; } + any = true; + const fs::path file = line.substr(0, tab); + const auto want = line.substr(tab + 1); + if (want == "=") continue; // immutable: its path is its identity + holds = digest(file) == want; + } + if (!holds || !any) continue; + const auto out = m.path(); + // Each output, unless it already holds the bytes (so ninja's restat sees no change). + auto put = [&](const fs::path& from, const fs::path& to) { + if (to.empty() || !fs::exists(from, ec)) return true; + const auto target = to.is_absolute() ? to : u.cwd / to; + if (fs::exists(target, ec) && mcpp::toolchain::hash_file(target) == mcpp::toolchain::hash_file(from)) + return true; + return place(from, target); + }; + if (!put(out / "object", u.object) || !put(out / "bmi", u.bmi) || !put(out / "depfile", shape->depfile)) + return false; + if (shape->splitDebug) { + auto dwo = u.object; + if (!put(out / "dwo", dwo.replace_extension(".dwo"))) return false; + } + // Its last use, which `mcpp cache gc` collects by. + fs::last_write_time(out / "manifest.tsv", fs::file_time_type::clock::now(), ec); + return true; + } + return false; +} + +void Cache::keep(const Unit& u) { + if (!refusal(u).empty()) return; + const auto shape = shape_of(u.argv); + const auto depfile = shape->depfile.is_absolute() ? shape->depfile : u.cwd / shape->depfile; + const auto text = read_all(depfile); + if (text.empty()) return; + const auto object = u.object.is_absolute() ? u.object : u.cwd / u.object; + const auto bmi = u.bmi.empty() ? fs::path{} : (u.bmi.is_absolute() ? u.bmi : u.cwd / u.bmi); + std::string manifest; + for (const auto& p : depfile_prerequisites(text)) { + // GCC names a module a unit imports or provides (`x.c++-module`) beside the files. + if (p.ends_with(".c++-module")) continue; + const auto file = fs::path(normal(u.cwd, p)); + if (file == object.lexically_normal() || (!bmi.empty() && file == bmi.lexically_normal())) continue; + if (immutable(file)) { + manifest += file.generic_string() + "\t=\n"; + continue; + } + const auto d = digest(file); + if (d.empty()) return; // a file it read is gone: nothing to record + manifest += file.generic_string() + "\t" + d + "\n"; + } + if (manifest.empty()) return; + const auto key = first_key(u, *shape); + const auto entry = dir_ / key.substr(0, 2) / key; + const auto name = mcpp::toolchain::hash_string(manifest); + std::error_code ec; + if (fs::is_directory(entry / name, ec)) return; + const auto staging = entry / std::format(".staging-{:x}", std::random_device{}()); + fs::create_directories(staging, ec); + if (ec) return; + bool ok = static_cast(std::ofstream(staging / "manifest.tsv", std::ios::binary) << manifest); + ok = ok && fs::exists(object, ec); + if (ok) fs::copy_file(object, staging / "object", ec), ok = !ec; + if (ok && !bmi.empty() && fs::exists(bmi, ec)) fs::copy_file(bmi, staging / "bmi", ec), ok = !ec; + if (ok) fs::copy_file(depfile, staging / "depfile", ec), ok = !ec; + if (ok && shape->splitDebug) { + auto dwo = object; + dwo.replace_extension(".dwo"); + if (fs::exists(dwo, ec)) fs::copy_file(dwo, staging / "dwo", ec), ok = !ec; + } + // Published whole by one rename; a compile of the same key that got there first wins. + if (ok) fs::rename(staging, entry / name, ec); + if (!ok || ec) fs::remove_all(staging, ec); +} + +} // namespace mcpp::build::unit_cache diff --git a/modules/unit-cache/tests/test_unit_cache.cpp b/modules/unit-cache/tests/test_unit_cache.cpp new file mode 100644 index 000000000..7008cf44a --- /dev/null +++ b/modules/unit-cache/tests/test_unit_cache.cpp @@ -0,0 +1,79 @@ +import std; +import mcpplibs.mtest; +import mcpp.build.unit_cache; + +using namespace mcpplibs::mtest; + +// SUBSYSTEM-LEVEL: a command's shape, a depfile's prerequisites, and a compile kept and served +// again by the contents of what it read (design 2026-11-20 §5.3). That the executor serves a +// fresh checkout's units from it is stated by the e2e suite (tests/e2e/cache/). + +namespace uc = mcpp::build::unit_cache; +namespace fs = std::filesystem; + +namespace { +struct Tmp { + fs::path dir = fs::temp_directory_path() / std::format("mcpp-unit-cache-{:08x}", std::random_device{}()); + Tmp() { fs::create_directories(dir); } + ~Tmp() { std::error_code ec; fs::remove_all(dir, ec); } +}; +void write(const fs::path& p, std::string_view text) { + fs::create_directories(p.parent_path()); + std::ofstream(p, std::ios::binary) << text; +} +std::string read(const fs::path& p) { + std::ifstream in(p, std::ios::binary); + return {std::istreambuf_iterator(in), {}}; +} +} // namespace + +UTest a_command_states_its_source_and_depfile{"UnitCache/ACommandStatesItsSourceAndDepfile", [] { + auto s = uc::shape_of({"/store/g++", "-isystem", "/store/inc", "-fmodule-mapper=modmap/p.map", + "@obj/a.rsp", "-MMD", "-MF", "obj/a.o.d.raw", "-gsplit-dwarf", "-c", "/w/a.cpp", "-o", "obj/a.o"}); + expect(fatal(s.has_value())); + expect(eq(s->source, fs::path("/w/a.cpp"))); + expect(eq(s->depfile, fs::path("obj/a.o.d.raw"))); + expect(eq(s->mapper, fs::path("modmap/p.map"))); + expect(eq(s->responses.size(), std::size_t{1})); + expect(eq(s->systemDirs.front(), fs::path("/store/inc"))); + expect(s->splitDebug); + expect(!uc::shape_of({"/store/g++", "-c", "a.cpp", "-o", "a.o"})) << "no depfile"; + expect(!uc::shape_of({"/store/g++", "-fmodule-mapper=|mapper", "-MF", "d", "-c", "a.cpp"})) << "a mapper program"; +}}; + +UTest a_depfile_lists_every_rules_prerequisites{"UnitCache/ADepfileListsEveryRulesPrerequisites", [] { + const auto p = uc::depfile_prerequisites( + "obj/a.o: /w/src/a.cpp /w/inc/a\\ b.h \\\n gcm.cache/x.gcm\nx.c++m: obj/a.o\nobj/a.o: /w/src/a.cpp C:/w/c.h\n"); + expect(eq(p, std::vector{"/w/src/a.cpp", "/w/inc/a b.h", "gcm.cache/x.gcm", "obj/a.o", "C:/w/c.h"})); +}}; + +UTest a_compile_is_served_while_what_it_read_holds{"UnitCache/ACompileIsServedWhileWhatItReadHolds", [] { + Tmp t; + const auto store = t.dir / "store", work = t.dir / "work", build = work / "target"; + write(store / "bin/g++", "compiler"); + write(work / "src/a.cpp", "int a();"); + write(work / "inc/a.h", "#define A 1"); + const std::vector argv = {(store / "bin/g++").string(), "-MMD", "-MF", "obj/a.o.d.raw", "-c", + (work / "src/a.cpp").string(), "-o", "obj/a.o"}; + const uc::Unit unit{argv, build, "obj/a.o", {}}; + uc::Cache cache(t.dir / "cache", {store}, build); + expect(eq(cache.refusal(unit), std::string{})); + expect(!cache.serve(unit)) << "an empty cache served"; + write(build / "obj/a.o", "OBJECT-1"); + write(build / "obj/a.o.d.raw", std::format("obj/a.o: {} {} {}\nobj/a.o: x.c++-module\nCXX_IMPORTS += x.c++-module\n", + (work / "src/a.cpp").generic_string(), (work / "inc/a.h").generic_string(), + (store / "inc/std.h").generic_string())); + cache.keep(unit); + fs::remove(build / "obj/a.o"); + fs::remove(build / "obj/a.o.d.raw"); + expect(fatal(cache.serve(unit))) << "the kept compile was not served"; + expect(eq(read(build / "obj/a.o"), std::string("OBJECT-1"))); + expect(read(build / "obj/a.o.d.raw").contains("inc/a.h")); + // A header it read changed: another compile. + write(work / "inc/a.h", "#define A 2"); + uc::Cache again(t.dir / "cache", {store}, build); + expect(!again.serve(unit)) << "a changed header was served"; + // A compiler outside the stores is refused. + const uc::Unit host{{"/usr/bin/g++", "-MF", "d", "-c", "a.cpp", "-o", "a.o"}, build, "a.o", {}}; + expect(ne(cache.refusal(host), std::string{})); +}}; diff --git a/src/bmi_cache/maintenance.cppm b/src/bmi_cache/maintenance.cppm index ca5b0a276..a1cf5ae10 100644 --- a/src/bmi_cache/maintenance.cppm +++ b/src/bmi_cache/maintenance.cppm @@ -381,11 +381,41 @@ std::vector walk_program_out_entries() { return out; } +// THE UNITS (design 2026-11-20 §5.3): one per compile key and manifest under +// `build-cache/v1/unit///`, last used when a build last +// placed it (its `manifest.tsv` is touched then). +std::vector walk_unit_entries() { + std::vector out; + std::error_code ec; + const auto root = mcpp::home::cache_root() / "unit"; + if (!std::filesystem::exists(root, ec)) return out; + // One walk, its third level the entries: `//`. + std::filesystem::recursive_directory_iterator it(root, ec), end; + for (; it != end; it.increment(ec)) { + if (ec) break; + if (it.depth() < 2 || !it->is_directory()) continue; + it.disable_recursion_pending(); + Entry e; + e.dir = it->path(); + e.kind = "unit"; + e.key = e.dir.parent_path().filename().string() + "/" + e.dir.filename().string(); + e.label = std::format("unit {}", e.key); + std::error_code fe; + e.complete = std::filesystem::is_regular_file(e.dir / "manifest.tsv", fe); + if (!e.complete) e.problem = "no manifest.tsv (an interrupted publication)"; + e.accessed = dir_mtime_seconds(e.complete ? e.dir / "manifest.tsv" : e.dir); + measure(e); + out.push_back(std::move(e)); + } + return out; +} + std::vector walk_collectable() { auto out = walk_pkg_entries(); for (auto& e : walk_tool_entries()) out.push_back(std::move(e)); for (auto& e : walk_variant_entries()) out.push_back(std::move(e)); for (auto& e : walk_program_out_entries()) out.push_back(std::move(e)); + for (auto& e : walk_unit_entries()) out.push_back(std::move(e)); return out; } diff --git a/src/build/execute.cppm b/src/build/execute.cppm index 2968b4eec..69e3a9ba1 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -2623,6 +2623,7 @@ std::optional run_ninja_fast(const std::string& ninjaProgram, compilerRun->environment(childEnv); childEnv.emplace_back("MCPP_MEMORY_THRESHOLD", std::to_string(mcpp::build::read_memory_threshold_sidecar(outputDir))); + for (auto& kv : mcpp::build::read_unit_cache_sidecar(outputDir)) childEnv.push_back(std::move(kv)); } // The build's record, as the full path keeps it (design 2026-11-20 §5.6). diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index de785696f..3c399163e 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -53,6 +53,7 @@ import mcpplibs.os.platform; import mcpplibs.os.windows; import mcpp.invocation; import mcpp.build.record; // what a build ran, edge by edge (design 2026-11-20 §5.6) +import mcpp.build.unit_cache; // the variables of the unit cache (design 2026-11-20 §5.3) import mcpplibs.os.fs; // the usage file's spelling for the executor import mcpp.runtime.launch_env; import mcpp.build.command_line; @@ -354,6 +355,12 @@ int read_compiler_cap_sidecar(const std::filesystem::path& outputDir); // executors as the full path does. void write_memory_threshold_sidecar(const std::filesystem::path& outputDir, std::uint64_t bytes); std::uint64_t read_memory_threshold_sidecar(const std::filesystem::path& outputDir); +// The unit cache the plan states (design 2026-11-20 §5.3), as the variables the +// compiler executor reads: nothing when the build uses no global cache. Kept +// beside build.ninja for the fast path, which has no plan. +void write_unit_cache_sidecar(const std::filesystem::path& outputDir, const std::filesystem::path& cacheDir, + const std::vector& roots); +std::vector> read_unit_cache_sidecar(const std::filesystem::path& outputDir); // #693 (M4): whether a Ninja reads build.ninja in the encoding mcpp writes it. // @@ -1301,6 +1308,27 @@ std::uint64_t read_memory_threshold_sidecar(const std::filesystem::path& outputD return f >> bytes ? bytes : 0; } +constexpr std::string_view kUnitCacheSidecar = ".mcpp-unit-cache"; + +void write_unit_cache_sidecar(const std::filesystem::path& outputDir, const std::filesystem::path& cacheDir, + const std::vector& roots) { + std::string text; + if (!cacheDir.empty()) { + text = mcpplibs::os::fs::utf8(cacheDir) + "\n"; + for (const auto& r : roots) text += mcpplibs::os::fs::utf8(r) + "\n"; + } + std::ofstream(outputDir / kUnitCacheSidecar, std::ios::binary | std::ios::trunc) << text; +} + +std::vector> read_unit_cache_sidecar(const std::filesystem::path& outputDir) { + std::ifstream f(outputDir / kUnitCacheSidecar, std::ios::binary); + std::string dir, roots; + if (!std::getline(f, dir) || dir.empty()) return {}; + for (std::string line; std::getline(f, line);) roots += line + "\n"; + return {{std::string(mcpp::build::unit_cache::kDirEnv), dir}, + {std::string(mcpp::build::unit_cache::kRootsEnv), roots}}; +} + std::vector read_consumer_include_sidecar(const std::filesystem::path& outputDir) { std::vector dirs; @@ -4921,6 +4949,7 @@ std::expected NinjaBackend::build(const BuildPlan& plan write_consumer_include_sidecar(plan.outputDir, root_include_dirs_of(plan)); write_compiler_cap_sidecar(plan.outputDir, plan.scheduleCompilerCap); write_memory_threshold_sidecar(plan.outputDir, plan.memoryThreshold); + write_unit_cache_sidecar(plan.outputDir, plan.unitCacheDir, plan.immutableRoots); write_graph_c_library_sidecar(plan.outputDir, plan.targetSide.cAbi); std::string placements; mcpp::build::progress::Attribution attribution; @@ -5273,6 +5302,7 @@ std::expected NinjaBackend::build(const BuildPlan& plan nenv.emplace_back(ev.key, ev.value); if (compilerRun) compilerRun->environment(nenv); nenv.emplace_back("MCPP_MEMORY_THRESHOLD", std::to_string(plan.memoryThreshold)); + for (auto& kv : read_unit_cache_sidecar(plan.outputDir)) nenv.push_back(std::move(kv)); // THE BUILD'S RECORD (design 2026-11-20 §5.6): where ninja's log ends now, // and the file each compile's executor appends its measurement to. const auto logMark = mcpp::build::progress::mark_log(plan.outputDir); diff --git a/src/build/plan.cppm b/src/build/plan.cppm index fb84e03aa..b5f6441d2 100644 --- a/src/build/plan.cppm +++ b/src/build/plan.cppm @@ -588,6 +588,11 @@ struct BuildPlan { // The memory a compiler may start below (design 2026-11-20 §5.5), in bytes: // the compiler executor waits below it (MCPP_MEMORY_THRESHOLD in its env). std::uint64_t memoryThreshold = 0; + // The unit cache (design 2026-11-20 §5.3): its directory, empty when the + // build uses no global cache, and the roots whose files never change in + // place, which the compiler executor reads a path of as its identity. + std::filesystem::path unitCacheDir; + std::vector immutableRoots; // One immutable snapshot selected before workspace member substitution. // Build/run/test and cache fast paths consume this value; none may re-read // xlings active/current state. diff --git a/src/build/prepare/plan.cpp b/src/build/prepare/plan.cpp index 9b2114c91..3075bce63 100644 --- a/src/build/prepare/plan.cpp +++ b/src/build/prepare/plan.cpp @@ -1067,6 +1067,10 @@ static void step13_graph_and_schedule(PrepareState& state, BuildContext& ctx) { machineMemoryThreshold = (*c)->memoryThreshold; } ctx.globalDefaultJobs = globalDefaultJobs; + if (ctx.cacheMode == CacheMode::Global) { + ctx.plan.unitCacheDir = mcpp::home::cache_root() / "unit"; + ctx.plan.immutableRoots = state.storeRoots; + } ctx.plan.memoryThreshold = mcpp::build::schedule::resolve_memory_threshold( ctx.manifest, machineMemoryThreshold, [](std::string_view bad) { mcpp::ui::warning(std::format( diff --git a/src/cli.cppm b/src/cli.cppm index ccec26e28..90b3d66ae 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -33,6 +33,7 @@ import mcpplibs.os.platform; import mcpplibs.os.fs; import mcpplibs.os.process; // the compiler's use, measured by the executor import mcpp.build.record; // the line the executor appends (design 2026-11-20 §5.6) +import mcpp.build.unit_cache; // a compile placed by the contents it read (design 2026-11-20 §5.3) import mcpplibs.os.text; import mcpp.invocation; import mcpp.effects; @@ -332,8 +333,28 @@ int run(int argc, char** argv) { std::chrono::system_clock::now().time_since_epoch()).count(); }; const auto started = unixMs(); + // THE UNIT CACHE (design 2026-11-20 §5.3): a compile whose command and + // read files hold the contents an earlier one's did is placed, not run. + std::optional units; + mcpp::build::unit_cache::Unit unit{command, std::filesystem::current_path(), object ? output : std::filesystem::path{}, bmi}; + if (const auto dir = mcpp::invocation::get(mcpp::build::unit_cache::kDirEnv); dir && !dir->empty() && object) { + std::vector roots; + std::istringstream in(mcpp::invocation::get(mcpp::build::unit_cache::kRootsEnv).value_or("")); + for (std::string line; std::getline(in, line);) + if (!line.empty()) roots.push_back(mcpp::home::from_utf8(line)); + units.emplace(mcpp::home::from_utf8(*dir), std::move(roots), unit.cwd); + if (units->serve(unit)) { + if (const auto file = mcpp::invocation::get(mcpp::build::record::kUsageEnv); file && !file->empty()) { + mcpp::build::record::Usage u{output.generic_string(), started, unixMs()}; + u.served = true; + (void)mcpplibs::os::fs::append(mcpp::home::from_utf8(*file), mcpp::build::record::usage_line(u)); + } + return 0; + } + } const int status = mcpp::build::schedule::detach::run_capped(command, semaphore, cap, output, depfile, bmi, object, relocate, reading); + if (status == 0 && units) units->keep(unit); if (const auto file = mcpp::invocation::get(mcpp::build::record::kUsageEnv); file && !file->empty()) { mcpp::build::record::Usage u{output.generic_string(), started, unixMs()}; if (const auto used = mcpplibs::os::process::resources(mcpplibs::os::process::of::children)) { diff --git a/tests/e2e/cache/entries.cpp b/tests/e2e/cache/entries.cpp index 307c45bd9..ae31d56ae 100644 --- a/tests/e2e/cache/entries.cpp +++ b/tests/e2e/cache/entries.cpp @@ -79,6 +79,40 @@ void age(const fs::path& root, std::chrono::hours ago) { } // namespace E2E entries{"entries", [] { + "927 a fresh checkout's units are placed from the unit cache, and a changed header compiles its includers"_test[s::gcc] = [] { + // Design 2026-11-20 §5.3. A unit whose command and read files hold the contents an earlier + // compile's did is placed by the compiler executor, and no compiler runs: after `target/` + // is removed (a fresh checkout), every object edge of the build record is `served` (A). A + // header changed in content compiles the units that read it, and only those (B). + s::project p{{"mcpp.toml", "[package]\nname = \"uc\"\nversion = \"0.1.0\"\n\n[targets.uc]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"}, + {"src/main.cpp", "#include \"value.h\"\nimport uc.a;\nint main() { return a() + VALUE == 3 ? 0 : 1; }\n"}, + {"src/value.h", "#define VALUE 2\n"}, + {"src/a.cppm", "export module uc.a;\nexport int a() { return 1; }\n"}}; + const auto record = [&](std::string_view name) { return s::case_dir() / std::format("{}.ndjson", name); }; + auto objects = [&](std::string_view name) { + std::map served; // object -> placed + for (const auto& l : s::json_lines(s::read_file(record(name)))) + if (const auto out = l.value("output", ""); out.ends_with(".o")) served[out] = l.value("served", false); + return served; + }; + auto build = [&](std::string_view name) { + auto r = p.mcpp({"build"}, {.env = {{"MCPP_BUILD_RECORD", record(name).string()}}}); + expect(fatal(e2e::succeeded(r))) << name << ": " << s::text(r); + }; + build("first"); + p.remove("target"); + build("fresh"); + const auto fresh = objects("fresh"); + expect(fatal(fresh.size() >= 2u)) << "A: the record names no object edges\n" << s::read_file(record("fresh")); + for (const auto& [object, placed] : fresh) expect(placed) << "A: " << object << " was compiled again"; + expect(e2e::succeeded(p.mcpp({"run"}))) << "A: the program linked from placed objects"; + p.write("src/value.h", "#define VALUE 5\n"); + p.remove("target"); + build("changed"); + for (const auto& [object, placed] : objects("changed")) + expect(placed == !object.contains("main")) << "B: " << object << (placed ? " was placed" : " was compiled"); + }; + "172 build cache cross project"_test[s::gcc][s::fresh_sandbox] = [] { // The global build cache saves work across projects: a dependency's key carries no // consumer identity, and the second project has zero compile edges for it (read from From 36b98a09089d0dfdaa6ac93c39fa9648c07160e0 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 04:32:09 +0900 Subject: [PATCH 14/38] feat: the split schedule's units are placed from the unit cache too; a BMI's producer key is stated before it is published, and a relocated attempt's BMI is not moved twice --- modules/unit-cache/src/unit_cache.cppm | 268 +++++++++++++++++-- modules/unit-cache/tests/test_unit_cache.cpp | 52 +++- src/build/schedule/detach_codegen.cppm | 105 +++++++- src/build/schedule/relocation.cppm | 7 +- 4 files changed, 401 insertions(+), 31 deletions(-) diff --git a/modules/unit-cache/src/unit_cache.cppm b/modules/unit-cache/src/unit_cache.cppm index 98b5c21fe..331560b04 100644 --- a/modules/unit-cache/src/unit_cache.cppm +++ b/modules/unit-cache/src/unit_cache.cppm @@ -32,6 +32,11 @@ inline constexpr std::string_view kRootsEnv = "MCPP_IMMUTABLE_ROOTS"; // A build directory's memory of file digests, by path, size and modification time: a header is // read once per build, not once per unit that includes it. inline constexpr std::string_view kDigestsFile = ".mcpp-digests"; +// Beside a BMI the cache kept or placed: the key of the compile that wrote it. A GCC BMI is not +// the same bytes from one compile of a module to the next (measured on mcpp's own modules: +// mcpp.build.prepare and mcpp.toolchain.triple), so an importer names the BMI it read by its +// producer's key, which equal inputs give equal, and its bytes only where no key is beside it. +inline constexpr std::string_view kProducerSuffix = ".mcpp-unit"; // One compile, as the executor runs it: the compiler's command, the directory it runs in, and the // outputs the executor publishes. @@ -40,8 +45,28 @@ struct Unit { std::filesystem::path cwd; std::filesystem::path object; std::filesystem::path bmi; // empty: the unit provides no module + // The files the compile reads, where the executor knows them before the compiler runs (the + // split schedule: the scan's depfile and the BMIs its imports name); unset: the command's + // depfile states them once it ran. + std::optional> reads; + // Where a hit places the object and the BMI when the compiler would have written them + // elsewhere than where they are published (a relocated attempt); empty: `object`, `bmi`. + std::filesystem::path placeObject, placeBmi; }; +// A unit as one text and back, for the executor that decides to the one that compiles. +std::string to_record(const Unit& u); +std::optional from_record(std::string_view text); + +// The words of a command written as one line, quotes and backslashes read as a POSIX shell reads +// them (a rule's command file). +std::vector split_command(std::string_view line); + +// The BMIs a scan's P1689 record says the unit imports, by the module mapper's lines +// (`$root `, then ` `), relative to `cwd`. +std::vector imported_bmis(std::string_view p1689, std::string_view mapper, + const std::filesystem::path& cwd); + // What the command names, read from its words: the source (`-c `), the depfile (`-MF`), the // module mapper (`-fmodule-mapper=`), the response files (`@file`), the directories searched as // the system's (`-isystem`, `-idirafter`), and whether the debug information is split. @@ -65,10 +90,20 @@ public: bool serve(const Unit& u); // Records what a compile that succeeded read, and keeps its outputs. void keep(const Unit& u); + // A compile the cache did not keep: its BMI has no producer key. + static void forget(const Unit& u); + // Before a compile whose reads are known: its BMI's producer key, beside the published BMI, so + // that an importer that starts once the BMI is published reads it (the split schedule + // publishes a BMI while its object is still being compiled). + void announce(const Unit& u); private: std::string first_key(const Unit& u, const Shape& s); + // The manifest of what `u` read, its lines sorted; nullopt when a file it read is gone. + std::optional manifest_of(const Unit& u, const std::vector& read); std::string digest(const std::filesystem::path& file); + // What a manifest records of a file it read: its producer's key, else its digest. + std::string identity(const std::filesystem::path& file); bool immutable(const std::filesystem::path& file) const; std::filesystem::path dir_; std::vector roots_; @@ -123,7 +158,7 @@ std::optional shape_of(const std::vector& argv) { s.systemDirs.emplace_back(w.starts_with("-isystem") ? w.substr(8) : w.substr(10)); else if (w == "-gsplit-dwarf") s.splitDebug = true; } - if (s.source.empty() || s.depfile.empty()) return std::nullopt; + if (s.source.empty()) return std::nullopt; // A module mapper that is a program, or a server, answers what a file cannot record. if (!s.mapper.empty() && (s.mapper.native().find('|') != fs::path::string_type::npos || s.mapper.native().find('=') != fs::path::string_type::npos @@ -166,6 +201,119 @@ std::vector depfile_prerequisites(std::string_view text) { return out; } +std::string to_record(const Unit& u) { + std::string out; + auto line = [&](std::string_view key, std::string_view value) { + out += key; + out += '\t'; + out += value; + out += '\n'; + }; + line("cwd", u.cwd.generic_string()); + line("object", u.object.generic_string()); + line("bmi", u.bmi.generic_string()); + line("place_object", u.placeObject.generic_string()); + line("place_bmi", u.placeBmi.generic_string()); + for (const auto& w : u.argv) line("argv", w); + if (u.reads) { + line("reads", ""); + for (const auto& r : *u.reads) line("read", r.generic_string()); + } + return out; +} + +std::optional from_record(std::string_view text) { + Unit u; + while (!text.empty()) { + const auto nl = text.find('\n'); + const auto line = text.substr(0, nl); + text = nl == std::string_view::npos ? std::string_view{} : text.substr(nl + 1); + const auto tab = line.find('\t'); + if (tab == std::string_view::npos) continue; + const auto key = line.substr(0, tab); + const std::string value(line.substr(tab + 1)); + if (key == "cwd") u.cwd = value; + else if (key == "object") u.object = value; + else if (key == "bmi") u.bmi = value; + else if (key == "place_object") u.placeObject = value; + else if (key == "place_bmi") u.placeBmi = value; + else if (key == "argv") u.argv.push_back(value); + else if (key == "reads") u.reads.emplace(); + else if (key == "read" && u.reads) u.reads->emplace_back(value); + } + if (u.argv.empty() || u.cwd.empty()) return std::nullopt; + return u; +} + +std::vector split_command(std::string_view line) { + std::vector out; + std::string word; + bool inWord = false; + char quote = 0; + for (std::size_t i = 0; i < line.size(); ++i) { + const char c = line[i]; + if (quote == '\'') { + if (c == '\'') quote = 0; + else word += c; + continue; + } + if (c == '\\' && i + 1 < line.size() && quote != '\'') { + word += line[++i]; + inWord = true; + continue; + } + if (quote == '"') { + if (c == '"') quote = 0; + else word += c; + continue; + } + if (c == '\'' || c == '"') { + quote = c; + inWord = true; + continue; + } + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + if (inWord) out.push_back(std::move(word)); + word.clear(); + inWord = false; + continue; + } + word += c; + inWord = true; + } + if (inWord) out.push_back(std::move(word)); + return out; +} + +std::vector imported_bmis(std::string_view p1689, std::string_view mapper, const fs::path& cwd) { + std::map> path_of; + fs::path root; + std::istringstream in{std::string(mapper)}; + for (std::string line; std::getline(in, line);) { + const auto sp = line.find(' '); + if (sp == std::string::npos) continue; + const auto name = line.substr(0, sp); + const auto where = line.substr(sp + 1); + if (name == "$root") root = where; + else path_of.emplace(name, where); + } + std::vector out; + const auto requires_at = p1689.find("\"requires\""); + if (requires_at == std::string_view::npos) return out; + constexpr std::string_view kName = "\"logical-name\""; + for (auto at = p1689.find(kName, requires_at); at != std::string_view::npos; at = p1689.find(kName, at + 1)) { + const auto open = p1689.find('"', at + kName.size()); + const auto close = open == std::string_view::npos ? open : p1689.find('"', open + 1); + if (close == std::string_view::npos) break; + const auto name = p1689.substr(open + 1, close - open - 1); + if (auto it = path_of.find(name); it != path_of.end()) { + const auto p = it->second.is_absolute() ? it->second : root / it->second; + out.push_back(p.is_absolute() ? p : (cwd / p).lexically_normal()); + } + } + return out; +} + Cache::Cache(fs::path dir, std::vector immutableRoots, fs::path buildDir) : dir_(std::move(dir)), roots_(std::move(immutableRoots)), buildDir_(std::move(buildDir)) { for (auto& r : roots_) r = r.lexically_normal(); @@ -208,10 +356,37 @@ std::string Cache::digest(const fs::path& file) { return d; } +std::string Cache::identity(const fs::path& file) { + auto producer = file; + producer += kProducerSuffix; + std::error_code ec; + if (fs::is_regular_file(producer, ec)) + if (auto key = read_all(producer); !key.empty()) return "unit:" + key; + return digest(file); +} + +namespace { +void write_producer(const fs::path& bmi, std::string_view key) { + if (bmi.empty()) return; + auto producer = bmi; + producer += kProducerSuffix; + std::ofstream(producer, std::ios::binary | std::ios::trunc) << key; +} +} // namespace + +void Cache::forget(const Unit& u) { + if (u.bmi.empty()) return; + auto producer = u.bmi.is_absolute() ? u.bmi : u.cwd / u.bmi; + producer += kProducerSuffix; + std::error_code ec; + fs::remove(producer, ec); +} + std::string Cache::refusal(const Unit& u) const { if (u.argv.empty()) return "no command"; const auto shape = shape_of(u.argv); - if (!shape) return "the command names no source and depfile"; + if (!shape) return "the command names no source"; + if (shape->depfile.empty() && !u.reads) return "nothing states the files the compile reads"; if (!immutable(fs::path(u.argv.front()))) return "the compiler is not in a store whose files never change"; for (const auto& d : shape->systemDirs) @@ -252,24 +427,30 @@ bool Cache::serve(const Unit& u) { const fs::path file = line.substr(0, tab); const auto want = line.substr(tab + 1); if (want == "=") continue; // immutable: its path is its identity - holds = digest(file) == want; + holds = identity(file) == want; } if (!holds || !any) continue; const auto out = m.path(); // Each output, unless it already holds the bytes (so ninja's restat sees no change). - auto put = [&](const fs::path& from, const fs::path& to) { + // A place the compiler would have written is written whatever it holds: the executor + // watches it for the compile's outputs. + auto put = [&](const fs::path& from, const fs::path& to, bool always) { if (to.empty() || !fs::exists(from, ec)) return true; const auto target = to.is_absolute() ? to : u.cwd / to; - if (fs::exists(target, ec) && mcpp::toolchain::hash_file(target) == mcpp::toolchain::hash_file(from)) + if (!always && fs::exists(target, ec) + && mcpp::toolchain::hash_file(target) == mcpp::toolchain::hash_file(from)) return true; return place(from, target); }; - if (!put(out / "object", u.object) || !put(out / "bmi", u.bmi) || !put(out / "depfile", shape->depfile)) + if (!put(out / "object", u.placeObject.empty() ? u.object : u.placeObject, !u.placeObject.empty()) + || !put(out / "bmi", u.placeBmi.empty() ? u.bmi : u.placeBmi, !u.placeBmi.empty()) + || (!shape->depfile.empty() && !put(out / "depfile", shape->depfile, false))) return false; if (shape->splitDebug) { - auto dwo = u.object; - if (!put(out / "dwo", dwo.replace_extension(".dwo"))) return false; + auto dwo = u.placeObject.empty() ? u.object : u.placeObject; + if (!put(out / "dwo", dwo.replace_extension(".dwo"), !u.placeObject.empty())) return false; } + if (!u.bmi.empty()) write_producer(u.bmi.is_absolute() ? u.bmi : u.cwd / u.bmi, key + "/" + out.filename().string()); // Its last use, which `mcpp cache gc` collects by. fs::last_write_time(out / "manifest.tsv", fs::file_time_type::clock::now(), ec); return true; @@ -277,33 +458,67 @@ bool Cache::serve(const Unit& u) { return false; } -void Cache::keep(const Unit& u) { - if (!refusal(u).empty()) return; - const auto shape = shape_of(u.argv); - const auto depfile = shape->depfile.is_absolute() ? shape->depfile : u.cwd / shape->depfile; - const auto text = read_all(depfile); - if (text.empty()) return; - const auto object = u.object.is_absolute() ? u.object : u.cwd / u.object; - const auto bmi = u.bmi.empty() ? fs::path{} : (u.bmi.is_absolute() ? u.bmi : u.cwd / u.bmi); - std::string manifest; - for (const auto& p : depfile_prerequisites(text)) { +std::optional Cache::manifest_of(const Unit& u, const std::vector& read) { + const auto object = (u.object.is_absolute() ? u.object : u.cwd / u.object).lexically_normal(); + const auto bmi = u.bmi.empty() ? fs::path{} : (u.bmi.is_absolute() ? u.bmi : u.cwd / u.bmi).lexically_normal(); + // One line per file, sorted: the same files read in another order are the same manifest. + std::set lines; + for (const auto& p : read) { // GCC names a module a unit imports or provides (`x.c++-module`) beside the files. if (p.ends_with(".c++-module")) continue; const auto file = fs::path(normal(u.cwd, p)); - if (file == object.lexically_normal() || (!bmi.empty() && file == bmi.lexically_normal())) continue; + if (file == object || (!bmi.empty() && file == bmi)) continue; if (immutable(file)) { - manifest += file.generic_string() + "\t=\n"; + lines.insert(file.generic_string() + "\t=\n"); continue; } - const auto d = digest(file); - if (d.empty()) return; // a file it read is gone: nothing to record - manifest += file.generic_string() + "\t" + d + "\n"; + const auto d = identity(file); + if (d.empty()) return std::nullopt; // a file it read is gone: nothing to record + lines.insert(file.generic_string() + "\t" + d + "\n"); } - if (manifest.empty()) return; + std::string manifest; + for (const auto& l : lines) manifest += l; + return manifest; +} + +void Cache::announce(const Unit& u) { + if (u.bmi.empty() || !u.reads || !refusal(u).empty()) return; + std::vector read; + for (const auto& r : *u.reads) read.push_back(r.generic_string()); + const auto manifest = manifest_of(u, read); + if (!manifest || manifest->empty()) return forget(u); + const auto shape = shape_of(u.argv); + write_producer(u.bmi.is_absolute() ? u.bmi : u.cwd / u.bmi, + first_key(u, *shape) + "/" + mcpp::toolchain::hash_string(*manifest)); +} + +void Cache::keep(const Unit& u) { + if (!refusal(u).empty()) return forget(u); + const auto shape = shape_of(u.argv); + const auto depfile = shape->depfile.empty() ? fs::path{} + : shape->depfile.is_absolute() ? shape->depfile : u.cwd / shape->depfile; + std::vector read; + if (u.reads) { + for (const auto& r : *u.reads) read.push_back(r.generic_string()); + } else { + const auto text = read_all(depfile); + if (text.empty()) return forget(u); + read = depfile_prerequisites(text); + } + const auto object = u.object.is_absolute() ? u.object : u.cwd / u.object; + const auto bmi = u.bmi.empty() ? fs::path{} : (u.bmi.is_absolute() ? u.bmi : u.cwd / u.bmi); + // A BMI the unit provides: where it is published, else where the compiler wrote it (phase 1 + // of the split schedule may not have published it yet). Neither: not kept, a hit would place none. + const auto written = u.placeBmi.empty() ? fs::path{} : (u.placeBmi.is_absolute() ? u.placeBmi : u.cwd / u.placeBmi); + const auto bmiBytes = bmi.empty() ? fs::path{} : fs::exists(bmi) ? bmi : (!written.empty() && fs::exists(written)) ? written : fs::path{}; + if (!bmi.empty() && bmiBytes.empty()) return forget(u); + const auto manifest = manifest_of(u, read).value_or(""); + if (manifest.empty()) return forget(u); const auto key = first_key(u, *shape); const auto entry = dir_ / key.substr(0, 2) / key; const auto name = mcpp::toolchain::hash_string(manifest); std::error_code ec; + write_producer(bmi, key + "/" + name); if (fs::is_directory(entry / name, ec)) return; const auto staging = entry / std::format(".staging-{:x}", std::random_device{}()); fs::create_directories(staging, ec); @@ -311,8 +526,8 @@ void Cache::keep(const Unit& u) { bool ok = static_cast(std::ofstream(staging / "manifest.tsv", std::ios::binary) << manifest); ok = ok && fs::exists(object, ec); if (ok) fs::copy_file(object, staging / "object", ec), ok = !ec; - if (ok && !bmi.empty() && fs::exists(bmi, ec)) fs::copy_file(bmi, staging / "bmi", ec), ok = !ec; - if (ok) fs::copy_file(depfile, staging / "depfile", ec), ok = !ec; + if (ok && !bmiBytes.empty()) fs::copy_file(bmiBytes, staging / "bmi", ec), ok = !ec; + if (ok && !depfile.empty()) fs::copy_file(depfile, staging / "depfile", ec), ok = !ec; if (ok && shape->splitDebug) { auto dwo = object; dwo.replace_extension(".dwo"); @@ -321,6 +536,7 @@ void Cache::keep(const Unit& u) { // Published whole by one rename; a compile of the same key that got there first wins. if (ok) fs::rename(staging, entry / name, ec); if (!ok || ec) fs::remove_all(staging, ec); + if (!ok) forget(u); } } // namespace mcpp::build::unit_cache diff --git a/modules/unit-cache/tests/test_unit_cache.cpp b/modules/unit-cache/tests/test_unit_cache.cpp index 7008cf44a..73e3262b2 100644 --- a/modules/unit-cache/tests/test_unit_cache.cpp +++ b/modules/unit-cache/tests/test_unit_cache.cpp @@ -37,7 +37,8 @@ UTest a_command_states_its_source_and_depfile{"UnitCache/ACommandStatesItsSource expect(eq(s->responses.size(), std::size_t{1})); expect(eq(s->systemDirs.front(), fs::path("/store/inc"))); expect(s->splitDebug); - expect(!uc::shape_of({"/store/g++", "-c", "a.cpp", "-o", "a.o"})) << "no depfile"; + expect(uc::shape_of({"/store/g++", "-c", "a.cpp", "-o", "a.o"})->depfile.empty()) << "no depfile"; + expect(!uc::shape_of({"/store/g++", "-o", "a.o"})) << "no source"; expect(!uc::shape_of({"/store/g++", "-fmodule-mapper=|mapper", "-MF", "d", "-c", "a.cpp"})) << "a mapper program"; }}; @@ -77,3 +78,52 @@ UTest a_compile_is_served_while_what_it_read_holds{"UnitCache/ACompileIsServedWh const uc::Unit host{{"/usr/bin/g++", "-MF", "d", "-c", "a.cpp", "-o", "a.o"}, build, "a.o", {}}; expect(ne(cache.refusal(host), std::string{})); }}; + +UTest an_importer_names_a_bmi_by_its_producer{"UnitCache/AnImporterNamesABmiByItsProducer", [] { + // A GCC BMI is not the same bytes from one compile to the next: the importer's manifest names + // it by the key of the compile that wrote it, so a BMI written again from the same inputs + // serves the importer. + Tmp t; + const auto store = t.dir / "store", build = t.dir / "build"; + write(store / "bin/g++", "compiler"); + write(t.dir / "src/m.cppm", "export module m;"); + write(t.dir / "src/user.cpp", "import m;"); + uc::Cache cache(t.dir / "cache", {store}, build); + const uc::Unit module{{(store / "bin/g++").string(), "-MF", "obj/m.o.d", "-c", (t.dir / "src/m.cppm").string(), "-o", "obj/m.o"}, + build, "obj/m.o", "gcm.cache/m.gcm"}; + write(build / "obj/m.o", "M"); + write(build / "gcm.cache/m.gcm", "BMI-1"); + write(build / "obj/m.o.d", std::format("obj/m.o gcm.cache/m.gcm: {}\n", (t.dir / "src/m.cppm").generic_string())); + cache.keep(module); + expect(fatal(fs::exists(build / "gcm.cache/m.gcm.mcpp-unit"))) << "the BMI has no producer key"; + const uc::Unit user{{(store / "bin/g++").string(), "-MF", "obj/user.o.d", "-c", (t.dir / "src/user.cpp").string(), "-o", "obj/user.o"}, + build, "obj/user.o", {}}; + write(build / "obj/user.o", "U"); + write(build / "obj/user.o.d", std::format("obj/user.o: {} gcm.cache/m.gcm\n", (t.dir / "src/user.cpp").generic_string())); + cache.keep(user); + // The module compiled again from the same inputs: other bytes, the same producer. + write(build / "gcm.cache/m.gcm", "BMI-2"); + fs::remove(build / "obj/user.o"); + uc::Cache again(t.dir / "cache", {store}, build); + expect(again.serve(user)) << "a BMI written again from the same inputs did not serve its importer"; + // A BMI whose producer is not the cache's is named by its bytes. + uc::Cache::forget(module); + uc::Cache third(t.dir / "cache", {store}, build); + expect(!third.serve(user)) << "a BMI without its producer's key served by name"; +}}; + +UTest a_unit_travels_as_a_record_and_its_imports_are_read_from_the_scan{"UnitCache/AUnitTravelsAsARecord", [] { + uc::Unit u{uc::split_command(R"(/store/g++ -x c++ "-DNAME=a b" -c /w/a.cppm -o obj/a.o)"), "/build", "obj/a.o", "gcm.cache/a.gcm"}; + expect(eq(u.argv[3], std::string("-DNAME=a b"))); + u.reads = std::vector{"/w/a.cppm", "gcm.cache/b.gcm"}; + u.placeBmi = ".mcpp-attempts/1/out/gcm.cache/a.gcm"; + const auto back = uc::from_record(uc::to_record(u)); + expect(fatal(back.has_value())); + expect(eq(back->argv, u.argv)); + expect(eq(back->placeBmi, u.placeBmi)); + expect(fatal(back->reads.has_value())); + expect(eq(back->reads->size(), std::size_t{2})); + const auto bmis = uc::imported_bmis(R"({"rules":[{"provides":[{"logical-name":"a"}],"requires":[{"logical-name":"b"},{"logical-name":"std"}]}]})", + "$root gcm.cache\nb b.gcm\nstd std.gcm\na a.gcm\n", "/build"); + expect(eq(bmis, std::vector{"/build/gcm.cache/b.gcm", "/build/gcm.cache/std.gcm"})); +}}; diff --git a/src/build/schedule/detach_codegen.cppm b/src/build/schedule/detach_codegen.cppm index c170e19d7..a15ff2ee7 100644 --- a/src/build/schedule/detach_codegen.cppm +++ b/src/build/schedule/detach_codegen.cppm @@ -76,6 +76,8 @@ import mcpplibs.os.platform; import mcpplibs.os.stdio; import mcpplibs.os.text; import mcpplibs.os.machine; +import mcpp.build.unit_cache; // a compile placed by the contents it read (design 2026-11-20 §5.3) +import mcpp.build.record; // the compile's measurement (design 2026-11-20 §5.6) import mcpp.build.depfile; import mcpp.build.command_line; import mcpp.run; @@ -936,6 +938,88 @@ std::expected RunScope::finish() { return state_->close(); } const std::filesystem::path& RunScope::directory() const { return state_->generation; } bool RunScope::failed() const {return state_->witnesses&&state_->witnesses->failed();} +namespace { +// THE UNIT, FOR THE SUPERVISOR TO PLACE OR KEEP (design 2026-11-20 §5.3): its +// key is the command as the rule wrote it (an attempt's own paths are new each +// time), its outputs the paths it is published at, the places a hit writes +// where the compiler would have, and the files it reads -- the scan's depfile +// (what the compiler's own is equivalent to, above) and the BMIs the scan says +// it imports, by the module mapper. Nothing when the build states no cache. +void record_unit(const CompileRequest& req, const CompileRequest& actual, const std::filesystem::path& watched) { + namespace uc = mcpp::build::unit_cache; + std::error_code ec; + const auto file = suffixed(actual.slot, ".unit"); + std::filesystem::remove(file, ec); + const auto dir = mcpp::invocation::get(uc::kDirEnv); + if (!dir || dir->empty() || req.depFrom.empty()) return; + auto read = [](const std::filesystem::path& p) { + std::ifstream in(p, std::ios::binary); + return std::string{std::istreambuf_iterator(in), {}}; + }; + uc::Unit u{uc::split_command(req.command), std::filesystem::current_path(), req.object, req.bmi}; + const auto shape = uc::shape_of(u.argv); + if (!shape) return; + std::vector reads; + for (const auto& p : uc::depfile_prerequisites(read(req.depFrom))) reads.emplace_back(p); + if (auto ddi = req.depFrom; ddi.extension() == ".dep") { + ddi.replace_extension(); + const auto mapper = shape->mapper.empty() ? std::string{} : read(shape->mapper); + for (auto& b : uc::imported_bmis(read(ddi), mapper, u.cwd)) reads.push_back(std::move(b)); + } + u.reads = std::move(reads); + const auto words = uc::split_command(actual.command); + for (std::size_t i = 0; i + 1 < words.size(); ++i) + if (words[i] == "-o") u.placeObject = words[i + 1]; + u.placeBmi = watched; + std::ofstream(file, std::ios::binary | std::ios::trunc) << uc::to_record(u); + // Before the BMI is published, its producer's key: an importer starts then. + std::vector roots; + std::istringstream lines(mcpp::invocation::get(uc::kRootsEnv).value_or("")); + for (std::string line; std::getline(lines, line);) + if (!line.empty()) roots.push_back(mcpp::home::from_utf8(line)); + uc::Cache(mcpp::home::from_utf8(*dir), std::move(roots), u.cwd).announce(u); +} + +// The cache the build states, for the unit phase 1 recorded beside `slot`. +std::optional> +recorded_unit(const std::filesystem::path& slot) { + namespace uc = mcpp::build::unit_cache; + const auto dir = mcpp::invocation::get(uc::kDirEnv); + if (!dir || dir->empty()) return std::nullopt; + std::ifstream in(suffixed(slot, ".unit"), std::ios::binary); + if (!in) return std::nullopt; + auto u = uc::from_record(std::string{std::istreambuf_iterator(in), {}}); + if (!u) return std::nullopt; + std::vector roots; + std::istringstream lines(mcpp::invocation::get(uc::kRootsEnv).value_or("")); + for (std::string line; std::getline(lines, line);) + if (!line.empty()) roots.push_back(mcpp::home::from_utf8(line)); + return std::pair{uc::Cache(mcpp::home::from_utf8(*dir), std::move(roots), u->cwd), std::move(*u)}; +} + +long long unix_ms() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); +} + +// The compile's line in the build record (design 2026-11-20 §5.6): its object, +// and the compiler's processor time and largest resident set -- the supervisor +// starts no other program -- or that the unit cache placed it. +void record_usage(const std::filesystem::path& object, long long started, bool served) { + namespace rec = mcpp::build::record; + const auto file = mcpp::invocation::get(rec::kUsageEnv); + if (!file || file->empty() || object.empty()) return; + rec::Usage u{object.generic_string(), started, unix_ms()}; + u.served = served; + if (!served) + if (const auto used = mcpplibs::os::process::resources(mcpplibs::os::process::of::children)) { + u.cpuMs = std::chrono::duration(used->user + used->system).count(); + u.peakBytes = used->peak_memory; + } + (void)mcpplibs::os::fs::append(mcpp::home::from_utf8(*file), rec::usage_line(u)); +} +} // namespace + int compile_release_at_bmi(const CompileRequest& req) { if (req.command.empty() || req.self.empty()) return 2; Actor actor; @@ -996,6 +1080,7 @@ int compile_release_at_bmi(const CompileRequest& req) { // A relocated attempt watches its own BMI, which did not exist before it. const auto watched = plan.relocated() ? relocation::written_bmi(plan) : actual.bmi; const BmiIdentity before = plan.relocated() ? BmiIdentity{} : bmi_identity(actual.bmi); + record_unit(req, actual, watched); if (!attempt && !plan.relocated() && !actual.bmi.empty()) { const auto backup = suffixed(actual.bmi, ".bak"); std::filesystem::remove(backup, ec); @@ -1208,11 +1293,19 @@ int supervise(const std::filesystem::path& slot, if(!transactions::publish(attempt.directory/"admitted","mcpp-admitted-v1",{})) { (void)write_rc(slot,2);return 2; } - const int ran=run_to_completion(command,suffixed(slot,".log"),actor->owner,unit.get()); + auto cached=recorded_unit(slot); + const auto started=unix_ms(); + const bool served=cached&&cached->first.serve(cached->second); + const int ran=served?0:run_to_completion(command,suffixed(slot,".log"),actor->owner,unit.get()); if(auto retired=retire_owned(*unit,actor->owner);!retired) { record_supervisor_error(slot,retired.error());return 1; } const int rc=publish_outputs(ran); + if(cached&&!served) { + if(rc==0)cached->first.keep(cached->second); + else mcpp::build::unit_cache::Cache::forget(cached->second); + } + if(rc==0)record_usage(cached?cached->second.object:object,started,served); if(!attempt.retired(rc)) {record_supervisor_error(slot,"cannot publish compiler retirement certificate");return 1;} std::string status_failure; if(!write_rc(slot,rc,&status_failure)) { @@ -1256,7 +1349,15 @@ int supervise(const std::filesystem::path& slot, // Acquire in the supervisor: no lock-handle handoff, and no interval in // which an early-returning phase 1 could release the compiler's permit. if (!actor->owner.active()) return 130; - const int rc = publish_outputs(run_to_completion(command, suffixed(slot, ".log"), actor->owner)); + auto cached = recorded_unit(slot); + const auto started = unix_ms(); + const bool served = cached && cached->first.serve(cached->second); + const int rc = publish_outputs(served ? 0 : run_to_completion(command, suffixed(slot, ".log"), actor->owner)); + if (cached && !served) { + if (rc == 0) cached->first.keep(cached->second); + else mcpp::build::unit_cache::Cache::forget(cached->second); + } + if (rc == 0) record_usage(cached ? cached->second.object : object, started, served); settle(mcpp::run::ending() ? 130 : rc); return 0; // the supervisor's own status is not the compiler's } diff --git a/src/build/schedule/relocation.cppm b/src/build/schedule/relocation.cppm index 84c4eabd8..0c8572759 100644 --- a/src/build/schedule/relocation.cppm +++ b/src/build/schedule/relocation.cppm @@ -424,11 +424,14 @@ std::expected publish(const Plan& plan, bool withBmi) { // was given (clang's `-ftime-trace` beside the object): its counterpart. std::error_code ec; if (!fs::is_directory(plan.root, ec)) return {}; + // Compared absolute: the supervisor loads the plan by an absolute directory, and the outputs + // were named below the relative one phase 1 made; a BMI the walk did not recognise was moved + // before phase 1 published it (a compile the unit cache placed, design 2026-11-20 §5.3). std::set named; - for (const auto& o : plan.outputs) named.insert(o.written.lexically_normal()); + for (const auto& o : plan.outputs) named.insert(fs::absolute(o.written, ec).lexically_normal()); for (fs::recursive_directory_iterator it(plan.root, ec), end; !ec && it != end; it.increment(ec)) { std::error_code fec; - if (!it->is_regular_file(fec) || named.contains(it->path().lexically_normal())) continue; + if (!it->is_regular_file(fec) || named.contains(fs::absolute(it->path(), fec).lexically_normal())) continue; const auto rel = it->path().lexically_relative(plan.root); if (auto done = replace(it->path(), rel); !done) return done; } From 47cefb2ef584ca31a15e8c89f04899f9a7236e19 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 04:32:15 +0900 Subject: [PATCH 15/38] fix: a compile the executor did not keep leaves no producer key beside its BMI --- src/cli.cppm | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cli.cppm b/src/cli.cppm index 90b3d66ae..148eb8b2c 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -355,6 +355,7 @@ int run(int argc, char** argv) { const int status = mcpp::build::schedule::detach::run_capped(command, semaphore, cap, output, depfile, bmi, object, relocate, reading); if (status == 0 && units) units->keep(unit); + else if (object) mcpp::build::unit_cache::Cache::forget(unit); if (const auto file = mcpp::invocation::get(mcpp::build::record::kUsageEnv); file && !file->empty()) { mcpp::build::record::Usage u{output.generic_string(), started, unixMs()}; if (const auto used = mcpplibs::os::process::resources(mcpplibs::os::process::of::children)) { From fbc2e4c2cb9528d4e8e363570fb0fb0c537b9f8b Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 05:01:49 +0900 Subject: [PATCH 16/38] =?UTF-8?q?mcpp=20emit=20surface;=20an=20invocation?= =?UTF-8?q?=20records=20the=20items=20it=20used=20(design=202026-11-20=20?= =?UTF-8?q?=C2=A75.16)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The surface is each command and flag of the command line's definition (mcpplibs.cmdline 0.0.5 exports the tree), each key of the manifest's registry and each reason token, one ' ' per line. With MCPP_SURFACE_RECORD, an invocation appends the command and flags it was given, the registry keys its manifests state and the reasons it records (mcpp.surface). --- mcpp.toml | 6 +- modules/manifest/mcpp.toml | 1 + modules/manifest/src/toml.cppm | 34 ++++++ modules/surface/mcpp.toml | 17 +++ modules/surface/src/surface.cppm | 80 ++++++++++++++ modules/surface/tests/test_surface.cpp | 35 +++++++ src/build/refusal.cppm | 13 +++ src/cli.cppm | 16 ++- src/cli/cmd_surface.cppm | 139 +++++++++++++++++++++++++ tests/e2e/cli/commands.cpp | 40 +++++++ 10 files changed, 377 insertions(+), 4 deletions(-) create mode 100644 modules/surface/mcpp.toml create mode 100644 modules/surface/src/surface.cppm create mode 100644 modules/surface/tests/test_surface.cpp create mode 100644 src/cli/cmd_surface.cppm diff --git a/mcpp.toml b/mcpp.toml index 91548ce89..f00310198 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -106,11 +106,11 @@ linkage = "static" # Eat our own dog food: mcpp uses mcpplibs.cmdline for argument parsing. # TOML-native namespaced form: `mcpplibs.cmdline` is the dotted subtable -# `[dependencies.mcpplibs] cmdline = "0.0.4"` ⇒ identity (mcpplibs, cmdline). +# `[dependencies.mcpplibs] cmdline = "0.0.5"` ⇒ identity (mcpplibs, cmdline). # The bootstrap mcpp used by the self-host build (xlings install mcpp, well # past 0.0.6) understands this, so the legacy quoted form is no longer needed. [dependencies] -mcpplibs.cmdline = "0.0.4" +mcpplibs.cmdline = "0.0.5" # JSON for Modern C++, as the index's module package (no copied header). nlohmann.json = "3.12.0" # The C library's macros (`stdout`, `stderr`, `EOF`, `errno` …) as C++ names: @@ -155,6 +155,7 @@ mcpp.toolchain-model = { path = "modules/toolchain-model" } mcpp.buildmcpp = { path = "modules/buildmcpp" } mcpp.build-record = { path = "modules/build-record" } mcpp.unit-cache = { path = "modules/unit-cache" } +mcpp.surface = { path = "modules/surface" } [target.linux.build] sources = ["!src/native/windows/**"] @@ -180,6 +181,7 @@ members = [ "modules/buildmcpp", "modules/build-record", "modules/unit-cache", + "modules/surface", ] # `mcpp build` ignores [dev-dependencies]; only `mcpp test` resolves them. diff --git a/modules/manifest/mcpp.toml b/modules/manifest/mcpp.toml index 7f4b4f6e3..6608ff1ec 100644 --- a/modules/manifest/mcpp.toml +++ b/modules/manifest/mcpp.toml @@ -21,6 +21,7 @@ license = "Apache-2.0" mcpp.invocation = { path = "../invocation" } mcpp.libs = { path = "../libs" } mcpp.source-kind = { path = "../source-kind" } +mcpp.surface = { path = "../surface" } mcpp.toolchain-model = { path = "../toolchain-model" } mcpp.versioning = { path = "../versioning" } diff --git a/modules/manifest/src/toml.cppm b/modules/manifest/src/toml.cppm index 0c25522f6..710e79413 100644 --- a/modules/manifest/src/toml.cppm +++ b/modules/manifest/src/toml.cppm @@ -12,6 +12,7 @@ import mcpp.targetside; import std; import mcpp.invocation; import mcpp.source_kind; +import mcpp.surface; import mcpp.libs.toml; import mcpp.toolchain.triple; import mcpp.pm.dep_spec; @@ -1045,6 +1046,38 @@ namespace { std::expected parse_document(const t::Document& document, const std::filesystem::path& origin, LoadContext ctx); + +// Whether `at` holds `key` of the registry table `table` ("" for the table itself); a segment +// `` or `` matches each child table. +bool declares(const t::Table& at, std::string_view table, std::string_view key) { + if (table.empty()) return key.empty() || at.find(key) != at.end(); + const auto dot = table.find('.'); + const auto seg = table.substr(0, dot); + const auto rest = dot == std::string_view::npos ? std::string_view{} : table.substr(dot + 1); + if (seg.starts_with('<')) { + for (auto const& [_, v] : at) + if (v.is_table() && declares(v.as_table(), rest, key)) return true; + return false; + } + auto it = at.find(seg); + return it != at.end() && it->second.is_table() && declares(it->second.as_table(), rest, key); +} + +// The registry's keys a manifest states are the keys this invocation used: each is recorded +// for the test run that asked (SPEC-017 P5.6 `exercise`, design 2026-11-20 §5.16). A key is +// one item wherever it is written: under `[target..build]` when it is conditional, under +// `[workspace.]` when it is shared. +void record_declared_keys(const t::Document& doc) { + for (auto const& row : kKeyRegistry) { + const bool here = declares(doc.root(), row.table, row.key) + || (row.conditional && row.table == "build" + && declares(doc.root(), "target..build", row.key)) + || (row.shared && declares(doc.root(), std::format("workspace.{}", row.table), row.key)); + if (here) + mcpp::surface::exercised(mcpp::surface::kKey, row.key.empty() + ? std::string(row.table) : std::format("{}.{}", row.table, row.key)); + } +} } std::expected parse_string(std::string_view content, @@ -1056,6 +1089,7 @@ std::expected parse_string(std::string_view content, } auto m = parse_document(*doc, origin, ctx); if (!m && m.error().line == 0) locate_named_key(m.error(), *doc); + if (m) record_declared_keys(*doc); return m; } diff --git a/modules/surface/mcpp.toml b/modules/surface/mcpp.toml new file mode 100644 index 000000000..3b6f7fb30 --- /dev/null +++ b/modules/surface/mcpp.toml @@ -0,0 +1,17 @@ +# mcpp.surface -- the items of mcpp's surface and the record of those one invocation used +# (SPEC-017 P5.6 `exercise`; design 2026-11-20 §5.16). +# +# A leaf: an item is a kind and a name (`command build`, `flag build --target`, `key build.cache`, +# `reason unknown-target`), one per line. The command line, the manifest reader and the refusal +# sites hand it the items they used; when MCPP_SURFACE_RECORD names a file, each item is appended +# to it once per process. It knows nothing of where the items come from. + +[package] +name = "surface" +namespace = "mcpp" +version = "0.1.0" +description = "The items of mcpp's surface and the record of those an invocation used" +license = "Apache-2.0" + +[dev-dependencies] +mcpplibs.mtest = "0.11.0" diff --git a/modules/surface/src/surface.cppm b/modules/surface/src/surface.cppm new file mode 100644 index 000000000..4c6df3e86 --- /dev/null +++ b/modules/surface/src/surface.cppm @@ -0,0 +1,80 @@ +// mcpp.surface -- see mcpp.toml for what this package is. +export module mcpp.surface; + +import std; + +export namespace mcpp::surface { + +// The kinds, in the order `mcpp emit surface` writes them. +inline constexpr std::string_view kCommand = "command"; +inline constexpr std::string_view kFlag = "flag"; +inline constexpr std::string_view kKey = "key"; +inline constexpr std::string_view kReason = "reason"; + +// The file an invocation appends the items it used to (SPEC-017 P5.6). Unset, nothing is written. +inline constexpr std::string_view kRecordEnv = "MCPP_SURFACE_RECORD"; + +struct Item { + std::string kind; + std::string name; + auto operator<=>(const Item&) const = default; +}; + +// ` \n`: the line of `mcpp emit surface` and of the record alike. +std::string line(const Item& item); + +// The items of `text`, one per line; a line without a kind and a name is skipped, since a +// process killed while appending leaves a partial last line. +std::vector parse(std::string_view text); + +// The flag item of `--name` on the command `path` ("" for the top level): `build --target`. +std::string flag_name(std::string_view path, std::string_view option); + +// Appends `item` to `file` unless this process appended it there already. +void append_once(const std::filesystem::path& file, const Item& item); + +// Appends the item to the file MCPP_SURFACE_RECORD names, once per process. +void exercised(std::string_view kind, std::string_view name); + +} // namespace mcpp::surface + +namespace mcpp::surface { + +std::string line(const Item& item) { return std::format("{} {}\n", item.kind, item.name); } + +std::vector parse(std::string_view text) { + std::vector out; + for (auto part : std::views::split(text, '\n')) { + std::string_view l(part.begin(), part.end()); + if (!l.empty() && l.back() == '\r') l.remove_suffix(1); + const auto sp = l.find(' '); + if (sp == std::string_view::npos || sp == 0 || sp + 1 == l.size()) continue; + out.push_back({std::string(l.substr(0, sp)), std::string(l.substr(sp + 1))}); + } + return out; +} + +std::string flag_name(std::string_view path, std::string_view option) { + return path.empty() ? std::format("--{}", option) : std::format("{} --{}", path, option); +} + +void append_once(const std::filesystem::path& file, const Item& item) { + // Sub-builds of one invocation run in the same process; a key read by each of them is one use. + static std::mutex m; + static std::set> seen; + std::lock_guard lock(m); + if (!seen.emplace(file, item).second) return; + // Appended, never rewritten: a build program's own mcpp writes to the same file. + std::ofstream os(file, std::ios::app | std::ios::binary); + os << line(item); +} + +void exercised(std::string_view kind, std::string_view name) { + static const std::string file = [] { + const char* v = std::getenv(std::string(kRecordEnv).c_str()); + return std::string(v ? v : ""); + }(); + if (!file.empty()) append_once(file, {std::string(kind), std::string(name)}); +} + +} // namespace mcpp::surface diff --git a/modules/surface/tests/test_surface.cpp b/modules/surface/tests/test_surface.cpp new file mode 100644 index 000000000..b0db7e950 --- /dev/null +++ b/modules/surface/tests/test_surface.cpp @@ -0,0 +1,35 @@ +import std; +import mcpplibs.mtest; +import mcpp.surface; + +using namespace mcpplibs::mtest; + +// SUBSYSTEM-LEVEL: the line of an item and the record it is appended to. That `mcpp emit +// surface` lists the items, and an invocation records those it used, is stated by the e2e suite +// (tests/e2e/test/protocol.cpp). + +namespace sf = mcpp::surface; + +UTest an_item_reads_back_from_its_line{"Surface/AnItemReadsBackFromItsLine", [] { + const auto text = sf::line({"flag", sf::flag_name("cache gc", "dry-run")}) + + sf::line({"key", "build.cache"}) + "\nreason\nkey build.cflags\r\nflag "; + const auto back = sf::parse(text); + expect(fatal(eq(back.size(), std::size_t{3}))); + expect(eq(back[0].name, std::string("cache gc --dry-run"))); + expect(eq(back[1].kind, std::string("key"))); + expect(eq(back[2].name, std::string("build.cflags"))) << "a CRLF line reads the same"; + expect(eq(sf::flag_name("", "quiet"), std::string("--quiet"))); +}}; + +UTest an_item_is_recorded_once_per_process{"Surface/AnItemIsRecordedOncePerProcess", [] { + const auto file = std::filesystem::temp_directory_path() + / std::format("mcpp-surface-{}.txt", std::random_device{}()); + std::filesystem::remove(file); + sf::append_once(file, {"key", "build.cache"}); + sf::append_once(file, {"reason", "unknown-target"}); + sf::append_once(file, {"key", "build.cache"}); + std::ifstream is(file); + const std::string text{std::istreambuf_iterator(is), {}}; + std::filesystem::remove(file); + expect(eq(sf::parse(text).size(), std::size_t{2})); +}}; diff --git a/src/build/refusal.cppm b/src/build/refusal.cppm index 816db83b3..c25e76655 100644 --- a/src/build/refusal.cppm +++ b/src/build/refusal.cppm @@ -33,6 +33,7 @@ export module mcpp.build.refusal; import std; import mcpp.effects; +import mcpp.surface; export namespace mcpp::build::refusal { @@ -280,6 +281,10 @@ constexpr std::string_view name(Code c) { void record(Code c); Code take(); +// Every code a refusal site can record, `other` included and `none` not: the reasons of +// `mcpp emit surface` (design 2026-11-20 §5.16). +std::vector codes(); + } // namespace mcpp::build::refusal // ── implementation ────────────────────────────────────────────────────────── @@ -296,6 +301,7 @@ thread_local Code g_last = Code::None; void record(Code c) { g_last = c; + mcpp::surface::exercised(mcpp::surface::kReason, name(c)); // Needing a download offline is the run stopping before a denied effect: // `--offline` is `--deny network`, and the process exits 5 (design // 2026-10-26 D23). @@ -305,4 +311,11 @@ void record(Code c) { Code take() { Code c = g_last; g_last = Code::None; return c; } +std::vector codes() { + std::vector out; + for (int i = static_cast(Code::None) + 1; i <= static_cast(Code::Other); ++i) + out.push_back(static_cast(i)); + return out; +} + } // namespace mcpp::build::refusal diff --git a/src/cli.cppm b/src/cli.cppm index 148eb8b2c..2ee75f9f2 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -29,6 +29,7 @@ import mcpp.pm.commands; import mcpp.toolchain.fingerprint; // MCPP_VERSION import mcpp.wire; import mcpp.cli.cmd_sbom; +import mcpp.cli.cmd_surface; import mcpplibs.os.platform; import mcpplibs.os.fs; import mcpplibs.os.process; // the compiler's use, measured by the executor @@ -664,6 +665,8 @@ int run(int argc, char** argv) { char** trimmed_argp = trimmed_argv.empty() ? nullptr : trimmed_argv.data(); // ─── Build the top-level App ──────────────────────────────────────── + // `emit surface` lists the definition it is part of; it is set once the definition exists. + const cl::App* self = nullptr; auto app = cl::App("mcpp") .version(std::string{mcpp::toolchain::MCPP_VERSION}) .description("modern C++ build tool") @@ -1135,7 +1138,7 @@ int run(int argc, char** argv) { // ─── emit (one nested subcommand: xpkg) ──────────────────────── .subcommand(cl::App("emit") - .description("Generate a document describing this project (xpkg, sbom, build-database)") + .description("Generate a document describing this project (xpkg, sbom, build-database) or mcpp (surface)") .subcommand(cl::App("xpkg") .description("Generate xpkg Lua entry") .option(cl::Option("version").short_name('V').takes_value().value_name("VER") @@ -1203,7 +1206,14 @@ int run(int argc, char** argv) { .help("Describe all workspace members in one document")) .option(cl::Option("exclude").takes_value().multiple().value_name("NAME") .help("With --workspace (or at a virtual workspace root), leave the named member out; repeatable, refused with -p")) - .action(mcpp::cli::cmd_emit_build_database))) + .action(mcpp::cli::cmd_emit_build_database)) + // The surface is the product's, not a project's; it is a document mcpp writes about + // itself, and `emit` already means "write a document" (design 2026-11-20 §5.16). + .subcommand(cl::App("surface") + .description("List mcpp's commands, flags, manifest keys and reason tokens, one per line") + .option(cl::Option("output").short_name('o').takes_value().value_name("FILE") + .help("Write to file instead of stdout")) + .action([&self](const cl::ParsedArgs& p) { return mcpp::cli::cmd_emit_surface(*self, p); }))) // ─── xpkg (descriptor tooling: parse) ────────────────────────── .subcommand(cl::App("xpkg") @@ -1532,6 +1542,7 @@ int run(int argc, char** argv) { .option(cl::Option("object").takes_value().value_name("PATH")) .action(cmd_bmi_await)) ; + self = &app; // The bareword `mcpp help` and `mcpp` (no args) both print the // canonical help screen. @@ -2011,6 +2022,7 @@ int run(int argc, char** argv) { mcpp::ui::error(parsed.error().message); return 2; } + mcpp::cli::record_invocation(app, *parsed); const int action_rc = app.run(*parsed); // The run's closing notices come last, after everything the command // printed, whatever its exit status (see mcpp::ui::add_closing_notice). diff --git a/src/cli/cmd_surface.cppm b/src/cli/cmd_surface.cppm new file mode 100644 index 000000000..2247733b9 --- /dev/null +++ b/src/cli/cmd_surface.cppm @@ -0,0 +1,139 @@ +// mcpp.cli.cmd_surface — mcpp's surface, and the part of it one invocation used +// (SPEC-017 P5.6 `exercise`, design 2026-11-20 §5.16). +// +// The surface is written out, never written down: the commands and flags come from the definition +// the command line is parsed with, the keys from the manifest's key registry, the reasons from the +// refusal codes. A ledger that compares the surface with what the tests used therefore cannot +// drift from the product the way a hand-kept list does. +// +// The options the parser adds itself (`--help` on every command, `--version` at the top) are +// mcpplibs.cmdline's, not mcpp's, and are not items. + +export module mcpp.cli.cmd_surface; + +import std; +import mcpplibs.cmdline; +import mcpp.manifest.key_registry; +import mcpp.build.refusal; +import mcpp.surface; +import mcpp.ui; + +export namespace mcpp::cli { + +// Each item of the surface, commands first, then flags, keys and reasons, each sorted by name. +std::vector surface_items(const mcpplibs::cmdline::App& app); + +// `mcpp emit surface [-o FILE]`. +int cmd_emit_surface(const mcpplibs::cmdline::App& app, const mcpplibs::cmdline::ParsedArgs& parsed); + +// The command and the flags `parsed` names, recorded as used (MCPP_SURFACE_RECORD). +void record_invocation(const mcpplibs::cmdline::App& app, const mcpplibs::cmdline::ParsedArgs& parsed); + +} // namespace mcpp::cli + +namespace mcpp::cli { +namespace { + +namespace cl = mcpplibs::cmdline; +namespace sf = mcpp::surface; +using Node = cl::completions::Command; + +bool parser_added(std::string_view path, std::string_view option) { + return option == "help" || (path.empty() && option == "version"); +} + +std::string child_path(std::string_view path, std::string_view name) { + return path.empty() ? std::string(name) : std::format("{} {}", path, name); +} + +void walk(const Node& node, const std::string& path, std::vector& out) { + if (!path.empty()) out.push_back({std::string(sf::kCommand), path}); + for (auto const& o : node.options) + if (!parser_added(path, o.long_name)) + out.push_back({std::string(sf::kFlag), sf::flag_name(path, o.long_name)}); + for (auto const& sub : node.subcommands) walk(sub, child_path(path, sub.name), out); +} + +const Node* find_sub(const Node& node, std::string_view name) { + for (auto const& sub : node.subcommands) + if (sub.name == name) return ⊂ + return nullptr; +} + +bool declares(const Node& node, std::string_view option) { + return std::ranges::any_of(node.options, [&](auto const& o) { return o.long_name == option; }); +} + +using Chain = std::vector>; + +// The command, at `depth` or above it, whose definition declares `option`. +const std::string* owner(const Chain& chain, std::size_t depth, std::string_view option) { + for (std::size_t at = depth + 1; at > 0; --at) + if (declares(*chain[at - 1].first, option)) return &chain[at - 1].second; + return nullptr; +} + +} // namespace + +std::vector surface_items(const cl::App& app) { + std::vector out; + walk(cl::snapshot(app), "", out); + for (auto const& row : mcpp::manifest::kKeyRegistry) + out.push_back({std::string(sf::kKey), row.key.empty() + ? std::string(row.table) : std::format("{}.{}", row.table, row.key)}); + for (auto c : mcpp::build::refusal::codes()) + out.push_back({std::string(sf::kReason), std::string(mcpp::build::refusal::name(c))}); + constexpr std::array kOrder{sf::kCommand, sf::kFlag, sf::kKey, sf::kReason}; + auto rank = [&](const sf::Item& i) { return std::ranges::find(kOrder, i.kind) - kOrder.begin(); }; + std::ranges::sort(out, [&](auto const& a, auto const& b) { + return std::pair(rank(a), std::string_view(a.name)) < std::pair(rank(b), std::string_view(b.name)); + }); + out.erase(std::unique(out.begin(), out.end()), out.end()); + return out; +} + +int cmd_emit_surface(const cl::App& app, const cl::ParsedArgs& parsed) { + std::string text; + const auto items = surface_items(app); + for (auto const& i : items) text += sf::line(i); + const auto o = parsed.value("output"); + if (!o) { + std::print("{}", text); + return 0; + } + std::error_code ec; + std::filesystem::create_directories(std::filesystem::path(*o).parent_path(), ec); + std::ofstream f(*o, std::ios::binary); + if (!f) { + mcpp::ui::error(std::format("cannot write {}", *o)); + return 2; + } + f << text; + mcpp::ui::info("surface", std::format("wrote {} ({} items)", *o, items.size())); + return 0; +} + +void record_invocation(const cl::App& app, const cl::ParsedArgs& parsed) { + if (std::getenv(std::string(sf::kRecordEnv).c_str()) == nullptr) return; + // The parser keeps a global option at the level it was dispatched from, so a flag is the item + // of the nearest command, from its level up, whose definition declares it. + const Node root = cl::snapshot(app); + Chain chain{{&root, ""}}; + std::vector levels{&parsed}; + for (auto* p = &parsed; p->subcommand_matches; p = p->subcommand_matches.get()) { + const Node* sub = find_sub(*chain.back().first, p->subcommand_name_); + if (sub == nullptr) break; + chain.emplace_back(sub, child_path(chain.back().second, sub->name)); + levels.push_back(p->subcommand_matches.get()); + sf::exercised(sf::kCommand, chain.back().second); + } + for (std::size_t d = 0; d < levels.size(); ++d) { + for (auto const& [name, value] : levels[d]->opts) { + const auto* path = value.is_set() ? owner(chain, d, name) : nullptr; + if (path != nullptr && !parser_added(*path, name)) + sf::exercised(sf::kFlag, sf::flag_name(*path, name)); + } + } +} + +} // namespace mcpp::cli diff --git a/tests/e2e/cli/commands.cpp b/tests/e2e/cli/commands.cpp index eab68aa6d..72c3f64c1 100644 --- a/tests/e2e/cli/commands.cpp +++ b/tests/e2e/cli/commands.cpp @@ -299,6 +299,46 @@ xpm = { linux = { ["1.0.0"] = { url = "https://example.invalid/w.tar.gz", sha256 expect(fatal(s::has_line(s::text(r), "^error: --toolchain = 'nosuch@1\\.0'"))) << "the refusal does not name --toolchain as where the value was written\n" << s::text(r); expect(fatal(s::lacks(s::text(r), "[toolchain]."))) << "the refusal credits a manifest key with a command-line value"; }; + + "1119 the surface is listed from the product and an invocation records what it used"_test = [] { + // Design 2026-11-20 §5.16, SPEC-017 P5.6: `mcpp emit surface` lists each command and flag of + // the command line's definition, each key of the manifest's registry and each reason token, + // one ` ` per line; an invocation run with MCPP_SURFACE_RECORD appends the items + // it used, each of them an item of the surface. + s::project p{{"mcpp.toml", "[package]\nname = \"p\"\nversion = \"0.1.0\"\n\n[build]\ncflags = [\"-O1\"]\n"}, + {"src/main.cpp", "int main() { return 0; }\n"}}; + auto r = p.mcpp({"emit", "surface"}); + expect(fatal(e2e::succeeded(r))) << s::text(r); + std::set surface; + std::vector kinds; + for (auto part : std::views::split(std::string_view(r.out), '\n')) { + const std::string line(part.begin(), part.end()); + if (line.empty()) continue; + expect(line.find(' ') != std::string::npos) << "a line without a kind and a name: " << line; + surface.insert(line); + const auto kind = line.substr(0, line.find(' ')); + if (kinds.empty() || kinds.back() != kind) kinds.push_back(kind); + } + expect(kinds == std::vector{"command", "flag", "key", "reason"}) << "the kinds in order, each once"; + for (auto item : {"command build", "command emit surface", "flag build --target", "flag --quiet", + "key build.cflags", "key toolchain", "reason unknown-target", "reason other"}) + expect(surface.contains(item)) << "the surface lacks " << item; + for (auto item : {"flag --help", "flag build --help", "flag --version", "reason none"}) + expect(!surface.contains(item)) << "the surface lists " << item << ", which is not mcpp's"; + + const auto record = s::case_dir() / "surface-record.txt"; + auto b = p.mcpp({"build", "--target", "nosuch-unknown-triple"}, + {.env = {{"MCPP_SURFACE_RECORD", record.string()}}}); + expect(fatal(failed(b))) << "an unknown target was built"; + const auto used = s::read_file(record); + for (auto item : {"command build", "flag build --target", "key package", "key build.cflags", + "reason unknown-target"}) + expect(used.contains(std::string(item) + "\n")) << "the record lacks " << item << "\n" << used; + for (auto part : std::views::split(std::string_view(used), '\n')) { + const std::string line(part.begin(), part.end()); + if (!line.empty()) expect(surface.contains(line)) << "a recorded item is not on the surface: " << line; + } + }; }}; } // namespace From 41e79b4aac93639ebd01ff10d483eeb8db40fb3c Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 05:01:53 +0900 Subject: [PATCH 17/38] =?UTF-8?q?mcpp=20test=20serves=20run=20fixtures=20(?= =?UTF-8?q?SPEC-017=20P11,=20design=202026-11-20=20=C2=A75.10)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A program that declares a fixture (a component event with fixture) serves it under MTEST_SERVE before the first program whose cases use it; each user gets its endpoint as MTEST_FIXTURE_; the last user's end closes its input. fixtures.ndjson states each fixture's preparation; programs.ndjson names the fixtures a program used; a fixture that is not ready leaves its users not run, 'infra' in the store (P11.4). --- src/build/execute.cppm | 98 +++++++++++++++- src/build/test_fixture.cppm | 217 +++++++++++++++++++++++++++++++++++ src/build/test_protocol.cppm | 7 +- tests/e2e/test/protocol.cpp | 80 +++++++++++++ 4 files changed, 396 insertions(+), 6 deletions(-) create mode 100644 src/build/test_fixture.cppm diff --git a/src/build/execute.cppm b/src/build/execute.cppm index 69e3a9ba1..2d0cdef3d 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -16,6 +16,7 @@ import mcpp.pack.compress; // the test archive's tar (SPEC-015 §4) import mcpp.pack.distribution; // its writer and reader import mcpp.build.test_targets; import mcpp.build.test_protocol; // the runner's side of the test protocol (docs/08 "Cases") +import mcpp.build.test_fixture; // the run fixtures it serves (SPEC-017 P11) import mcpp.build.test_impact; // the tests a change affects (docs/08) import mcpp.build.test_schedule; // the order, division and shards of a run (docs/08) import mcpp.diag; @@ -4619,6 +4620,10 @@ struct TestResult { // (seconds): a case's `limit_ms` in the store (B4). double limitSecs = 0; std::map caseSecs; + // The run fixtures it used (P11.2), and whether one it uses was not ready: the machine's + // failure, `infra` in the store (P11.4). + std::vector fixtures; + bool infra = false; // The program and the files it is compiled from, for a test built and not // run (`--no-run`): what a runner that selects by change reads (docs/50). std::string program; @@ -4656,6 +4661,7 @@ static double case_limit(const TestResult& t, std::string_view id) { // `mtest report|analyze|graph --store ` read `mcpp test`'s runs as the // engine's own. Written whole into `runs//` and then appended to the index. void write_mtest_store(const std::filesystem::path& dir, const std::vector& results, + const std::vector& fixtures, std::chrono::system_clock::time_point started, long long ms, int jobs, std::string_view selection, std::string_view shard = {}) { namespace fs = std::filesystem; @@ -4676,7 +4682,7 @@ void write_mtest_store(const std::filesystem::path& dir, const std::vector 0) p["build_ms"] = t.buildMs; // The program a part is of: what `--failed` builds again. if (!t.partOf.empty()) p["part_of"] = t.partOf; @@ -5071,8 +5085,30 @@ struct TestRunnable { // limit a case is held to in the store (B4 `limit_ms`), for the report's // margin (design 2026-11-20 §5.8). std::map caseSecs; + // P11: the run fixtures each listed case uses, and those the program serves. + std::map> caseUses; + std::vector serves; }; +// The run fixtures the cases a program selects use (P11.2): a listed case is selected by a line +// naming it or a case under it, plain or `?`; a `!` line selects nothing. +static std::vector fixtures_used(const TestRunnable& r) { + std::vector lines; + if (r.select) { + std::ifstream in(*r.select); + for (std::string line; std::getline(in, line);) + if (!line.empty() && !line.starts_with('!')) lines.push_back(line.starts_with('?') ? line.substr(1) : line); + } + std::set out; + for (auto const& [id, uses] : r.caseUses) { + const bool selected = !r.select || std::ranges::any_of(lines, [&](std::string_view l) { + return l == id || (l.size() > id.size() && l.starts_with(id) && l[id.size()] == '/'); + }); + if (selected) out.insert(uses.begin(), uses.end()); + } + return {out.begin(), out.end()}; +} + // THE PROTOCOL A TEST PROGRAM IMPLEMENTS (P2.3): what its entry states, else // the marker its file carries. A test that keeps what it reaches may have lost // a marker nothing references (a framework before mtest 0.5.1) at the link; @@ -5587,8 +5623,16 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, const bool exclusiveDevice = (ctx->manifest.buildConfig.runExclusive || ctx->plan.platform.fields.runner.exclusive) && !runnerChoice.tmpl.empty(); + std::vector fixtureRuns; auto run_tests_now = [&](std::vector& list) { if (list.empty()) return; + // THE RUN FIXTURES (SPEC-017 P11, design 2026-11-20 §5.10): the program that serves each, + // taken before the shard keeps its own, since a shard's users need not include it. + namespace tf = mcpp::build::test_fixture; + std::map providers; + for (auto const& r : list) + for (auto const& f : r.serves) + providers.try_emplace(f, tf::Provider{r.partOf.empty() ? r.name : r.partOf, r.argv, r.env, r.cwd}); // Each test's measured duration: of the cases it selects, or of the // program whole. namespace ts = mcpp::build::test_schedule; @@ -5623,6 +5667,18 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, } // The longest first, so that the longest work does not start last. std::ranges::stable_sort(list, std::greater{}, &Runnable::estimateMs); + // Each fixture starts before its first user and ends after its last; a run whose cases + // use none starts nothing. + std::vector> usesOf(list.size()); + std::map users; + for (std::size_t i = 0; i < list.size(); ++i) { + usesOf[i] = fixtures_used(list[i]); + for (auto const& f : usesOf[i]) ++users[f]; + } + std::optional fixtures; + if (!users.empty()) + fixtures.emplace(std::move(providers), std::move(users), tLoop, + std::chrono::seconds(testOpts.timeoutSecs > 0 ? testOpts.timeoutSecs : 3600)); const bool capture = json || list.size() > 1; // The limit of one test's process: `--timeout`, or what its cases // state when that is longer; `--timeout 0` disables both. @@ -5784,6 +5840,29 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, } else { for (auto v : mcpp::build::test_protocol::kVariables) runOpts.unset.emplace_back(v); } + // A fixture that is not ready is the machine's failure: the program does not run + // and its record says `infra` (P11.4). + std::string notReady; + for (auto const& f : usesOf[i]) { + auto endpoint = fixtures->acquire(f); + if (!endpoint) { notReady = endpoint.error(); break; } + runOpts.env.emplace_back(tf::variable(f), *endpoint); + } + const auto release_fixtures = [&] { + for (auto const& f : usesOf[i]) fixtures->release(f); + }; + if (!notReady.empty()) { + release_fixtures(); + std::scoped_lock lock(reportMutex); + if (!json) mcpp::ui::plain(std::format("{} ... not run ({})", r.name, notReady)); + results.push_back({r.name, TestResult::St::NotRun, 0, {}, {}, 0, false, notReady, r.buildMs}); + results.back().infra = true; + results.back().fixtures = usesOf[i]; + stamp_schedule(); + std::fflush(libc::stdout()); + emit_json(results.back()); + continue; + } if (!capture) mcpp::ui::status("Running", std::format("bin/{}", r.name)); // THE CASES AS THEY HAPPEN (feature `test-case-stream`): the // report is NDJSON the program appends event by event (P5.1), @@ -5828,6 +5907,7 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, }); auto rr = mcpp::run::execute(r.argv, runOpts); if (watcher.joinable()) { watcher.request_stop(); watcher.join(); } + release_fixtures(); exitCode = rr.exit_code; timedOut = rr.timed_out; spawnErr = rr.spawn_error; @@ -5917,6 +5997,7 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, } results.back().protocol = spoken; results.back().report = std::move(report); + results.back().fixtures = usesOf[i]; stamp_schedule(); results.back().guide = r.guide; // The captured output belongs directly under its own line, or @@ -5938,6 +6019,7 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, for (int w = 0; w < workers; ++w) pool.emplace_back(worker, w); for (auto& t : pool) t.join(); } + if (fixtures) std::ranges::move(fixtures->finish(), std::back_inserter(fixtureRuns)); // WALL time of the phase, not the sum of the per-test durations: with // N running at once that sum exceeds the elapsed time and the summary // would report a run phase longer than the whole command. @@ -6382,8 +6464,11 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, } for (const auto& [_, secs] : selectedSecs) r.declaredSecs += secs; selectedSecs.clear(); - for (const auto& c : report.listed) + for (const auto& c : report.listed) { if (c.timeout > 0) r.caseSecs[c.id] = c.timeout; + if (!c.uses.empty()) r.caseUses[c.id] = c.uses; + } + r.serves = report.fixtures; if (lines.empty()) { ++notSelected; continue; } r.select = r.protocolDir / "select.txt"; std::ofstream(*r.select, std::ios::binary) << lines; @@ -6473,8 +6558,11 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, continue; } const auto& report = *listings[at]; - for (const auto& c : report.listed) + for (const auto& c : report.listed) { if (c.timeout > 0) r.caseSecs[c.id] = c.timeout; + if (!c.uses.empty()) r.caseUses[c.id] = c.uses; + } + r.serves = report.fixtures; // THE MACHINE, HELD ALONE (P7.1). A case that holds `machine` // states a property that holds only while nothing beside it // changes the machine (the shared home, its caches, the default @@ -6607,7 +6695,7 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, // program, and not a listing. if (const auto& st = ctx->manifest.testStore; st != "none" && !testOpts.listCases && std::ranges::any_of(results, [](const TestResult& r) { return r.status != TestResult::St::Built; })) - write_mtest_store(storeDir, results, + write_mtest_store(storeDir, results, fixtureRuns, testStartedWall, std::chrono::duration_cast( std::chrono::steady_clock::now() - tLoop).count(), diff --git a/src/build/test_fixture.cppm b/src/build/test_fixture.cppm new file mode 100644 index 000000000..7b7d279b8 --- /dev/null +++ b/src/build/test_fixture.cppm @@ -0,0 +1,217 @@ +// mcpp.build.test_fixture — the run fixtures `mcpp test` serves (SPEC-017 O6, P11; design +// 2026-11-20 §5.10). +// +// A run fixture is one object of the whole run: a sandboxed home, a server, an index. A program +// declares it (a `component` event with `fixture`), its cases name it in `uses`. The runner starts +// a program that declares it, with MTEST_SERVE naming it and its input a pipe, before the first +// program whose cases use it; reads `mtest-ready ` from its output; gives the endpoint to +// each user as MTEST_FIXTURE_; and closes the input after the last user ended, which ends the +// fixture. Its preparation is recorded on the fixture (`fixtures.ndjson`), not on whichever case +// happened to run first. + +export module mcpp.build.test_fixture; + +import std; +import mcpplibs.os.process; +import mcpp.build.test_protocol; + +export namespace mcpp::build::test_fixture { + +// How a program is started: what the runner starts for its cases, without the protocol's variables. +struct Provider { + std::string program; // its name in the run + std::vector argv; + std::vector> env; + std::filesystem::path cwd; +}; + +// One fixture of the run, as `fixtures.ndjson` states it: milliseconds after the run began. +struct Run { + std::string name; + std::string program; + std::string status; // ready, failed, or unused when no user acquired it + double startedMs = 0; + double readyMs = 0; + double endedMs = 0; + std::string error; +}; + +// MTEST_FIXTURE_: the name upper-cased, each character not a letter or digit an underscore. +std::string variable(std::string_view name); + +class Server { +public: + // `providers`: the program that serves each fixture; `users`: how many programs of the run use + // each, so that the last to release it ends it. `limit`: how long a fixture may take to be ready. + Server(std::map providers, std::map users, + std::chrono::steady_clock::time_point origin, std::chrono::milliseconds limit); + ~Server(); + Server(const Server&) = delete; + Server& operator=(const Server&) = delete; + + // The endpoint of `name`, started on the first call; a caller that arrives while another starts + // it waits for the outcome. The error names the fixture and why it is not ready (P11.4). + std::expected acquire(const std::string& name); + // One user of `name` ended; the last ends the fixture. + void release(const std::string& name); + // Ends every fixture still served and states each. + std::vector finish(); + +private: + struct Served { + bool started = false; + bool settled = false; + Run run; + std::string endpoint; + std::optional process; + }; + double now() const; + void start(const Provider& p, Served& s); + void end(Served& s); + + std::map providers_; + std::map users_; + std::chrono::steady_clock::time_point origin_; + std::chrono::milliseconds limit_; + std::mutex m_; + std::condition_variable cv_; + std::map served_; +}; + +} // namespace mcpp::build::test_fixture + +namespace mcpp::build::test_fixture { + +namespace proc = mcpplibs::os::process; + +std::string variable(std::string_view name) { + std::string key = "MTEST_FIXTURE_"; + for (char c : name) + key += std::isalnum(static_cast(c)) ? char(std::toupper(static_cast(c))) : '_'; + return key; +} + +Server::Server(std::map providers, std::map users, + std::chrono::steady_clock::time_point origin, std::chrono::milliseconds limit) + : providers_(std::move(providers)), users_(std::move(users)), origin_(origin), limit_(limit) {} + +Server::~Server() { (void)finish(); } + +double Server::now() const { + return std::chrono::duration(std::chrono::steady_clock::now() - origin_).count(); +} + +std::expected Server::acquire(const std::string& name) { + std::unique_lock lock(m_); + auto& s = served_[name]; + if (!s.started) { + s.started = true; + s.run.name = name; + auto p = providers_.find(name); + if (p == providers_.end()) { + s.run.status = "failed"; + s.run.error = std::format("no program of the run declares the run fixture {}", name); + s.settled = true; + } else { + s.run.program = p->second.program; + // Started outside the lock: a fixture that takes minutes holds back no other fixture. + lock.unlock(); + Served fresh; + fresh.run = s.run; + start(p->second, fresh); + lock.lock(); + auto& at = served_[name]; + at.run = fresh.run; + at.endpoint = std::move(fresh.endpoint); + at.process = std::move(fresh.process); + at.settled = true; + } + cv_.notify_all(); + } + cv_.wait(lock, [&] { return served_[name].settled; }); + auto const& done = served_[name]; + if (done.run.status != "ready") return std::unexpected(done.run.error); + return done.endpoint; +} + +void Server::start(const Provider& p, Served& s) { + s.run.startedMs = now(); + proc::spawn_options o; + o.program = p.argv.front(); + o.args.assign(p.argv.begin() + 1, p.argv.end()); + o.cwd = p.cwd; + o.in = proc::stdio::pipe; + o.out = proc::stdio::pipe; + o.err = proc::stdio::inherit; + // Its own unit, which `end` ends. Not `bound_lifetime`: on Linux that binds the fixture to the + // thread that started it, a worker, which ends before the fixture's last user does (measured). + o.own_unit = true; + for (auto const& [k, v] : p.env) o.env.emplace_back(k, v); + // The program serves; it runs no case (P11.3), whatever this runner's own environment holds. + for (auto v : mcpp::build::test_protocol::kVariables) o.env.emplace_back(std::string(v), std::nullopt); + o.env.emplace_back("MTEST_SERVE", s.run.name); + auto spawned = proc::process::spawn(o); + if (!spawned) { + s.run.status = "failed"; + s.run.error = std::format("the run fixture {} did not start: {}", s.run.name, spawned.error().message()); + return; + } + const auto deadline = std::chrono::steady_clock::now() + limit_; + std::string text; + for (;;) { + const auto left = std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + if (left.count() <= 0) break; + auto chunk = spawned->read_output_for(left); + if (!chunk || !*chunk || (*chunk)->empty()) break; + text += **chunk; + for (auto nl = text.find('\n'); nl != std::string::npos; nl = text.find('\n')) { + std::string line = text.substr(0, nl); + text.erase(0, nl + 1); + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (!line.starts_with("mtest-ready ")) continue; + s.endpoint = line.substr(12); + s.run.status = "ready"; + s.run.readyMs = now(); + s.process = std::move(*spawned); + return; + } + } + spawned->kill(); + (void)spawned->wait(); + s.run.status = "failed"; + s.run.endedMs = now(); + s.run.error = std::chrono::steady_clock::now() >= deadline + ? std::format("the run fixture {} was not ready within {} s", s.run.name, limit_.count() / 1000) + : std::format("the run fixture {} ended before it was ready", s.run.name); +} + +void Server::end(Served& s) { + if (!s.process) return; + s.process->close_input(); + // A fixture that does not end when its input ends is ended, after the time it had to be ready. + if (auto ended = s.process->wait_for(limit_); !ended || !*ended) { + s.process->kill(); + (void)s.process->wait(); + } + s.process->end_unit(); + s.process.reset(); + s.run.endedMs = now(); +} + +void Server::release(const std::string& name) { + std::lock_guard lock(m_); + if (--users_[name] > 0) return; + if (auto it = served_.find(name); it != served_.end()) end(it->second); +} + +std::vector Server::finish() { + std::lock_guard lock(m_); + std::vector out; + for (auto& [name, s] : served_) { + end(s); + out.push_back(s.run); + } + return out; +} + +} // namespace mcpp::build::test_fixture diff --git a/src/build/test_protocol.cppm b/src/build/test_protocol.cppm index 8e71f1ad8..23a370f0b 100644 --- a/src/build/test_protocol.cppm +++ b/src/build/test_protocol.cppm @@ -77,6 +77,7 @@ struct Case { std::vector bounds; // P6.5 std::vector resources; // P7.1: `machine` holds the machine alone double timeout = 0; // seconds the case states it may run; 0: none stated + std::vector uses; // P11.2: the run fixtures it uses }; // A step of a case (P5.6): a command an end-to-end case ran, as the program @@ -151,6 +152,7 @@ struct Report { bool selectsWithin = false; bool done = false; // the `done` event came std::vector listed; // `case` events (list mode) + std::vector fixtures; // P11.1: the run fixtures it serves (`component` events) std::vector cases; // in the order they began std::vector missing; // P4.1 Usage usage; // the `usage` event (P10.2) @@ -332,7 +334,10 @@ take(Report& r, std::map& open, const nlohmann::json& r.listed.push_back({id, text(e, "name"), text(e, "file"), number(e, "line"), text(e, "kind"), strings(e, "tags"), text(e, "expect"), bounds_of(e), strings(e, "resources"), - e.contains("timeout") && e["timeout"].is_number() ? e["timeout"].get() : 0.0}); + e.contains("timeout") && e["timeout"].is_number() ? e["timeout"].get() : 0.0, + strings(e, "uses")}); + } else if (kind == "component") { + if (auto f = text(e, "fixture"); !f.empty()) r.fixtures.push_back(f); } else if (kind == "begin") { open[id] = r.cases.size(); r.cases.push_back({id, "", 0, "", {}}); diff --git a/tests/e2e/test/protocol.cpp b/tests/e2e/test/protocol.cpp index d3d6d6077..1ed728f2e 100644 --- a/tests/e2e/test/protocol.cpp +++ b/tests/e2e/test/protocol.cpp @@ -372,6 +372,86 @@ bool cases(std::ofstream& r, bool list, const std::set& selected) { expect(fatal(e2e::succeeded(one))); expect(e2e::contains(record(one.out, "t")["cases"][0].value("output", ""), "written by plain")); }; + + "293 a run fixture is served once, before its first user and until its last"_test[s::gcc] = [] { + // SPEC-017 P11 (design 2026-11-20 §5.10): the program that declares the fixture serves it + // under MTEST_SERVE; both programs whose cases use it receive its endpoint; it ends after + // them; its preparation is its own row. A fixture that is not ready is the machine's + // failure: its users do not run, and the store says `infra` (P11.4). + const auto root = s::case_dir() / "t"; + s::project t(root, {{"fw/mcpp.toml", R"([package] +name = "fw" +version = "0.1.0" + +[targets.main] +kind = "entry" +main = "src/main.cpp" +test-protocol = 1 +test-main = true +)"}, + {"fw/src/main.cpp", std::format(R"CPP(import std; +extern std::string case_id(); +int main() {{ + const std::filesystem::path root{{"{}"}}; + if (const char* serve = std::getenv("MTEST_SERVE")) {{ + if (std::getenv("MTEST_PROTOCOL") || std::getenv("MTEST_REPORT")) return 5; + std::ofstream(root / "served.txt", std::ios::app) << serve << "\n"; + if (std::filesystem::exists(root / "not-ready")) return 3; + std::filesystem::create_directories(root / "home"); + std::cout << "mtest-ready " << (root / "home").generic_string() << std::endl; + for (std::string line; std::getline(std::cin, line);) {{}} + std::ofstream(root / "ended.txt") << "ended\n"; + return 0; + }} + const char* path = std::getenv("MTEST_REPORT"); + if (!path) return 4; + std::ofstream r(path, std::ios::app); + const auto id = case_id(); + r << R"({{"e":"program","protocol":1,"level":1,"framework":"fw","version":"0.1.0"}})" << "\n"; + if (std::string_view(std::getenv("MTEST_MODE")) == "list") {{ + r << R"({{"e":"component","fixture":"home","scope":"run"}})" << "\n"; + r << std::format(R"({{{{"e":"case","id":"{{}}","name":"{{}}","file":"{{}}.cpp","line":1,"uses":["home"]}}}})", id, id, id) << "\n"; + }} else {{ + r << std::format(R"({{{{"e":"begin","id":"{{}}","t":0}}}})", id) << "\n"; + const char* home = std::getenv("MTEST_FIXTURE_HOME"); + const bool ok = home && std::filesystem::path(home) == root / "home" + && !std::filesystem::exists(root / "ended.txt"); + if (ok) std::ofstream(root / "home" / "users.txt", std::ios::app) << id << "\n"; + r << std::format(R"({{{{"e":"end","id":"{{}}","status":"{{}}","ms":0}}}})", id, ok ? "pass" : "fail") << "\n"; + }} + r << R"({{"e":"done","status":"pass","counts":{{}}}})" << "\n"; + return 0; +}} +)CPP", generic(root))}, + {"app/tests/a.cpp", "import std;\nstd::string case_id() { return \"a\"; }\n"}, + {"app/tests/b.cpp", "import std;\nstd::string case_id() { return \"b\"; }\n"}}); + t.write("app/mcpp.toml", std::format("[package]\nname = \"app\"\nversion = \"0.1.0\"\n\n[dev-dependencies]\nfw = {{ path = \"{}\" }}\n", + generic(t.at("fw")))); + auto r = t.mcpp_in("app", {"test", "--message-format", "json"}); + expect(fatal(e2e::succeeded(r))) << s::text(r); + expect(eq(s::read_file(root / "served.txt"), std::string("home\n"))) << "the fixture is served once"; + expect(fs::exists(root / "ended.txt")) << "the fixture ended after its last user"; + const auto users = s::read_file(root / "home" / "users.txt"); + expect(users.contains("a\n") && users.contains("b\n")) << users; + const auto fixtures = s::glob(t.at("app/target/mtest/runs"), "*/fixtures.ndjson"); + expect(fatal(eq(fixtures.size(), std::size_t{1}))); + const auto rows = s::json_lines(s::read_file(fixtures.front())); + expect(fatal(eq(rows.size(), std::size_t{1}))); + expect(rows[0]["name"] == "home" && rows[0]["status"] == "ready" + && rows[0]["ready_ms"].get() >= rows[0]["started_ms"].get() + && rows[0]["ended_ms"].get() >= rows[0]["ready_ms"].get()) << rows[0].dump(); + // P11.4 + std::ofstream(root / "not-ready").put('\n'); + fs::remove(root / "ended.txt"); + auto nr = t.mcpp_in("app", {"test"}); + expect(nr.started && nr.status != 0) << "a run whose fixture was not ready passed"; + expect(e2e::contains(s::text(nr), "the run fixture home ended before it was ready")) << s::text(nr); + bool infra = false; + for (const auto& f : s::glob(t.at("app/target/mtest/runs"), "*/programs.ndjson")) + for (const auto& p : s::json_lines(s::read_file(f))) + infra = infra || (p.value("status", "") == "infra" && p["fixtures"] == json::parse(R"(["home"])")); + expect(infra) << "no program's record says infra"; + }; }}; } // namespace From e71fa6da6ab41b16abcb16fa0e896af2936bf0d6 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 05:14:11 +0900 Subject: [PATCH 18/38] =?UTF-8?q?--since:=20a=20header=20affects=20the=20t?= =?UTF-8?q?ests=20built=20from=20the=20units=20that=20read=20it=20(design?= =?UTF-8?q?=202026-11-20=20=C2=A75.11)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The build directory's records name the files each unit read: ninja's deps log (format 4, read directly) and the scans' depfiles. When every unit of the project has a record, a changed header selects the tests built from a unit that read it, and the tests that run a program of the package when such a unit is not a test's own; otherwise every test is affected, as before. --- src/build/test_impact.cppm | 113 ++++++++++++++++++++++++++++++++++-- tests/e2e/test/affected.cpp | 27 +++++++++ 2 files changed, 135 insertions(+), 5 deletions(-) diff --git a/src/build/test_impact.cppm b/src/build/test_impact.cppm index 2f42a7cb7..70bcf9880 100644 --- a/src/build/test_impact.cppm +++ b/src/build/test_impact.cppm @@ -5,14 +5,18 @@ // module unit -- and by a change of what decides how everything is built // (a manifest, a lock, a build program, a header). A test that runs a program // of the package (a file it is built from names `MCPP_BIN_`) is affected by a -// change of any source of the package that is not a test's own. Another file -// affects the tests whose own files name it. Selection by change narrows; it never adds -// a test that a pattern or `-p` left out. +// change of any source of the package that is not a test's own. A header affects +// the tests built from a unit that read it when the build directory last built it +// (ninja's deps log and the scans' depfiles, design 2026-11-20 §5.11), and every +// test when the directory holds no such record. Another file affects the tests +// whose own files name it. Selection by change narrows; it never adds a test that +// a pattern or `-p` left out. export module mcpp.build.test_impact; import std; import mcpp.build.plan; import mcpp.modgraph.graph; +import mcpp.build.unit_cache; // depfile_prerequisites import mcpplibs.os; export namespace mcpp::build::test_impact { @@ -110,6 +114,76 @@ std::set closure_of(const BuildPlan& plan, const LinkUnit return files; } +// THE FILES EACH OUTPUT READ, as ninja's deps log states them (`.ninja_deps`, format 4: a magic +// line, a version, then records whose size word's high bit tells a dependency record from a +// path record). A later record of an output replaces an earlier one, as in ninja. Nothing when +// the log is absent or of another version. +std::map> ninja_deps(const std::filesystem::path& file) { + std::map> out; + const auto bytes = read_text(file); + constexpr std::string_view kMagic = "# ninjadeps\n"; + if (!bytes.starts_with(kMagic) || bytes.size() < kMagic.size() + 4) return out; + auto word = [&](std::size_t at) { + std::uint32_t v = 0; + std::memcpy(&v, bytes.data() + at, 4); + return v; + }; + if (word(kMagic.size()) != 4) return out; + std::vector paths; + for (std::size_t at = kMagic.size() + 4; at + 4 <= bytes.size();) { + const auto head = word(at); + const std::size_t size = head & 0x7FFFFFFFu; + at += 4; + if (at + size > bytes.size() || size < 4) break; + if ((head & 0x80000000u) == 0) { + std::string path = bytes.substr(at, size - 4); + while (!path.empty() && path.back() == '\0') path.pop_back(); + paths.push_back(std::move(path)); + } else if (const auto id = word(at); id < paths.size() && size >= 12) { + auto& inputs = out[paths[id]]; + inputs.clear(); + for (std::size_t i = at + 12; i + 4 <= at + size; i += 4) + if (const auto in = word(i); in < paths.size()) inputs.push_back(paths[in]); + } + at += size; + } + return out; +} + +// The units that read each file, from the build directory's records: an object's inputs in the +// deps log, a scan's depfile (`/.ddi.dep`). `complete` when every unit +// of a file under the project's root has a record: only then does a header no record names +// affect no test. +struct Readers { + std::map> of; + bool complete = true; +}; +Readers readers_of(const BuildPlan& plan) { + Readers out; + const auto deps = ninja_deps(plan.outputDir / ".ninja_deps"); + auto add = [&](const std::filesystem::path& file, const CompileUnit* cu) { + const auto abs = (file.is_absolute() ? file : plan.outputDir / file).lexically_normal(); + auto& units = out.of[abs]; + if (units.empty() || units.back() != cu) units.push_back(cu); + }; + const auto root = plan.projectRoot.lexically_normal().generic_string() + "/"; + for (auto const& cu : plan.compileUnits) { + bool recorded = false; + if (auto it = deps.find(cu.object.generic_string()); it != deps.end()) { + recorded = true; + for (auto const& in : it->second) add(mcpplibs::os::fs::from_utf8(in), &cu); + } + const auto scan = plan.outputDir / cu.object.parent_path() / (cu.source.filename().string() + ".ddi.dep"); + if (const auto text = read_text(scan); !text.empty()) { + recorded = true; + for (auto const& in : mcpp::build::unit_cache::depfile_prerequisites(text)) + add(mcpplibs::os::fs::from_utf8(in), &cu); + } + if (!recorded && cu.source.lexically_normal().generic_string().starts_with(root)) out.complete = false; + } + return out; +} + // The test's own file that names `rel` (or its file name), if one does. std::optional naming_file(const LinkUnit& lu, const std::string& rel, const std::string& name) { for (auto const& own : lu.ownSources) { @@ -147,6 +221,8 @@ Affected affected_tests(const BuildPlan& plan, const std::vector& c if (fresh) it->second = read_text(f).find("MCPP_BIN_") != std::string::npos; return it->second; }; + // Read once, and only when a header changed. + std::optional readers; std::vector runners; std::set testsOwn; for (auto const& [lu, files] : tests) { @@ -155,16 +231,43 @@ Affected affected_tests(const BuildPlan& plan, const std::vector& c } for (auto const& rel : changed) { const auto path = (plan.projectRoot / mcpplibs::os::fs::from_utf8(rel)).lexically_normal(); - if (decides_the_build(path) || is_header(path) + if (is_header(path) && !readers) readers = readers_of(plan); + const bool unknownHeader = is_header(path) && !readers->complete; + if (decides_the_build(path) || unknownHeader || (is_source(path) && !bySource.contains(path))) { out.everything = std::format("{} {}", rel, decides_the_build(path) ? "decides how the tests are built" - : is_header(path) ? "is a header, and which units include it is not known" + : unknownHeader ? "is a header, and the build directory has no record of what some unit read" : "is a source this build does not plan"); out.notes.push_back(*out.everything); break; } std::size_t hits = 0; + if (is_header(path)) { + const auto it = readers->of.find(path); + const std::size_t units = it == readers->of.end() ? 0 : it->second.size(); + for (auto const& [lu, files] : tests) { + const bool reads = it != readers->of.end() && std::ranges::any_of(it->second, [&](const CompileUnit* cu) { + return files.contains(cu->source.lexically_normal()); + }); + if (reads) { + affect(lu->targetName, std::format("it is built from a unit that read {}", rel)); + ++hits; + } + } + // A unit of the package that is no test's own: the tests that run its programs. + const bool programReads = it != readers->of.end() && std::ranges::any_of(it->second, [&](const CompileUnit* cu) { + return !testsOwn.contains(cu->source.lexically_normal()); + }); + for (const auto* lu : runners) + if (programReads && !out.tests.contains(lu->targetName)) { + affect(lu->targetName, std::format("it runs a program of the package, which read {}", rel)); + ++hits; + } + out.notes.push_back(std::format("{}: read by {} unit{}, {} test{}", rel, units, units == 1 ? "" : "s", + hits, hits == 1 ? "" : "s")); + continue; + } for (auto const& [lu, files] : tests) if (files.contains(path)) { affect(lu->targetName, std::format("it is built from {}", rel)); diff --git a/tests/e2e/test/affected.cpp b/tests/e2e/test/affected.cpp index c881571f3..94c64b1c6 100644 --- a/tests/e2e/test/affected.cpp +++ b/tests/e2e/test/affected.cpp @@ -136,6 +136,33 @@ UTest g{"g", [] { expect(e2e::succeeded(f2)); expect(s::has_line(s::text(f2), R"(test_smoke \.\.\. ok \(.*3 cases\))")) << "a pattern below the listed case does not run it whole"; }; + + "1120 a header affects the tests built from the units that read it"_test[s::gcc] = [] { + // Design 2026-11-20 §5.11: the build directory's records (ninja's deps log, the scans' + // depfiles) name the units that read a header; while a unit of the project has none, + // every test is affected. + auto p = s::project::scaffold("hdr"); + p.write("src/util.h", "inline int twice(int x) { return 2 * x; }\n"); + p.write("src/unused.h", "inline int never() { return 0; }\n"); + p.write("tests/reads.cpp", "#include \"../src/util.h\"\nint main() { return twice(2) == 4 ? 0 : 1; }\n"); + p.write("tests/other.cpp", "int main() { return 0; }\n"); + auto changed = [&](std::string_view path) { + auto r = p.mcpp({"test", "--changed", std::string(path)}); + expect(e2e::succeeded(r)) << s::text(r); + return s::text(r); + }; + const auto before = changed("src/util.h"); + expect(e2e::contains(before, "Affected 3 of 3 tests")) << before; + expect(e2e::contains(before, "has no record of what some unit read")) << before; + expect(fatal(e2e::succeeded(p.mcpp({"test"})))); + const auto after = changed("src/util.h"); + expect(e2e::contains(after, "Affected 1 of 3 tests")) << after; + expect(e2e::contains(after, "reads ... ok")) << after; + expect(s::lacks(after, "other ...")) << "a test that read no changed header ran"; + expect(e2e::contains(after, "src/util.h: read by 1 unit, 1 test")) << after; + const auto unused = changed("src/unused.h"); + expect(e2e::contains(unused, "Affected 0 of 3 tests")) << unused; + }; }}; } // namespace From 7c618f2d4884f56b65f86fba095eb8a264cd0ae8 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 05:15:59 +0900 Subject: [PATCH 19/38] =?UTF-8?q?SPEC-004=20=C2=A79=20items=2017=20to=2020?= =?UTF-8?q?;=20SPEC-017=20O6=20and=20the=20surface;=20features=20for=20the?= =?UTF-8?q?=20runner's=20new=20clauses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Item 17 states the instance digest as a function of the package's configuration; 18 the roles of paths in keys and the home OUT_DIR of a named-source package's build program; 19 [build] memory_threshold; 20 [profile.] dependency_debug. SPEC-017 states mcpp's side of O6 and the surface mcpp emits and records. self env announces test-steps, test-budget, test-jitter and test-fixtures, which SPEC-017 names. An invalid memory threshold names the source that holds instead. --- docs/specs/manifest-semantics.md | 33 ++++++++++++++++++++++++++------ docs/specs/test.md | 7 +++++-- src/build/prepare/plan.cpp | 2 +- src/wire.cppm | 6 +++++- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/docs/specs/manifest-semantics.md b/docs/specs/manifest-semantics.md index bfa72bf23..70dfa1166 100644 --- a/docs/specs/manifest-semantics.md +++ b/docs/specs/manifest-semantics.md @@ -677,17 +677,38 @@ mcpp 2026.9.26.2,#703)。** 16. **虚拟根旁的 `build.mcpp`。** 不带 `[package]` 的工作空间根旁的 `build.mcpp` 不属于任何包, 不运行;实现**必须**给出警告,`--strict` 下**必须**拒绝。 17. **包实例的目录。** 一个配置的构建目录中,根以外的每个包按实例存放:对象在 - `obj/<包>/<摘要>/`,BMI 在 BMI 目录的 `<包>/<摘要>/`。摘要**必须**覆盖决定该包编译命令的因素 - (它的 include 目录与编译参数,含其 feature 的宏)以及它所导入模块的各包的摘要,**禁止**含版本 - 与编译器的构建号。因此不同命令或选择对同一个包请求的不同 feature 各自占有一个实例,交替执行 - 时不重新编译。实现以 `instances.tsv` 记录每个实例最近被构建使用的时间;`mcpp clean --stale` - 删除窗口内未被使用的实例,写出该配置构建图的规划所用的实例**必须**保留。 + `obj/<包>/<摘要>/`,BMI 在 BMI 目录的 `<包>/<摘要>/`。摘要**必须**是包的配置的函数:声明的 + 标志与宏、展开后的包含目录、按 glob 的规则(以规则本身,而不以它匹配到的单元)、生效的特性、 + 语言标准,以及它所导入模块的各包的摘要;单元的集合与单元的输出路径**禁止**进入,版本与编译器 + 的构建号**禁止**进入。实例目录的摘要与缓存键的配置部分由同一个函数产生(@NEXT@,mcpp#986)。 + 因此不同命令或选择对同一个包请求的不同 feature 各自占有一个实例,交替执行时不重新编译;特性 + 相同时 `mcpp build` 与 `mcpp test` 共用一个实例。实现以 `instances.tsv` 记录每个实例最近被 + 构建使用的时间;`mcpp clean --stale` 删除窗口内未被使用的实例,写出该配置构建图的规划所用的 + 实例**必须**保留。 +18. **键中的路径按角色出现。** 缓存键中的路径**必须**以其角色写出:``(每一个存储根)、 + ``(包的根)、``(构建程序的输出目录);绝对路径**禁止**进入键。来源被命名的包 + (索引的版本、git 的提交、路径依赖的内容摘要)的构建程序,其输出目录**必须**位于 + `/build-cache/v1/out/<包>@<版本>/<键>/out`,键为该包不含输出目录一项的配置键;同一 + 身份的输出由所有工程共用,运行时持有该目录的文件锁。构建程序收到 `MCPP_OUT_DIR` 并可以把它 + 写进生成的文件,因此位置是内容的一部分,由身份决定。 +19. **`[build] memory_threshold`。** 编译执行器启动编译器之前读取系统的可用内存:低于该值时等待 + 已运行的编译器结束,不结束它们。值为大小(`"2GiB"`、`"512MiB"`、`"1.5GB"`)或字节数,`0` + 不等待。取值按 §2 的四层:`MCPP_MEMORY_THRESHOLD`、清单的 `[build] memory_threshold`、机器的 + `config.toml` 的 `[build] memory_threshold`、引擎的 1 GiB。它是配置键(作用域 C),不进入包的 + 配置。无法读出的值被报告,并取下一层。 +20. **依赖的调试信息。** `[profile.<名>] dependency_debug` 陈述来自存储与 git 的依赖的调试信息: + `true`(与 profile 的 `debug` 相同)、`false`(不带调试信息)、`"lines"`(只带行表:GCC 与 + Clang 的 `-g1`/`-gline-tables-only` 一类,由工具链行给出)。省略时,`dev` 与 `debug` 在 + `debug` 打开时取 `"lines"`,其余 profile 取 `debug` 的值。它只作用于这两类依赖,工作空间的 + 成员与路径依赖保持 profile 的 `debug`。选出的标志进入依赖的配置,因此依赖在缓存中另有条目。 + 其他取值被拒绝。 **状态:已实现(第 1 至 7 条 mcpp 2026.9.25.1;第 8 至 10 条 mcpp 2026.9.27.1,mcpp#713、 #714、#710;第 10 条的 `dialect_cxxflags` 为 mcpp 2026.9.28.1,#717;第 1 条的根包、第 10 条的 `[profile.]` 与第 11 条为 mcpp 2026.10.5.2,#771;第 7、8、10 条的工作空间层与第 12、13 条为 mcpp 2026.10.10.1,#785、#786;第 14 至 16 条与第 13 条的依赖位置为 mcpp 2026.10.11.1,#797; -第 17 条为 mcpp 2026.11.19.1,#953)。** +第 17 条为 mcpp 2026.11.19.1,#953,其摘要为配置的函数与第 18 至 20 条为 @NEXT@,mcpp#986、 +#802、#927)。** ## 10. 依赖的程序 diff --git a/docs/specs/test.md b/docs/specs/test.md index 1d4915ca1..a672ef462 100644 --- a/docs/specs/test.md +++ b/docs/specs/test.md @@ -7,7 +7,7 @@ | 状态 | 评审中 v2.5 | | 版本 | 2.5 | | 最后修改 | 2026-11-21 | -| 对应实现 | 逐条标注。B1–B4 与 O1、O2、O4–O7 mcpp >= 2026.11.8.1 至 2026.11.10.1(逐条);B5、B6、O3 的运行器一侧、O8 mcpp >= 2026.11.10.1;O9 由框架的工具实现(mtest >= 0.6.0);O10 mcpp >= 2026.11.13.1;P5 的 `step` 与 `exercise`、B4 的 `builds.ndjson` 与机器的字段、O5 在选择之下的时限、O11、O12 mcpp >= @NEXT@,mtest >= 0.11.0 | +| 对应实现 | 逐条标注。B1–B4 与 O1、O2、O4–O7 mcpp >= 2026.11.8.1 至 2026.11.10.1(逐条);B5、B6、O3 的运行器一侧、O8 mcpp >= 2026.11.10.1;O9 由框架的工具实现(mtest >= 0.6.0);O10 mcpp >= 2026.11.13.1;P5 的 `step` 与 `exercise`、B4 的 `builds.ndjson` 与机器的字段、O5 在选择之下的时限、O6 的运行器一侧、O11、O12 mcpp >= @NEXT@,mtest >= 0.11.0 | | 取代 | SPEC-017 v1.0(`test-runner.md`);mtest 测试协议版本 1(mtest `spec/protocol.md`)的规范正文;SPEC-004 中 `[test]` 的键 | | 相关规范 | [SPEC-004](manifest-semantics.md)(清单的平面与命名)、[SPEC-006](build-database.md)(构建数据库) | | 相关 issue | mcpp#914、mcpp#915、mcpp#917 | @@ -133,12 +133,13 @@ | `artifact` | `id`、`path`、`kind` | 用例产生的文件 | | `fixture` | `id`、`name`、`ms` | 用例促成为共享它的用例而构造的对象,及构造耗时 | | `step` | `id`、`name`、`t`(自程序启动起的毫秒)、`ms`,可选 `cpu_ms`、`status`(步骤运行的程序的退出状态)、`build`(该步骤是一次构建时,构建记录的标识) | 用例的一个步骤结束(mtest >= 0.11.0) | -| `exercise` | `id`、`items`:用例用到的产品面的项(字符串的列表,形式由产品给出,例如 `command build`、`flag build --target`、`key build.sources`、`reason build/cache`) | 用例运行期间,可多次(mtest >= 0.11.0) | +| `exercise` | `id`、`items`:用例用到的产品面的项(字符串的列表,形式由产品给出,例如 `command build`、`flag build --target`、`key build.sources`、`reason unknown-target`) | 用例运行期间,可多次(mtest >= 0.11.0) | | `end` | `id`、`status`、`ms`,可选 `reason`、`checks`(O3) | 用例结束 | | `missing` | `id` | P4.1 | | `done` | `status`、`counts`(状态到数目的对象) | 末个 | - **P5.6 步骤与用到的面(mtest >= 0.11.0;mcpp >= @NEXT@)。** 一个用例由若干步骤组成时(端到端用例运行的每条命令),程序**可以**为每个步骤写一个 `step` 事件;运行器把它们按到达顺序归入用例。`exercise` 列出用例用到的产品面的项;运行器把一个用例的各次 `exercise` 合并为一个集合。二者都不影响用例的结论。 +- **mcpp 的面(mcpp >= @NEXT@)。** `mcpp emit surface` 每行写出一项 `<种类> <名字>`:命令(`command emit surface`)、选项(`flag build --target`,全局选项为 `flag --quiet`)取自命令行的定义,键(`key build.cflags`,整张表为 `key toolchain`)取自清单的键登记表,原因(`reason unknown-target`)取自拒绝的原因标记;解析器自己加入的 `--help` 与 `--version` 不是项。环境变量 `MCPP_SURFACE_RECORD` 指明一个文件时,mcpp 的每个进程把它用到的项追加到该文件,每项每进程一次:所给的命令与选项、它读取的清单所陈述的登记表中的键、它记录的原因。测试 kit 把该文件转为用例的 `exercise` 事件。 - **P5.3** 用例的 `status` 为以下之一:`pass`、`fail`、`skip`(带 `reason`)、`xfail`(如预期地失败)、`xpass`(预期失败却通过;运行失败)、`invalid`(无法判定的测量;P6.3)。 - **P5.4** 运行器**必须**忽略未知字段与未知事件种类。运行器**必须**接受没有 `done` 的报告(程序崩溃):有 `begin` 而无 `end` 的用例失败,以程序的退出状态为原因。 - **P5.5** 程序的退出状态为零,当且仅当没有用例失败或意外通过,且没有被选用例缺失。 @@ -271,6 +272,7 @@ - **P11.2** 运行夹具是整次运行的一个对象。用例在 `uses` 中写出它使用的夹具;运行器在第一个使用者开始前服务它,在最后一个结束后结束它,并以 `MTEST_FIXTURE_` 把端点交给使用者的程序。 - **P11.3** 运行器以 `MTEST_SERVE` 启动列出该夹具的程序来服务它,输入为管道,不带 `MTEST_PROTOCOL`。程序不运行用例:准备夹具,向输出写一行 `mtest-ready <端点>`,在输入结束时结束夹具。 - **P11.4** 未就绪的运行夹具是机器的失败:使用它的用例以状态 `infra` 结束,不运行。 +- **mcpp 的实现(已实现,mcpp >= @NEXT@)。** `mcpp test` 从程序的列出(`list` 模式)取得夹具与用例的 `uses`;所选用例使用夹具的程序在启动前取得端点,第一个取得者启动列出该夹具的程序(缺省等待就绪的时间为 `--timeout`),最后一个使用者结束后关闭其输入,等待同样长的时间后结束其进程单元。运行存储的 `fixtures.ndjson` 每个夹具一行,`programs.ndjson` 的 `fixtures` 写出程序用到的夹具;夹具未就绪时,使用它的程序不运行,记录的状态为 `infra`,`mcpp test` 以不运行的测试结束(退出码 2)。不经列出而运行的程序(运行器之后的程序)不取得夹具。 ### O7. 资源用量(P10) @@ -358,6 +360,7 @@ mcpp 以 feature(docs/50 §4)宣告它实现的条款;客户端按需门 | `test-steps` | P5.6 的运行器一侧;B4 的 `steps`、`exercised` 与 `builds.ndjson` | @NEXT@ | | `test-budget` | O11 | @NEXT@ | | `test-jitter` | O12 的运行器一侧 | @NEXT@ | +| `test-fixtures` | O6 的运行器一侧;B4 的 `fixtures.ndjson` | @NEXT@ | 框架实现的可选能力由 B6 宣告,不由 mcpp 的 feature 宣告。 diff --git a/src/build/prepare/plan.cpp b/src/build/prepare/plan.cpp index 3075bce63..30e2b3a19 100644 --- a/src/build/prepare/plan.cpp +++ b/src/build/prepare/plan.cpp @@ -1075,7 +1075,7 @@ static void step13_graph_and_schedule(PrepareState& state, BuildContext& ctx) { ctx.manifest, machineMemoryThreshold, [](std::string_view bad) { mcpp::ui::warning(std::format( "ignoring invalid memory threshold '{}' (expected a size such as \"2GiB\" or bytes); " - "the engine's 1 GiB holds", bad)); + "the next of the environment, the manifest, the machine and the engine's 1 GiB holds", bad)); }); ctx.intent = state.overrides.intent; diff --git a/src/wire.cppm b/src/wire.cppm index 97dec0b54..fdb0018ec 100644 --- a/src/wire.cppm +++ b/src/wire.cppm @@ -260,7 +260,7 @@ inline nlohmann::json to_json(const Diagnostic& in) { // exist, which a client turns something on or off by. A name is added when a // client gates on it, is never renamed and never removed; docs/50 names the // release that added it and the client that reads it. -inline constexpr std::array kFeatures{{ +inline constexpr std::array kFeatures{{ "target-sets", // SPEC-015 §1: `--target openkal` is the set's legs (an IDE's target picker) "pkg-format-list", // SPEC-014 §5.4: `--pkg-format tar,okb` writes each container (an IDE's pack form) "test-launch", // docs/50 §6: `mcpp test --print-launch` (an IDE debugging a test) @@ -273,6 +273,10 @@ inline constexpr std::array kFeatures{{ "test-frameworks", // SPEC-017 B6: `--guide` names each framework's capabilities and session (an IDE's choice of runner) "test-checks", // SPEC-017 O3: a failure's `source` and `fatal`, a case's `checks` (an IDE's check list) "test-output", // SPEC-017 O8: a case's own output, `output` (an IDE's case output) + "test-steps", // SPEC-017 P5.6, B4: a case's steps and the surface it used, the run's builds (a run's report) + "test-budget", // SPEC-017 O11: the run states a budget its programs' compilers take slots of (mtest run) + "test-jitter", // SPEC-017 O12: the jitter a run is told to inject (the jitter workflow) + "test-fixtures", // SPEC-017 O6: run fixtures served for the cases that use them (a framework's shared home) }}; // The same list as JSON: `--protocol-version` and every envelope's `mcpp` field From e10d52f7eac9154a9fac88b037554fb148dd0eb4 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 05:43:54 +0900 Subject: [PATCH 20/38] =?UTF-8?q?The=20test=20kit:=20surface=20items,=20th?= =?UTF-8?q?e=20shared=20home=20as=20a=20run=20fixture,=20homes=20that=20re?= =?UTF-8?q?ad=20the=20user's=20as=20upstreams,=20invariants,=20waits,=20di?= =?UTF-8?q?agnostic=20lines,=20golden=20outputs=20(design=202026-11-20=20?= =?UTF-8?q?=C2=A75.10,=20=C2=A75.16,=20=C2=A75.18,=20=C2=A75.20,=20=C2=A75?= =?UTF-8?q?.21,=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Each mcpp command a case runs records the surface items it used (MCPP_SURFACE_RECORD) as the case's exercise event. - The shared home is the run fixture 'home', filled once per run; every group uses it. A case whose assertion holds only while nothing else writes its home runs mcpp in a home of its own (own_home) instead of holding the machine. - Inherited homes, in the kit and in _inherit_toolchain.sh, name the user's homes as upstreams instead of linking each payload, and copy no subos (mcpp#293); 914 checks that no file the upstream's entries reach is written. - Invariant cases with bounds: an unchanged rebuild compiles nothing, and a dev-dependency is served from the cache to a second project. - Waits for a time only fall (check_waits.sh, waits.baseline); five that only moved a modification time touch the file instead. - The kit declares mcpp's warning and error lines diagnostic, under report. - The machine protocol and the surface are compared whole with golden files. --- .gitattributes | 2 + .github/tools/check_waits.sh | 35 ++ .github/tools/waits.baseline | 48 ++ .github/workflows/ci.yml | 3 + ...ings_is_replaced_from_the_newest_source.sh | 6 +- ...n_upstream_home_is_linked_never_written.sh | 12 +- tests/e2e/_inherit_toolchain.sh | 84 ++-- tests/e2e/actions/checks.cpp | 2 +- tests/e2e/actions/commands.cpp | 2 +- tests/e2e/actions/declared.cpp | 2 +- tests/e2e/actions/devices.cpp | 2 +- tests/e2e/actions/generated.cpp | 2 +- tests/e2e/actions/hooks.cpp | 2 +- tests/e2e/actions/outputs.cpp | 2 +- tests/e2e/actions/prepare.cpp | 2 +- tests/e2e/actions/progress.cpp | 2 +- tests/e2e/actions/rules.cpp | 2 +- tests/e2e/android/rows.cpp | 2 +- tests/e2e/baremetal/rows.cpp | 2 +- tests/e2e/basics/new_build_run.cpp | 2 +- tests/e2e/build-program/accel.cpp | 2 +- tests/e2e/build-program/cross.cpp | 2 +- tests/e2e/build-program/deps.cpp | 2 +- tests/e2e/build-program/graph.cpp | 2 +- tests/e2e/build-program/hosttools.cpp | 21 +- tests/e2e/build-program/protocol.cpp | 4 +- tests/e2e/build/basics.cpp | 2 +- tests/e2e/build/flags.cpp | 2 +- tests/e2e/build/link.cpp | 2 +- tests/e2e/build/sources.cpp | 2 +- tests/e2e/cache/directories.cpp | 2 +- tests/e2e/cache/entries.cpp | 2 +- tests/e2e/cache/fastpath.cpp | 16 +- tests/e2e/cache/invariants.cpp | 89 ++++ tests/e2e/cdb/configure.cpp | 2 +- tests/e2e/cdb/emit.cpp | 2 +- tests/e2e/cdb/planning.cpp | 2 +- tests/e2e/cdb/root.cpp | 14 +- tests/e2e/cdb/sets.cpp | 2 +- tests/e2e/cdb/toolchains.cpp | 2 +- tests/e2e/cli/commands.cpp | 2 +- tests/e2e/cli/golden.cpp | 33 ++ tests/e2e/cli/output.cpp | 4 +- tests/e2e/cli/signals.cpp | 2 +- tests/e2e/cli/templates.cpp | 2 +- tests/e2e/cli/terminal.cpp | 2 +- tests/e2e/deps/advice.cpp | 2 +- tests/e2e/deps/edges.cpp | 2 +- tests/e2e/deps/entry.cpp | 2 +- tests/e2e/deps/floors.cpp | 4 +- tests/e2e/deps/git.cpp | 2 +- tests/e2e/deps/graph.cpp | 2 +- tests/e2e/deps/identity.cpp | 2 +- tests/e2e/deps/lock.cpp | 2 +- tests/e2e/deps/names.cpp | 2 +- tests/e2e/deps/path.cpp | 2 +- tests/e2e/deps/resolve.cpp | 2 +- tests/e2e/deps/tools.cpp | 2 +- tests/e2e/features/activation.cpp | 2 +- tests/e2e/features/conditional.cpp | 2 +- tests/e2e/features/defaults.cpp | 2 +- tests/e2e/features/layers.cpp | 2 +- tests/e2e/features/sources.cpp | 2 +- tests/e2e/golden/protocol-version.json | 148 ++++++ tests/e2e/golden/surface.txt | 429 ++++++++++++++++++ tests/e2e/index/listing.cpp | 2 +- tests/e2e/index/local.cpp | 2 +- tests/e2e/index/refresh.cpp | 2 +- tests/e2e/index/routes.cpp | 2 +- tests/e2e/index/xlings.cpp | 4 +- tests/e2e/macos/rows.cpp | 2 +- tests/e2e/modules/basics.cpp | 2 +- tests/e2e/modules/bmi.cpp | 4 +- tests/e2e/modules/extensions.cpp | 12 +- tests/e2e/modules/programs.cpp | 2 +- tests/e2e/modules/scanner.cpp | 2 +- tests/e2e/openkal/bases.cpp | 2 +- tests/e2e/openkal/images.cpp | 2 +- tests/e2e/openkal/layers.cpp | 2 +- tests/e2e/openkal/machine.cpp | 2 +- tests/e2e/openkal/platforms.cpp | 2 +- tests/e2e/openkal/sets.cpp | 2 +- tests/e2e/openkal/stack.cpp | 2 +- tests/e2e/openkal/trees.cpp | 2 +- tests/e2e/pack/formats.cpp | 2 +- tests/e2e/pack/library.cpp | 2 +- tests/e2e/pack/members.cpp | 2 +- tests/e2e/pack/modes.cpp | 2 +- tests/e2e/run/contract.cpp | 14 +- tests/e2e/run/environment.cpp | 2 +- tests/e2e/run/program.cpp | 2 +- tests/e2e/run/runners.cpp | 2 +- tests/e2e/runtime/binding.cpp | 14 +- tests/e2e/runtime/contract.cpp | 2 +- tests/e2e/runtime/deploy.cpp | 2 +- tests/e2e/runtime/machine.cpp | 2 +- tests/e2e/runtime/packages.cpp | 2 +- tests/e2e/runtime/records.cpp | 2 +- tests/e2e/self/config.cpp | 2 +- tests/e2e/self/homes.cpp | 2 +- tests/e2e/self/machine.cpp | 2 +- tests/e2e/self/subos.cpp | 2 +- tests/e2e/targets/abi.cpp | 2 +- tests/e2e/targets/graph.cpp | 2 +- tests/e2e/targets/machine.cpp | 2 +- tests/e2e/targets/requests.cpp | 2 +- tests/e2e/targets/rows.cpp | 2 +- tests/e2e/targets/selection.cpp | 2 +- tests/e2e/targets/side.cpp | 2 +- tests/e2e/test/affected.cpp | 2 +- tests/e2e/test/archive.cpp | 2 +- tests/e2e/test/fast.cpp | 8 +- tests/e2e/test/frameworks.cpp | 2 +- tests/e2e/test/granularity.cpp | 2 +- tests/e2e/test/graph.cpp | 2 +- tests/e2e/test/launch.cpp | 2 +- tests/e2e/test/protocol.cpp | 2 +- tests/e2e/test/report.cpp | 2 +- tests/e2e/test/selection.cpp | 2 +- tests/e2e/test/skeleton.cpp | 2 +- tests/e2e/test/streams.cpp | 2 +- tests/e2e/test/workspace.cpp | 2 +- tests/e2e/toolchain/custom.cpp | 2 +- tests/e2e/toolchain/layers.cpp | 2 +- tests/e2e/toolchain/llvm.cpp | 2 +- tests/e2e/toolchain/management.cpp | 2 +- tests/e2e/toolchain/standard.cpp | 2 +- tests/e2e/web/emscripten.cpp | 2 +- tests/e2e/windows/crt.cpp | 2 +- tests/e2e/windows/deploy.cpp | 2 +- tests/e2e/windows/exports.cpp | 2 +- tests/e2e/windows/images.cpp | 2 +- tests/e2e/windows/libraries.cpp | 2 +- tests/e2e/windows/mingw.cpp | 2 +- tests/e2e/windows/msvc.cpp | 2 +- tests/e2e/windows/pack.cpp | 2 +- tests/e2e/windows/placement.cpp | 2 +- tests/e2e/windows/resources.cpp | 2 +- tests/e2e/windows/rows.cpp | 2 +- tests/e2e/workspace/graph.cpp | 2 +- tests/e2e/workspace/inheritance.cpp | 2 +- tests/e2e/workspace/members.cpp | 2 +- tests/e2e/workspace/statements.cpp | 2 +- tests/support/src/support.cpp | 155 +++++-- tests/support/src/support.cppm | 9 + 145 files changed, 1151 insertions(+), 261 deletions(-) create mode 100644 .github/tools/check_waits.sh create mode 100644 .github/tools/waits.baseline create mode 100644 tests/e2e/cache/invariants.cpp create mode 100644 tests/e2e/cli/golden.cpp create mode 100644 tests/e2e/golden/protocol-version.json create mode 100644 tests/e2e/golden/surface.txt diff --git a/.gitattributes b/.gitattributes index 647206538..116c35487 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,5 @@ tests/fixtures/openkal-hosted-threads/src/main.cpp.in text eol=lf # The rows' snapshot is compared line by line with what the engine answers, in its own bytes. tests/fixtures/target-rows.snapshot text eol=lf +# The golden outputs are compared whole with what mcpp writes, in its own bytes (tests/e2e/cli/golden.cpp). +tests/e2e/golden/* text eol=lf diff --git a/.github/tools/check_waits.sh b/.github/tools/check_waits.sh new file mode 100644 index 000000000..2ae66f957 --- /dev/null +++ b/.github/tools/check_waits.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# The waits for time in the tests only fall (design 2026-11-20 §5.20). +# +# A test waits for an event the program under test gives -- a line of output, a +# file, its exit -- and not for a time: a sleep long enough on one machine is +# short on a loaded one, and a poll hides how long the event took. Each file's +# count of `sleep_for`, `sleep_until` and shell `sleep ` is at most its line +# in waits.baseline; a file not listed has none. A wait whose time is itself +# what the case asserts says so in a comment on the line above it, and stays. +set -u +cd "$(dirname "$0")/../.." + +baseline=.github/tools/waits.baseline +[ -f "$baseline" ] || { echo "::error::missing $baseline"; exit 1; } + +current=$(grep -rcE 'sleep_for|sleep_until|(^|[;&|[:space:]])sleep [0-9]' tests \ + --include='*.cpp' --include='*.cppm' --include='*.sh' 2>/dev/null \ + | grep -v ':0$' | sed 's/:/ /' | sort) + +rc=0 +while read -r file n; do + [ -n "$file" ] || continue + allowed=$(awk -v f="$file" '$1 == f { print $2 }' "$baseline") + if [ -z "$allowed" ]; then + echo "::error::$file waits for a time $n time(s); a test waits for an event (design 2026-11-20 §5.20)" + rc=1 + elif [ "$n" -gt "$allowed" ]; then + echo "::error::$file waits for a time $n times, more than the $allowed of $baseline" + rc=1 + elif [ "$n" -lt "$allowed" ]; then + echo "note: $file waits $n times, fewer than the $allowed of $baseline; lower its line" + fi +done <<< "$current" +[ "$rc" = 0 ] && echo "OK: no test waits for a time more often than $baseline states" +exit "$rc" diff --git a/.github/tools/waits.baseline b/.github/tools/waits.baseline new file mode 100644 index 000000000..7951f5e69 --- /dev/null +++ b/.github/tools/waits.baseline @@ -0,0 +1,48 @@ +# One line per test file that waits for a time: its count of sleep_for, sleep_until and shell +# `sleep ` (.github/tools/check_waits.sh). The numbers only fall. +tests/e2e/288_the_openkal_stack_on_a_machine_with_no_os.sh 1 +tests/e2e/actions/checks.cpp 1 +tests/e2e/actions/commands.cpp 2 +tests/e2e/actions/declared.cpp 2 +tests/e2e/actions/generated.cpp 2 +tests/e2e/actions/hooks.cpp 3 +tests/e2e/actions/outputs.cpp 2 +tests/e2e/actions/prepare.cpp 1 +tests/e2e/build/basics.cpp 2 +tests/e2e/build-program/graph.cpp 3 +tests/e2e/build-program/hosttools.cpp 2 +tests/e2e/cache/directories.cpp 3 +tests/e2e/cache/fastpath.cpp 1 +tests/e2e/cdb/root.cpp 2 +tests/e2e/cli/commands.cpp 1 +tests/e2e/cli/output.cpp 1 +tests/e2e/cli/signals.cpp 1 +tests/e2e/cli/terminal.cpp 2 +tests/e2e/deps/advice.cpp 1 +tests/e2e/index/refresh.cpp 4 +tests/e2e/modules/bmi.cpp 1 +tests/e2e/modules/extensions.cpp 1 +tests/e2e/modules/programs.cpp 2 +tests/e2e/run/environment.cpp 1 +tests/e2e/run/program.cpp 1 +tests/e2e/runtime/binding.cpp 2 +tests/e2e/self/homes.cpp 2 +tests/e2e/test/fast.cpp 3 +tests/e2e/test/report.cpp 1 +tests/e2e/test/streams.cpp 1 +tests/e2e/toolchain/llvm.cpp 1 +tests/e2e/windows/images.cpp 1 +tests/e2e/windows/msvc.cpp 2 +tests/e2e/windows/resources.cpp 3 +tests/e2e/workspace/graph.cpp 1 +tests/matrix/scan.sh 1 +tests/release/verify-published.sh 1 +tests/support/src/support.cpp 1 +tests/unit/test_build_legs.cpp 1 +tests/unit/test_build_progress.cpp 3 +tests/unit/test_config.cpp 1 +tests/unit/test_detach_status.cpp 4 +tests/unit/test_network.cpp 3 +tests/unit/test_platform_fs.cpp 3 +tests/unit/test_process_run_exec.cpp 9 +tests/unit/test_xlings_version_pin.cpp 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f33399331..bb4efa0ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,6 +229,9 @@ jobs: - name: Reason tokens agree with the engine, and with the mirror run: bash .github/tools/check_reason_tokens.sh + - name: The tests wait for a time no more often than the baseline states + run: bash .github/tools/check_waits.sh + - name: Only the adapter and the composition root name the provider run: python3 .github/tools/check_provider_boundary.py diff --git a/tests/e2e/846_the_vendored_xlings_is_replaced_from_the_newest_source.sh b/tests/e2e/846_the_vendored_xlings_is_replaced_from_the_newest_source.sh index bad274dfa..6bb3ca32d 100755 --- a/tests/e2e/846_the_vendored_xlings_is_replaced_from_the_newest_source.sh +++ b/tests/e2e/846_the_vendored_xlings_is_replaced_from_the_newest_source.sh @@ -43,8 +43,12 @@ VENDORED="$MCPP_HOME/registry/bin/xlings$EXE" || fail "setup: the vendored binary does not answer as xlings" setup.err NINJA="" +# The test's home reads the developer's homes as upstreams (_inherit_toolchain.sh): the +# ninja payload is in one of them, and is not installed here before a build asks for it. for cand in "$MCPP_HOME"/registry/data/xpkgs/xim-x-ninja/*/ninja$EXE \ - "$MCPP_HOME"/registry/data/xpkgs/xim-x-ninja/*/bin/ninja$EXE; do + "$MCPP_HOME"/registry/data/xpkgs/xim-x-ninja/*/bin/ninja$EXE \ + "$USER_MCPP"/registry/data/xpkgs/xim-x-ninja/*/ninja$EXE \ + "$USER_MCPP"/registry/data/xpkgs/xim-x-ninja/*/bin/ninja$EXE; do if [ -f "$cand" ]; then NINJA="$cand"; break; fi done [ -n "$NINJA" ] || fail "setup: no ninja binary to stand in for an older xlings" diff --git a/tests/e2e/914_an_upstream_home_is_linked_never_written.sh b/tests/e2e/914_an_upstream_home_is_linked_never_written.sh index b81a54b09..8de526dc7 100755 --- a/tests/e2e/914_an_upstream_home_is_linked_never_written.sh +++ b/tests/e2e/914_an_upstream_home_is_linked_never_written.sh @@ -8,7 +8,8 @@ # A. The isolated home builds and runs a program; its C runtime payload is a # link into the upstream's store, and the program's loader is the path in # the isolated home's own store. -# B. The upstream's store is the same before and after (names, link targets). +# B. The upstream's store is the same before and after (names, link targets), +# and no file its entries reach was written. # C. `mcpp self doctor` names the upstream; `mcpp self store dedupe` finds # nothing to link that is not linked already. # D. An upstream another user could change is refused when the configuration @@ -27,8 +28,15 @@ UP="$TMP/upstream" export MCPP_HOME="$UP" source "$(dirname "$0")/_inherit_toolchain.sh" "$MCPP" self init > "$TMP/up.env" 2>&1 || fail "the upstream home does not initialise" "$TMP/up.env" +# The upstream holds the toolchain and the C runtime a build installs: it reads the developer's +# homes as upstreams itself (_inherit_toolchain.sh), so a build in it installs them by link. +(mkdir -p "$TMP/up-fill" && cd "$TMP/up-fill" && "$MCPP" new fill > new.log 2>&1 && cd fill \ + && "$MCPP" build > build.log 2>&1) || fail "the upstream home does not build" "$TMP/up-fill/fill/build.log" snapshot() { (cd "$UP/registry/data/xpkgs" && find . -maxdepth 2 -printf '%p %l\n' | sort); } snapshot > "$TMP/up.before" +# Nothing the upstream's entries reach is written: a link entry's payload is its owner's, and a +# hook that rewrote it wrote this home's paths into the owner's toolchain (mcpp#293). +touch "$TMP/up.mark" # The home under test: nothing installed, the upstream named. export MCPP_HOME="$TMP/home" @@ -56,6 +64,8 @@ echo "ok: A. built against the upstream's runtime, by link" # ── B ───────────────────────────────────────────────────────────────────────── snapshot > "$TMP/up.after" cmp -s "$TMP/up.before" "$TMP/up.after" || { diff "$TMP/up.before" "$TMP/up.after"; fail "B: the upstream changed"; } +find -L "$UP/registry/data/xpkgs" -type f -newer "$TMP/up.mark" 2>/dev/null > "$TMP/up.written" +[ ! -s "$TMP/up.written" ] || fail "B: files the upstream's entries reach were written" "$TMP/up.written" echo "ok: B. the upstream is unchanged" # ── C ───────────────────────────────────────────────────────────────────────── diff --git a/tests/e2e/_inherit_toolchain.sh b/tests/e2e/_inherit_toolchain.sh index b4e691bb0..77556d5f2 100644 --- a/tests/e2e/_inherit_toolchain.sh +++ b/tests/e2e/_inherit_toolchain.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Tests that override $MCPP_HOME for isolation (fresh BMI cache / git cache / -# etc.) but still need a working toolchain should source this. It links the -# user's installed toolchain payloads and copies the config so the default -# toolchain pin resolves. +# etc.) but still need a working toolchain should source this. It names the +# user's homes as upstreams, links the indices and copies the config so the +# default toolchain pin resolves. # # Usage: source "$(dirname "$0")/_inherit_toolchain.sh" # # ($MCPP_HOME must be set before sourcing) @@ -19,56 +19,31 @@ if [[ ! -d "$USER_MCPP" && -n "${USERPROFILE:-}" ]]; then USER_MCPP="$USERPROFILE/.mcpp" fi -# Links the developer's payloads into the test's home ONE VERSION AT A TIME. -# -# The package directory (`xpkgs/xim-x-glibc`) is created as a real directory in -# the test's home and only its version directories are links. Linking the -# package directory itself made every version a test INSTALLED land in the -# developer's registry: the install wrote `xim-x-glibc/2.44.3` through the link, -# with linker scripts naming the test's temporary home, which was deleted when -# the test ended. Measured 2026-09-25: `31_transitive_deps.sh` recreated -# `~/.mcpp/registry/data/xpkgs/xim-x-glibc/2.44.3/lib/libc.so` pointing at -# `/tmp/tmp.*/mcpp-home/...` (mcpp#293, second shape). A version that exists in -# the developer's registry is still shared by link, so an in-place rewrite of an -# existing payload remains the first shape of #293. -link_xpkg_payloads() { - local source_dir="$1" - # The provider's store by default; the store of mcpp's own indices when - # named (docs/25 §1). - local target_dir="${2:-$MCPP_HOME/registry/data/xpkgs}" - [[ -d "$source_dir" ]] || return 0 - mkdir -p "$target_dir" - - local entry base version vbase - shopt -s nullglob - for entry in "$source_dir"/*; do - base="$(basename "$entry")" - if [[ -d "$entry" && ! -L "$target_dir/$base" ]]; then - mkdir -p "$target_dir/$base" - for version in "$entry"/*; do - vbase="$(basename "$version")" - [[ -e "$target_dir/$base/$vbase" ]] && continue - ln -sf "$version" "$target_dir/$base/$vbase" 2>/dev/null \ - || cp -r "$version" "$target_dir/$base/$vbase" - done - continue - fi - [[ -e "$target_dir/$base" ]] && continue - ln -sf "$entry" "$target_dir/$base" 2>/dev/null \ - || cp -r "$entry" "$target_dir/$base" +# THE DEVELOPER'S HOMES ARE UPSTREAMS OF THE TEST'S HOME, NEVER LINKS INTO IT +# (mcpp#293, design 2026-11-20 §5.10). Each payload linked into the test's home +# made the developer's store part of it: a hook the test's home ran again (gcc's +# specs, an ELF patch) wrote through the link into the developer's toolchain, +# with paths of the test's temporary home, and an install landed in the +# developer's registry. An upstream is read by link at install and never written +# (docs/25 §3); the test's home installs what it needs from it, and registers +# the programs in its own subos, so no subos is copied either. +name_upstreams() { + local ups=() u list="" config="$MCPP_HOME/config.toml" + for u in "$USER_MCPP" "$HOME/.xlings" ${USERPROFILE:+"$USERPROFILE/.xlings"}; do + [[ -d "$u" ]] && ups+=("\"$(cd "$u" && pwd -P)\"") done - shopt -u nullglob + [[ ${#ups[@]} -gt 0 ]] || return 0 + list="upstream = [$(IFS=,; echo "${ups[*]}" | sed 's/,/, /g')]" + touch "$config" + # The line goes into the [xlings] table, replacing an upstream it had. + awk -v line="$list" ' + /^\[/ { if (inx && !done) { print line; done = 1 } inx = ($0 ~ /^\[xlings\]/) } + inx && /^upstream[ \t]*=/ { next } + { print } + END { if (inx && !done) { print line; done = 1 } if (!done) { print ""; print "[xlings]"; print line } } + ' "$config" > "$config.tmp" && mv "$config.tmp" "$config" } -if [[ -d "$USER_MCPP/registry/data/xpkgs" || -d "$HOME/.xlings/data/xpkgs" || ( -n "${USERPROFILE:-}" && -d "$USERPROFILE/.xlings/data/xpkgs" ) ]]; then - mkdir -p "$MCPP_HOME/registry/data" - link_xpkg_payloads "$USER_MCPP/registry/data/xpkgs" - link_xpkg_payloads "$USER_MCPP/xpkgs" "$MCPP_HOME/xpkgs" - link_xpkg_payloads "$HOME/.xlings/data/xpkgs" - if [[ -n "${USERPROFILE:-}" ]]; then - link_xpkg_payloads "$USERPROFILE/.xlings/data/xpkgs" - fi -fi if [[ -d "$USER_MCPP/registry/data/xim-pkgindex" ]]; then mkdir -p "$MCPP_HOME/registry/data" [[ -e "$MCPP_HOME/registry/data/xim-pkgindex" ]] \ @@ -81,17 +56,10 @@ if [[ -d "$USER_MCPP/registry/data/xim-index-repos" ]]; then || ln -sf "$USER_MCPP/registry/data/xim-index-repos" "$MCPP_HOME/registry/data/xim-index-repos" 2>/dev/null \ || cp -r "$USER_MCPP/registry/data/xim-index-repos" "$MCPP_HOME/registry/data/xim-index-repos" fi -# The subos is the test's own copy, its links kept as links (`cp -a`): linked -# whole, this test's installs wrote into the developer's subos, and an install -# in the developer's home by a test beside this one appeared in this test's. -if [[ "${MCPP_INHERIT_SUBOS:-1}" != "0" && -d "$USER_MCPP/registry/subos" ]]; then - mkdir -p "$MCPP_HOME/registry" - [[ -e "$MCPP_HOME/registry/subos" ]] \ - || cp -a "$USER_MCPP/registry/subos" "$MCPP_HOME/registry/subos" -fi if [[ "${MCPP_INHERIT_CONFIG:-1}" != "0" && -f "$USER_MCPP/config.toml" ]]; then cp -f "$USER_MCPP/config.toml" "$MCPP_HOME/config.toml" 2>/dev/null || true fi +name_upstreams if [[ -d "$USER_MCPP/bin" ]]; then mkdir -p "$MCPP_HOME" [[ -e "$MCPP_HOME/bin" ]] \ diff --git a/tests/e2e/actions/checks.cpp b/tests/e2e/actions/checks.cpp index 9d83677ce..d22e403b4 100644 --- a/tests/e2e/actions/checks.cpp +++ b/tests/e2e/actions/checks.cpp @@ -18,7 +18,7 @@ std::optional ninja_file(const fs::path& target) { return std::nullopt; } -E2E checks{"checks", [] { +E2E checks{"checks", s::home_fixture, [] { "313 check stamp on every platform"_test = [] { // A `role = "check"` whose command never touches its stamp, on every platform: the // engine writes the stamp (one), and a no-op rebuild does not run the check again. diff --git a/tests/e2e/actions/commands.cpp b/tests/e2e/actions/commands.cpp index 88dde0c95..57b7a45b2 100644 --- a/tests/e2e/actions/commands.cpp +++ b/tests/e2e/actions/commands.cpp @@ -26,7 +26,7 @@ long long mtime(const fs::path& p) { std::string mcpp_host() { return fs::absolute(s::mcpp_path()).generic_string(); } -E2E commands{"commands", [] { +E2E commands{"commands", s::home_fixture, [] { "631 action depfile tracks a runtime discovered include"_test[s::gcc] = [] { // `mcpp::action`'s `depfile`: a file the command discovers by running, named only in the // depfile it writes, reruns the action when touched; nothing else does. diff --git a/tests/e2e/actions/declared.cpp b/tests/e2e/actions/declared.cpp index 5c8e38b6f..1e2866fc0 100644 --- a/tests/e2e/actions/declared.cpp +++ b/tests/e2e/actions/declared.cpp @@ -25,7 +25,7 @@ std::optional ninja_file(const fs::path& target) { return std::nullopt; } -E2E declared{"declared", [] { +E2E declared{"declared", s::home_fixture, [] { "188 build actions"_test[s::gcc] = [] { // `mcpp:action=`: a build program declares work and the work becomes an edge: a // source action's output is compiled, a check's stamp and an artifact appear; an action diff --git a/tests/e2e/actions/devices.cpp b/tests/e2e/actions/devices.cpp index 477d621c3..680a6c923 100644 --- a/tests/e2e/actions/devices.cpp +++ b/tests/e2e/actions/devices.cpp @@ -48,7 +48,7 @@ s::project scaffold(std::string_view name) { return p; } -E2E devices{"devices", [] { +E2E devices{"devices", s::home_fixture, [] { "606 constrained source globs narrow to the build"_test[s::gcc] = [] { // A `[build] sources` entry may carry the accel it is for: covered, its C++ compiles and // its device sources reach the build program; under --no-accel it is left out; outside diff --git a/tests/e2e/actions/generated.cpp b/tests/e2e/actions/generated.cpp index c9ca70eb3..aa7b678c2 100644 --- a/tests/e2e/actions/generated.cpp +++ b/tests/e2e/actions/generated.cpp @@ -26,7 +26,7 @@ std::optional ninja_file(const fs::path& target) { return std::nullopt; } -E2E generated{"generated", [] { +E2E generated{"generated", s::home_fixture, [] { "51 package generated files"_test[s::gcc][s::fresh_sandbox] = [] { // A package descriptor's generated_files are materialized in the package before it is // scanned and compiled. diff --git a/tests/e2e/actions/hooks.cpp b/tests/e2e/actions/hooks.cpp index 0e0e2a138..1a47ccc8e 100644 --- a/tests/e2e/actions/hooks.cpp +++ b/tests/e2e/actions/hooks.cpp @@ -38,7 +38,7 @@ bool frozen(const fs::path& beat) { return before != 0 && before == size_of(beat); } -E2E hooks{"hooks", [] { +E2E hooks{"hooks", s::home_fixture, [] { "317 project build hooks"_test = [] { // [hooks]: start/finished/failed in order, run in the project root; a failing or timed // out hook warns and keeps the result; side_effect = true is refused, enabled = false diff --git a/tests/e2e/actions/outputs.cpp b/tests/e2e/actions/outputs.cpp index 8f5c4bcf7..b5e60bc7f 100644 --- a/tests/e2e/actions/outputs.cpp +++ b/tests/e2e/actions/outputs.cpp @@ -24,7 +24,7 @@ std::optional ending(const fs::path& root, std::string_view name, std: return std::nullopt; } -E2E outputs{"outputs", [] { +E2E outputs{"outputs", s::home_fixture, [] { "827 many placements run as one process"_test[s::unix_shell] = [] { // The files a program's runtime needs beside it are placed by one `stage_list` edge, not // one per file; the edge does not run when no placement changed; a changed source diff --git a/tests/e2e/actions/prepare.cpp b/tests/e2e/actions/prepare.cpp index 6399176e5..d9972b9e4 100644 --- a/tests/e2e/actions/prepare.cpp +++ b/tests/e2e/actions/prepare.cpp @@ -13,7 +13,7 @@ bool failed(const e2e::outcome& r) { return r.started && r.status != 0; } constexpr std::string_view prep_manifest = "[package]\nname = \"prep\"\nversion = \"0.1.0\"\n\n[toolchain]\nlinux = \"gcc@16.1.0\"\n\n[targets.prep]\nkind = \"bin\"\nmain = \"src/main.cpp\"\n"; -E2E prepare{"prepare", [] { +E2E prepare{"prepare", s::home_fixture, [] { "790 a prepare action populates an unknown directory"_test[s::gcc] = [] { // A `prepare` action declares only the directory a command fills; the build reads it by // directory. It runs on the first build, not on an unchanged one (nor does the compile), diff --git a/tests/e2e/actions/progress.cpp b/tests/e2e/actions/progress.cpp index a81691b55..82c74ec75 100644 --- a/tests/e2e/actions/progress.cpp +++ b/tests/e2e/actions/progress.cpp @@ -173,7 +173,7 @@ e2e::outcome python(std::string_view driver, std::vectorstring(), file.string()}, .env = std::move(env), .timeout = std::chrono::minutes{10}}); } -E2E progress{"progress", [] { +E2E progress{"progress", s::home_fixture, [] { "1013 optional action progress preserves outputs"_test[s::gcc][s::unix_shell][s::python3] = [] { // An action that reports progress through its execution channel, for every role: the // channel is shown through a terminal (plain and verbose) and removed after; turning it diff --git a/tests/e2e/actions/rules.cpp b/tests/e2e/actions/rules.cpp index 9fc481833..21d46859a 100644 --- a/tests/e2e/actions/rules.cpp +++ b/tests/e2e/actions/rules.cpp @@ -34,7 +34,7 @@ fs::path example_copy() { return ex; } -E2E rules{"rules", [] { +E2E rules{"rules", s::home_fixture, [] { "610 feature controlled rule collection"_test[s::gcc] = [] { // One package, several rules, the set chosen by the consumer's feature request: an // activated feature's unit is importable under its declared name, an inactive one is diff --git a/tests/e2e/android/rows.cpp b/tests/e2e/android/rows.cpp index e0ebc4ca4..1ae012170 100644 --- a/tests/e2e/android/rows.cpp +++ b/tests/e2e/android/rows.cpp @@ -85,7 +85,7 @@ bool lines_contain(const std::string& text, std::string_view line) { std::string first_line(const std::string& text) { return text.substr(0, text.find('\n')); } -E2E rows{"rows", [] { +E2E rows{"rows", s::home_fixture, [] { "641 the android rows are wired and the simulator is a row"_test[s::gcc] = [] { // The vocabulary half, which a runner without the NDK can assert; nothing installs a // payload. 1 both Android rows name android-ndk; 2 they are not refused as planned; 3 the diff --git a/tests/e2e/baremetal/rows.cpp b/tests/e2e/baremetal/rows.cpp index 199eeba35..e0cb70ec0 100644 --- a/tests/e2e/baremetal/rows.cpp +++ b/tests/e2e/baremetal/rows.cpp @@ -57,7 +57,7 @@ void placed_and_closed(const fs::path& image, std::string_view address) { expect(fatal(s::count_matching(undefined, R"(\S)") == 0)) << "the image expects symbols nothing provides: " << undefined; } -E2E rows{"rows", [] { +E2E rows{"rows", s::home_fixture, [] { "136 aarch64 zero libc target"_test[s::llvm] = [] { // A the row builds with no C library anywhere in the project; B the image is aarch64 and // starts where the script says; C nothing is undefined; D the ABI flag is the measured diff --git a/tests/e2e/basics/new_build_run.cpp b/tests/e2e/basics/new_build_run.cpp index 80524cf9e..6a86f4246 100644 --- a/tests/e2e/basics/new_build_run.cpp +++ b/tests/e2e/basics/new_build_run.cpp @@ -8,7 +8,7 @@ namespace s = mcpp::test::support; namespace { -E2E basics{"basics", s::fresh_sandbox, [] { +E2E basics{"basics", s::home_fixture, s::fresh_sandbox, [] { "02 a new program is scaffolded, built and run with its arguments"_test = [] { s::project root; expect(fatal(e2e::succeeded(root.mcpp({"new", "hello"})))); diff --git a/tests/e2e/build-program/accel.cpp b/tests/e2e/build-program/accel.cpp index 52a1abe10..a8e0cece7 100644 --- a/tests/e2e/build-program/accel.cpp +++ b/tests/e2e/build-program/accel.cpp @@ -21,7 +21,7 @@ s::project scaffold_plain(std::string_view name, const std::string& manifest) { return p; } -E2E accel{"accel", [] { +E2E accel{"accel", s::home_fixture, [] { "600 accel variant selection"_test[s::elf] = [] { // gpuonly publishes one artifact, for sm_90f: an sm_86 build is refused naming the // dimension. gpukit publishes a CPU-only artifact first: the same build takes it (the diff --git a/tests/e2e/build-program/cross.cpp b/tests/e2e/build-program/cross.cpp index 5bfa7089d..6fe58e9f0 100644 --- a/tests/e2e/build-program/cross.cpp +++ b/tests/e2e/build-program/cross.cpp @@ -10,7 +10,7 @@ namespace fs = std::filesystem; namespace { -E2E cross{"cross", [] { +E2E cross{"cross", s::home_fixture, [] { "112 build mcpp cross"_test[s::gcc][s::mingw_cross] = [] { // Under `--target x86_64-windows-gnu` build.mcpp compiles and runs on the host and sees // MCPP_TARGET = the cross triple. diff --git a/tests/e2e/build-program/deps.cpp b/tests/e2e/build-program/deps.cpp index 868196fe3..863b8e2b1 100644 --- a/tests/e2e/build-program/deps.cpp +++ b/tests/e2e/build-program/deps.cpp @@ -32,7 +32,7 @@ std::string library(std::string_view name) { return std::format("[package]\nname = \"{0}\"\nversion = \"0.1.0\"\n\n[targets.{0}]\nkind = \"lib\"\n", name); } -E2E deps{"deps", [] { +E2E deps{"deps", s::home_fixture, [] { "111 dep build mcpp"_test[s::gcc] = [] { // A dependency's build.mcpp runs with Cargo's scope: its define reaches only the // dependency's own units, its generated source (in MCPP_OUT_DIR) compiles into it, and diff --git a/tests/e2e/build-program/graph.cpp b/tests/e2e/build-program/graph.cpp index 6f7875d47..ad4d9ed64 100644 --- a/tests/e2e/build-program/graph.cpp +++ b/tests/e2e/build-program/graph.cpp @@ -19,7 +19,7 @@ std::string run_line(const s::project& p, std::string_view prefix, std::vector check, std::string_vi return out; } -E2E hosttools{"hosttools", [] { +E2E hosttools{"hosttools", s::home_fixture, [] { "187 dep host tool"_test[s::gcc] = [] { // #355: the tool is built for the host into the store, found by mcpp::dep_bin(), and what // it generates is linked; a tool behind required_features is built with them active; a @@ -358,8 +358,8 @@ int main() { std::ranges::sort(objs); expect(eq(objs, std::vector{"embed_greeting.o", "main.o"})) << "a rule package reached the link, or the denominator is wrong"; // The generated source is a graph node: an edited input runs its edge again. - std::this_thread::sleep_for(std::chrono::seconds(1)); app.write("assets/greeting.txt", "a different embedded asset\n"); + s::touch(app.at("assets/greeting.txt")); expect(fatal(e2e::succeeded(app.mcpp({"build"})))) << "rebuild failed"; expect(eq(s::last_line(app.mcpp({"run"}).out), std::string("a different embedded asset"))) << "editing the asset did not re-run the embed action"; @@ -1119,12 +1119,14 @@ int main() { }; }}; -// The cases that hold the machine (P7.1): a suite of their own, which the runner runs with no -// other test beside it. 831 states that the fast path replays the build, and the fast path declines -// when the runtime environment's description changed: another program installing into the shared -// home beside it rewrote that description (measured on Linux CI, 2026-11-14). -E2E hosttools_machine{"hosttools-machine", resource{"machine"}, [] { +// 831 states that the fast path replays the build, and the fast path declines when the runtime +// environment's description changed: another program installing into the shared home beside it +// rewrote that description (measured on Linux CI, 2026-11-14). It runs mcpp in a home of its own +// whose payloads are the user's, as upstreams (`own_home`, design 2026-11-20 §5.10), instead of +// holding the machine. +E2E hosttools_own_home{"hosttools-own-home", s::home_fixture, [] { "831 a path dependency host module outside src"_test[s::unix_shell] = [] { + s::own_home(); // #734: a path dependency's host module is compiled into the consumer's build program, // which no edge of build.ninja names, so the fast path sees an edit to it itself (it swept // only src/). S1 an edit to rules/hm.cppm re-runs the program; S2 a newer file of a @@ -1141,16 +1143,15 @@ E2E hosttools_machine{"hosttools-machine", resource{"machine"}, [] { expect(fatal(e2e::succeeded(app.mcpp({"build"})))) << "the first build failed"; expect(fatal(e2e::succeeded(app.mcpp({"build", "-v"})))) << "the second build failed"; // S1. - std::this_thread::sleep_for(std::chrono::milliseconds(1100)); plug.write("rules/hm.cppm", "export module probe.hm831;\nexport inline int value() { return 2; }\n"); + s::touch(plug.at("rules/hm.cppm")); auto s1 = app.mcpp({"build"}); expect(s1.started && s1.status != 0) << "S1: the build after an edit to rules/hm.cppm succeeded without re-running the build program"; expect(e2e::contains(s::text(s1), "host module value 2")) << "S1: the build program did not read the edited host module"; // S2. plug.write("rules/hm.cppm", "export module probe.hm831;\nexport inline int value() { return 1; }\n"); expect(fatal(e2e::succeeded(app.mcpp({"build"})))) << "S2: the build after restoring the host module failed"; - std::this_thread::sleep_for(std::chrono::milliseconds(1100)); - fs::last_write_time(plug.at("tests/fixture/src/main.cpp"), fs::file_time_type::clock::now()); + s::touch(plug.at("tests/fixture/src/main.cpp")); auto s2 = app.mcpp({"build", "-v"}); expect(fatal(e2e::succeeded(s2))) << "S2: the build failed"; expect(s::lacks(s::text(s2), "declined")) << "S2: the fast path declined after a file of a package nested in the dependency changed"; diff --git a/tests/e2e/build-program/protocol.cpp b/tests/e2e/build-program/protocol.cpp index 17557328c..b566dbedf 100644 --- a/tests/e2e/build-program/protocol.cpp +++ b/tests/e2e/build-program/protocol.cpp @@ -13,7 +13,7 @@ namespace { constexpr std::string_view ran_again = R"(^ *build\.mcpp .* ran [0-9])"; -E2E protocol{"protocol", [] { +E2E protocol{"protocol", s::home_fixture, [] { "89 build mcpp"_test[s::gcc] = [] { // mcpp compiles build.mcpp with the host toolchain and runs it before the build; its // directives add a define and a generated source; a declared-input cache runs it again @@ -842,7 +842,7 @@ int main() { // The cases that hold the machine (P7.1): a suite of their own, which the runner runs with no // other test beside it. -E2E protocol_machine{"protocol-machine", resource{"machine"}, [] { +E2E protocol_machine{"protocol-machine", s::home_fixture, resource{"machine"}, [] { "139 build program advisory"_test[s::unix_shell] = [] { // A program that succeeds and still says something (`mcpp:warning=`). The second build is // the test: the program's result is cached, and an advisory printed only where the diff --git a/tests/e2e/build/basics.cpp b/tests/e2e/build/basics.cpp index 93e12a9e5..4a306aa74 100644 --- a/tests/e2e/build/basics.cpp +++ b/tests/e2e/build/basics.cpp @@ -69,7 +69,7 @@ std::optional path_ending(const fs::path& root, std::string_view tail) return std::nullopt; } -E2E basics{"basics", [] { +E2E basics{"basics", s::home_fixture, [] { "02 new build run"_test[s::fresh_sandbox] = [] { // `mcpp new` writes a project that builds into target/ and runs, passing its arguments. auto p = s::project::scaffold("hello"); diff --git a/tests/e2e/build/flags.cpp b/tests/e2e/build/flags.cpp index 6b96e2357..e19f1a866 100644 --- a/tests/e2e/build/flags.cpp +++ b/tests/e2e/build/flags.cpp @@ -69,7 +69,7 @@ std::optional path_ending(const fs::path& root, std::string_view tail) return std::nullopt; } -E2E flags{"flags", [] { +E2E flags{"flags", s::home_fixture, [] { "50 package owned build flags"_test[s::gcc] = [] { // A dependency's own cflags and cxxflags compile its C and C++ sources; the consumer // declares neither, and the graph carries both. diff --git a/tests/e2e/build/link.cpp b/tests/e2e/build/link.cpp index a89ff98e2..adbc6ba20 100644 --- a/tests/e2e/build/link.cpp +++ b/tests/e2e/build/link.cpp @@ -89,7 +89,7 @@ std::size_t longest_line(const fs::path& f) { return longest; } -E2E link{"link", [] { +E2E link{"link", s::home_fixture, [] { "190 link rspfile newlines"_test = [] { // Link rules write their response file one input per line ($in_newline), and a relink // kept with -d keeprsp holds the objects one per line. diff --git a/tests/e2e/build/sources.cpp b/tests/e2e/build/sources.cpp index 5e121c256..a838a6a02 100644 --- a/tests/e2e/build/sources.cpp +++ b/tests/e2e/build/sources.cpp @@ -68,7 +68,7 @@ std::optional path_ending(const fs::path& root, std::string_view tail) return std::nullopt; } -E2E sources{"sources", [] { +E2E sources{"sources", s::home_fixture, [] { "104 asm sources gas"_test[s::gcc] = [] { // GAS sources (.S) are globbed by default, routed to asm_object (no -std), preprocessed, // linked, rebuilt when touched, and covered on the test path too (x86_64 only). diff --git a/tests/e2e/cache/directories.cpp b/tests/e2e/cache/directories.cpp index a8ae3ef6d..912d4d696 100644 --- a/tests/e2e/cache/directories.cpp +++ b/tests/e2e/cache/directories.cpp @@ -86,7 +86,7 @@ void copy_tree(const fs::path& from, const fs::path& to) { } // namespace -E2E directories{"directories", [] { +E2E directories{"directories", s::home_fixture, [] { "609 clean stale"_test = [] { // #565: `clean --stale` removes only the fingerprint directories no record considers // current and older than the window; --dry-run lists; --older-than; refusals; the diff --git a/tests/e2e/cache/entries.cpp b/tests/e2e/cache/entries.cpp index ae31d56ae..2153d6b06 100644 --- a/tests/e2e/cache/entries.cpp +++ b/tests/e2e/cache/entries.cpp @@ -78,7 +78,7 @@ void age(const fs::path& root, std::chrono::hours ago) { } // namespace -E2E entries{"entries", [] { +E2E entries{"entries", s::home_fixture, [] { "927 a fresh checkout's units are placed from the unit cache, and a changed header compiles its includers"_test[s::gcc] = [] { // Design 2026-11-20 §5.3. A unit whose command and read files hold the contents an earlier // compile's did is placed by the compiler executor, and no compiler runs: after `target/` diff --git a/tests/e2e/cache/fastpath.cpp b/tests/e2e/cache/fastpath.cpp index f6be7cfab..f0323063b 100644 --- a/tests/e2e/cache/fastpath.cpp +++ b/tests/e2e/cache/fastpath.cpp @@ -62,11 +62,14 @@ bool version_less(std::string_view a, std::string_view b) { } // namespace -// It holds the machine (P7.1): each case states that a build is replayed, and a build is planned -// again when the runtime environment description changed -- another program installing into the -// shared home beside it rewrites that description (measured on Linux CI, 2026-11-14). -E2E fastpath{"fastpath", resource{"machine"}, [] { +// The cases whose assertion holds only while nothing else writes their home (design 2026-11-20 +// §5.10): each states that a build is replayed, and a build is planned again when the runtime +// environment description changed -- another program installing into the shared home beside it +// rewrote that description (measured on Linux CI, 2026-11-14). Each runs mcpp in a home of its own +// whose payloads are the user's, as upstreams (`own_home`), instead of holding the machine. +E2E fastpath{"fastpath", s::home_fixture, [] { "173 build profile isolation"_test[s::gcc][s::fresh_sandbox] = [] { + s::own_home(); // The profile is an invalidation axis: --dev, --release and --profile dist have three build // directories (-O0, -O2, -O3, read from build.ninja), each announced as built; a bare // build after --release is dev, not the release graph's fast path; and switching back is @@ -130,6 +133,7 @@ E2E fastpath{"fastpath", resource{"machine"}, [] { }; "324 fast path sees path deps"_test[s::gcc] = [] { + s::own_home(); // A new file in a `path` dependency is compiled (the staleness sweep reaches the // dependency's tree), and an edit of the dependency's manifest plans again. const s::options cdb{.env = {{"MCPP_COMPILE_COMMANDS", "compile_commands.json"}}}; @@ -159,6 +163,7 @@ E2E fastpath{"fastpath", resource{"machine"}, [] { }; "645 the fast path compares the toolchain request"_test = [] { + s::own_home(); // T1 of the 2026-09-12 engine-gaps record: the fast path replays a build only for the // toolchain request that recorded it, A-B-A through `build` and `run`, and a changed // machine default resolves again. @@ -247,6 +252,7 @@ E2E fastpath{"fastpath", resource{"machine"}, [] { }; "832 the fast path resumes after an edit"_test[s::unix_shell] = [] { + s::own_home(); // #734: after an edit and one build, the next build is replayed by the fast path, and the // replay still sees the next edit. s::project p({{"mcpp.toml", "[package]\nname = \"resume832\"\nversion = \"0.1.0\"\n"}, {"src/main.cpp", "#include \nint main() { std::puts(\"one\"); return 0; }\n"}}); @@ -265,6 +271,7 @@ E2E fastpath{"fastpath", resource{"machine"}, [] { }; "878 an upgraded engine does not replay the old graph"_test[s::unix_shell] = [] { + s::own_home(); // #757: the record carries the engine that wrote it, and every fast path compares it; the // graph runs the engine through the build directory's link (design 2026-11-12 §17). The // same binary run from another path is what a reinstall looks like to the record. @@ -376,6 +383,7 @@ E2E fastpath{"fastpath", resource{"machine"}, [] { }; "879 a providers module extension reaches the fast path"_test[s::unix_shell] = [] { + s::own_home(); // #756: the sweep classifies a path dependency's files with that package's own extension // table, so an edit to a provider's `.ixx` re-runs the consumer's build program, as a // project (build and run) and as a workspace member. diff --git a/tests/e2e/cache/invariants.cpp b/tests/e2e/cache/invariants.cpp new file mode 100644 index 000000000..a95afff37 --- /dev/null +++ b/tests/e2e/cache/invariants.cpp @@ -0,0 +1,89 @@ +// The invariants of a build that each pull request keeps (design 2026-11-20 §6): what a second +// build of the same thing does, measured from the build's own record (MCPP_BUILD_RECORD, +// SPEC-017 B4) and stated as metrics with bounds. Each was a defect when it was introduced, and no +// test saw it: #986 (a test compiled the package again), 5.2 (a second project compiled the +// platform), 5.3 (a fresh checkout compiled what the cache held), 5.4 (a dev-dependency missed the +// cache). The cases of #986, 5.2 and 5.3 are 986 (workspace/inheritance.cpp), 955 (deps/git.cpp) +// and 927 (cache/entries.cpp). +import std; +import mcpplibs.mtest; +import mcpp.test.support; + +using namespace mcpplibs::mtest; +namespace s = mcpp::test::support; +namespace fs = std::filesystem; +using nlohmann::json; + +namespace { + +// The compile edges of the builds a record file holds, and the units the cache served each package. +struct Built { + std::size_t compiled = 0; + std::map cached; + std::set compiledPackages; +}; + +bool compiles(std::string_view output) { + for (auto ext : {".o", ".obj", ".gcm", ".pcm", ".ifc"}) + if (output.ends_with(ext)) return true; + return false; +} + +Built read_record(const fs::path& file) { + Built b; + for (const auto& line : s::json_lines(s::read_file(file))) { + if (line.value("cached", false)) { + b.cached[line.value("package", "")] += line.value("units", std::size_t{0}); + continue; + } + if (compiles(line.value("output", "")) && !line.value("served", false)) { + ++b.compiled; + b.compiledPackages.insert(line.value("package", "")); + } + } + return b; +} + +// mcpp in `p` with its build record written to a file of the case: what the build did. +std::pair recorded(const s::project& p, std::vector args, std::string_view name) { + const auto file = s::case_dir() / std::format("record-{}.ndjson", name); + std::error_code ec; + fs::remove(file, ec); + auto r = p.mcpp(std::move(args), {.env = {{"MCPP_BUILD_RECORD", file.string()}}}); + return {r, read_record(file)}; +} + +E2E invariants{"invariants", s::home_fixture, [] { + "1121 an unchanged project builds again with no compile"_test[s::gcc][at_most{"edges.compiled", 0}] = [] { + s::own_home(); + auto p = s::project::scaffold("same"); + auto [first, built] = recorded(p, {"build"}, "first"); + expect(fatal(e2e::succeeded(first))) << s::text(first); + expect(fatal(gt(built.compiled, std::size_t{0}))) << "the first build compiled nothing: its record is empty"; + auto [again, rebuilt] = recorded(p, {"build"}, "again"); + expect(fatal(e2e::succeeded(again))) << s::text(again); + metric("edges.compiled", static_cast(rebuilt.compiled), "edges"); + }; + + "1122 a dev-dependency is served from the cache to a second project"_test[s::gcc][at_most{"edges.compiled.dependency", 0}] = [] { + // 5.4: mcpplibs.mtest, a dev-dependency of every test, is compiled once per home, not once + // per project. + s::own_home(); + auto one = s::project::scaffold("one"); + auto [first, built] = recorded(one, {"test"}, "one"); + expect(fatal(e2e::succeeded(first))) << s::text(first); + auto two = s::project::scaffold("two"); + auto [second, rebuilt] = recorded(two, {"test"}, "two"); + expect(fatal(e2e::succeeded(second))) << s::text(second); + std::size_t dependency = 0; + for (const auto& pkg : rebuilt.compiledPackages) + if (pkg.contains("mtest")) ++dependency; + expect(eq(dependency, std::size_t{0})) << "the second project compiled the dev-dependency"; + bool served = false; + for (const auto& [pkg, units] : rebuilt.cached) served = served || (pkg.contains("mtest") && units > 0); + expect(served) << "the record names no unit of mcpplibs.mtest served from the cache"; + metric("edges.compiled.dependency", static_cast(dependency), "edges"); + }; +}}; + +} // namespace diff --git a/tests/e2e/cdb/configure.cpp b/tests/e2e/cdb/configure.cpp index 6773cf4e8..54b2f0d4d 100644 --- a/tests/e2e/cdb/configure.cpp +++ b/tests/e2e/cdb/configure.cpp @@ -26,7 +26,7 @@ std::string folded(std::string p) { return p; } -E2E configure{"configure", [] { +E2E configure{"configure", s::home_fixture, [] { "211 configure only cdb"_test = [] { const auto tmp = s::case_dir(); s::project(tmp / "devkit", {{"mcpp.toml", "[package]\nname = \"devkit\"\nversion = \"0.1.0\"\n\n[build]\ninclude_dirs = [\"include\"]\n"}, diff --git a/tests/e2e/cdb/emit.cpp b/tests/e2e/cdb/emit.cpp index f1fed8a8d..55332f48f 100644 --- a/tests/e2e/cdb/emit.cpp +++ b/tests/e2e/cdb/emit.cpp @@ -43,7 +43,7 @@ std::vector compiled(const fs::path& root, const fs::path& prune = {}) return out; } -E2E emit{"emit", [] { +E2E emit{"emit", s::home_fixture, [] { "688 emit build database"_test[s::python3] = [] { const auto tmp = s::case_dir() / "measured"; const auto out = s::case_dir() / "out"; diff --git a/tests/e2e/cdb/planning.cpp b/tests/e2e/cdb/planning.cpp index c2d7ec84f..476b28889 100644 --- a/tests/e2e/cdb/planning.cpp +++ b/tests/e2e/cdb/planning.cpp @@ -17,7 +17,7 @@ std::vector set_names(const nlohmann::json& envelope) { return out; } -E2E planning{"planning", [] { +E2E planning{"planning", s::home_fixture, [] { "787 emit plans every member on its own"_test[s::python3] = [] { // A member whose plan fails leaves the others described (exit 1, one diagnostic naming // its manifest, both members watched); the bare document likewise, the reason on standard diff --git a/tests/e2e/cdb/root.cpp b/tests/e2e/cdb/root.cpp index bf957a28b..9c8de41d5 100644 --- a/tests/e2e/cdb/root.cpp +++ b/tests/e2e/cdb/root.cpp @@ -13,7 +13,7 @@ namespace { const s::options at_root{.env = {{"MCPP_COMPILE_COMMANDS", "compile_commands.json"}}}; -E2E root{"root", [] { +E2E root{"root", s::home_fixture, [] { "47 cdb prebuilt module path abs"_test = [] { // Every -fprebuilt-module-path= of the database is absolute, unquoted, free of ninja // escapes, and names a pcm.cache or gcm.cache directory. @@ -147,12 +147,14 @@ E2E root{"root", [] { }}; -// The cases that hold the machine (P7.1): a suite of their own, which the runner runs with no -// other test beside it. Each states that a build is replayed, and a build is planned again when -// the runtime environment description changed: another program installing into the shared home -// beside it rewrote that description (measured on Linux CI, 2026-11-14). -E2E root_machine{"root-machine", resource{"machine"}, [] { +// The cases whose assertion holds only while nothing else writes their home (design 2026-11-20 +// §5.10): each states that a build is replayed, and a build is planned again when the runtime +// environment description changed -- another program installing into the shared home beside it +// rewrote that description (measured on Linux CI, 2026-11-14). Each runs mcpp in a home of its own +// whose payloads are the user's, as upstreams (`own_home`), instead of holding the machine. +E2E root_own_home{"root-own-home", s::home_fixture, [] { "781 deleted root cdb is restored"_test = [] { + s::own_home(); // A root database removed by hand is restored by the next build, without a full plan, // equal to the configuration's own; a second removal likewise. auto p = s::project::scaffold("app"); diff --git a/tests/e2e/cdb/sets.cpp b/tests/e2e/cdb/sets.cpp index c7c343a43..0fe7f309a 100644 --- a/tests/e2e/cdb/sets.cpp +++ b/tests/e2e/cdb/sets.cpp @@ -20,7 +20,7 @@ std::string slash(std::string p) { std::string prefix_of(const std::string& n) { return n.contains('/') ? n.substr(0, n.find('/')) : std::string(); } std::string base_of(const std::string& n) { return n.substr(n.rfind('/') + 1); } -E2E sets{"sets", [] { +E2E sets{"sets", s::home_fixture, [] { "840 a workspace database describes each package once"_test[s::python3] = [] { // A: one configuration: each set and each source once, names unprefixed; B: two // configurations: names prefixed by their fingerprint, core once in each, the root diff --git a/tests/e2e/cdb/toolchains.cpp b/tests/e2e/cdb/toolchains.cpp index 142f465a4..32f24506d 100644 --- a/tests/e2e/cdb/toolchains.cpp +++ b/tests/e2e/cdb/toolchains.cpp @@ -57,7 +57,7 @@ const nlohmann::json& entry_ending(const nlohmann::json& entries, std::string_vi fail(std::format("no entry whose file ends with {}", suffix)); } -E2E toolchains{"toolchains", [] { +E2E toolchains{"toolchains", s::home_fixture, [] { "783 cdb switches whole with the configuration"_test[s::llvm] = [] { // `mcpp test` then `mcpp build` in one configuration keep the test entry; switching the // toolchain switches the whole root file; switching back restores that configuration's diff --git a/tests/e2e/cli/commands.cpp b/tests/e2e/cli/commands.cpp index 72c3f64c1..0905bdb58 100644 --- a/tests/e2e/cli/commands.cpp +++ b/tests/e2e/cli/commands.cpp @@ -42,7 +42,7 @@ std::string graph_text(const s::project& p) { return s::read_file(*g); } -E2E commands{"commands", [] { +E2E commands{"commands", s::home_fixture, [] { "01 help and version"_test = [] { // --version is `mcpp `; --help names the commands; an unknown // command exits 127. diff --git a/tests/e2e/cli/golden.cpp b/tests/e2e/cli/golden.cpp new file mode 100644 index 000000000..1eecbcd7f --- /dev/null +++ b/tests/e2e/cli/golden.cpp @@ -0,0 +1,33 @@ +// The outputs whose form is itself the contract, compared whole (design 2026-11-20 §5.21): the +// machine protocol a client reads before anything else (docs/50 §4) and mcpp's surface (§5.16). +// A change of either is a change of what clients and the surface's ledger read, and its golden file +// changes with it in the same pull request: `MTEST_GOLDEN=update` writes the files and prints what +// it changed, which the review reads. +import std; +import mcpplibs.mtest; +import mcpp.test.support; + +using namespace mcpplibs::mtest; +namespace s = mcpp::test::support; +namespace fs = std::filesystem; + +namespace { + +fs::path golden_file(std::string_view name) { return s::repository() / "tests/e2e/golden" / name; } + +E2E golden{"golden", s::home_fixture, [] { + "1123 the machine protocol is the golden one"_test = [] { + auto r = s::mcpp({"--protocol-version"}, s::case_dir()); + expect(fatal(e2e::succeeded(r))) << s::text(r); + expect(e2e::golden(golden_file("protocol-version.json"), r.out, + {{R"("version": "[0-9][0-9.]*")", R"("version": "")"}})); + }; + + "1124 the surface is the golden one"_test = [] { + auto r = s::mcpp({"emit", "surface"}, s::case_dir()); + expect(fatal(e2e::succeeded(r))) << s::text(r); + expect(e2e::golden(golden_file("surface.txt"), r.out)); + }; +}}; + +} // namespace diff --git a/tests/e2e/cli/output.cpp b/tests/e2e/cli/output.cpp index 3442ecdba..8bc6e3a4a 100644 --- a/tests/e2e/cli/output.cpp +++ b/tests/e2e/cli/output.cpp @@ -90,7 +90,7 @@ std::string slow_program(std::string_view prefix, std::string_view id, int secon return out; } -E2E output{"output", [] { +E2E output{"output", s::home_fixture, [] { "48 build error output"_test = [] { // A failed build shows the compiler's diagnostic once, after the package's status line, // without ninja's progress or a private environment prefix; the graph holds no such @@ -349,7 +349,7 @@ int main() { // The cases that hold the machine (P7.1): a suite of their own, which the runner runs with no // other test beside it. -E2E output_machine{"output-machine", resource{"machine"}, [] { +E2E output_machine{"output-machine", s::home_fixture, resource{"machine"}, [] { "1085 a replayed build reports its artifacts"_test = [] { // The second `mcpp build --message-format json` of an unchanged tree replays the recorded // build; its stream and envelope name the program as the first build's did. A of a diff --git a/tests/e2e/cli/signals.cpp b/tests/e2e/cli/signals.cpp index 00ee61539..1e04bca5f 100644 --- a/tests/e2e/cli/signals.cpp +++ b/tests/e2e/cli/signals.cpp @@ -30,7 +30,7 @@ std::size_t ninjas_under(const fs::path& dir) { void sleep_ms(int ms) { std::this_thread::sleep_for(std::chrono::milliseconds{ms}); } -E2E signals{"signals", [] { +E2E signals{"signals", s::home_fixture, [] { "340 no orphan survives a killed mcpp"_test[s::gcc][s::unix_shell] = [] { // A serial build is running ninja when mcpp alone is sent SIGTERM; ten seconds later no // ninja of the project is alive. diff --git a/tests/e2e/cli/templates.cpp b/tests/e2e/cli/templates.cpp index a57f93938..c20515a26 100644 --- a/tests/e2e/cli/templates.cpp +++ b/tests/e2e/cli/templates.cpp @@ -51,7 +51,7 @@ void no_stage(const fs::path& parent) { expect(fatal(left.empty())) << "transaction staging directory survived" << left; } -E2E templates{"templates", [] { +E2E templates{"templates", s::home_fixture, [] { "69 package templates"_test = [] { // A package's templates by exact identity ([ns.]pkg, pkg:tmpl, pkg@ver:tmpl), {{var}} // rendering, injected features, --list-templates, a sole template as the default, the diff --git a/tests/e2e/cli/terminal.cpp b/tests/e2e/cli/terminal.cpp index 2482dae35..a4dc0706d 100644 --- a/tests/e2e/cli/terminal.cpp +++ b/tests/e2e/cli/terminal.cpp @@ -233,7 +233,7 @@ std::vector prepare_actions_alive(const fs::path& dir) { return alive; } -E2E terminal{"terminal", [] { +E2E terminal{"terminal", s::home_fixture, [] { "843 a terminal names the action that runs"_test[s::python3][s::unix_shell] = [] { // T1 the status row names a running prepare action, its package and clock; T2 the // package once, a blank line before Finished; T3 no status row after Finished; T4 the diff --git a/tests/e2e/deps/advice.cpp b/tests/e2e/deps/advice.cpp index b95165a2d..6eb47cc39 100644 --- a/tests/e2e/deps/advice.cpp +++ b/tests/e2e/deps/advice.cpp @@ -10,7 +10,7 @@ namespace fs = std::filesystem; namespace { -E2E advice{"advice", [] { +E2E advice{"advice", s::home_fixture, [] { "821 an edge states its advice on both paths"_test[s::unix_shell] = [] { // An edge that has something to say on success writes its advice file, and mcpp reports // it once after a successful build, by one function the full build path and the fast diff --git a/tests/e2e/deps/edges.cpp b/tests/e2e/deps/edges.cpp index 825f478e7..c0fee91cd 100644 --- a/tests/e2e/deps/edges.cpp +++ b/tests/e2e/deps/edges.cpp @@ -49,7 +49,7 @@ std::string reported_root(std::string_view why) { return {}; } -E2E edges{"edges", [] { +E2E edges{"edges", s::home_fixture, [] { "1001 a dependency carries its patches"_test[s::gcc] = [] { // A patched library is another library: the verified source is copied to the variant's // own tree under the mcpp home and patched there; the lock names the variant and each diff --git a/tests/e2e/deps/entry.cpp b/tests/e2e/deps/entry.cpp index 799d0d1a8..553f26bd7 100644 --- a/tests/e2e/deps/entry.cpp +++ b/tests/e2e/deps/entry.cpp @@ -55,7 +55,7 @@ int main() { } )cpp"; -E2E entry{"entry", [] { +E2E entry{"entry", s::home_fixture, [] { "946 a program begins in a dependency entry"_test[s::gcc][s::elf] = [] { // A framework supplies `main` as an entry target; the application writes only the // callback. The entry is in no image that does not select it; required features gate diff --git a/tests/e2e/deps/floors.cpp b/tests/e2e/deps/floors.cpp index 00f04acea..7b56132b5 100644 --- a/tests/e2e/deps/floors.cpp +++ b/tests/e2e/deps/floors.cpp @@ -12,7 +12,7 @@ namespace { bool failed(const e2e::outcome& r) { return r.started && r.status != 0; } -E2E floors{"floors", [] { +E2E floors{"floors", s::home_fixture, [] { "326 dependency standard floor"_test[s::gcc] = [] { // A module graph has one standard, the root's; a dependency that declared a higher one is // built at the root's, and that is reported (degraded, an error under --strict), naming @@ -84,7 +84,7 @@ E2E floors{"floors", [] { // The cases that hold the machine (P7.1): a suite of their own, which the runner runs with no // other test beside it. -E2E floors_machine{"floors-machine", resource{"machine"}, [] { +E2E floors_machine{"floors-machine", s::home_fixture, resource{"machine"}, [] { "822 a package states the oldest mcpp it supports"_test[s::unix_shell] = [] { // `[package] mcpp = ">=V"` states the oldest mcpp release a package supports: an engine // below it stops first, naming the package, the floor, its own release and the way to diff --git a/tests/e2e/deps/git.cpp b/tests/e2e/deps/git.cpp index dabe16add..8cd486952 100644 --- a/tests/e2e/deps/git.cpp +++ b/tests/e2e/deps/git.cpp @@ -40,7 +40,7 @@ void library_repository(const fs::path& dir, std::string_view name, std::string_ bool failed(const e2e::outcome& r) { return r.started && r.status != 0; } -E2E git_deps{"git", [] { +E2E git_deps{"git", s::home_fixture, [] { "24 git dependency"_test[s::fresh_sandbox] = [] { // A git dependency is cloned to $MCPP_HOME/git// and treated as a path dependency; // a rebuild does not clone again; a branch is locked (`git+...branch=main`), replayed diff --git a/tests/e2e/deps/graph.cpp b/tests/e2e/deps/graph.cpp index 72fa3d067..e4ecbaca5 100644 --- a/tests/e2e/deps/graph.cpp +++ b/tests/e2e/deps/graph.cpp @@ -13,7 +13,7 @@ namespace { bool failed(const e2e::outcome& r) { return r.started && r.status != 0; } -E2E graph{"graph", [] { +E2E graph{"graph", s::home_fixture, [] { "945 a miss anywhere in the graph asks for a refresh"_test[s::gcc][s::fresh_sandbox] = [] { // A request by version the local index cannot answer asks for a refresh wherever it is // in the graph, not only in the root's [dependencies]: declined offline, declined inside diff --git a/tests/e2e/deps/identity.cpp b/tests/e2e/deps/identity.cpp index 24a7007de..5a3cd53cd 100644 --- a/tests/e2e/deps/identity.cpp +++ b/tests/e2e/deps/identity.cpp @@ -19,7 +19,7 @@ std::optional resolution(const fs::path& root) { return std::nullopt; } -E2E identity{"identity", [] { +E2E identity{"identity", s::home_fixture, [] { "679 a path dependency takes its manifests identity"_test = [] { // A `path` or `git` dependency's identity is the one its manifest declares, whatever key // reached it: one package, compiled once, recorded with both keys; two identities over one diff --git a/tests/e2e/deps/lock.cpp b/tests/e2e/deps/lock.cpp index 2fb667e2d..b23272812 100644 --- a/tests/e2e/deps/lock.cpp +++ b/tests/e2e/deps/lock.cpp @@ -64,7 +64,7 @@ std::string widget_descriptor(std::string_view name) { )cpp", name); } -E2E lock{"lock", [] { +E2E lock{"lock", s::home_fixture, [] { "196 version identity and lock"_test[s::gcc][s::fresh_sandbox] = [] { // mcpp#363: a resolved version is an index key (an address re-rendered from parsed // numbers did not exist), and mcpp.lock records what was resolved, stably, whatever the diff --git a/tests/e2e/deps/names.cpp b/tests/e2e/deps/names.cpp index d5c939aa8..ea667d9ae 100644 --- a/tests/e2e/deps/names.cpp +++ b/tests/e2e/deps/names.cpp @@ -42,7 +42,7 @@ std::string renamed_asio(std::string_view ns, std::string_view name) { return out; } -E2E names{"names", [] { +E2E names{"names", s::home_fixture, [] { "161 xpkg name form"_test = [] { // SPEC-001 §3.2: `package.name` is a single atomic segment; the hierarchy belongs in // `package.namespace` (the legacy fully qualified `compat.zlib` keeps working). A dotted diff --git a/tests/e2e/deps/path.cpp b/tests/e2e/deps/path.cpp index 789d47ef6..d2ad536fd 100644 --- a/tests/e2e/deps/path.cpp +++ b/tests/e2e/deps/path.cpp @@ -28,7 +28,7 @@ std::string index_package(std::string_view ns, std::string_view name, std::strin )lua", ns, name, description, version); } -E2E path{"path", [] { +E2E path{"path", s::home_fixture, [] { "09 path dependency"_test[s::elf][s::gcc] = [] { // Package B imports modules of package A through `[dependencies.A] path`: the // multi-package scanner and link; a path whose manifest names another package is refused. diff --git a/tests/e2e/deps/resolve.cpp b/tests/e2e/deps/resolve.cpp index e207f21cf..dd9dbf985 100644 --- a/tests/e2e/deps/resolve.cpp +++ b/tests/e2e/deps/resolve.cpp @@ -57,7 +57,7 @@ std::string local_package(std::string_view ns, std::string_view name, std::strin )lua", ns, name, description, archive, source, target); } -E2E resolve{"resolve", [] { +E2E resolve{"resolve", s::home_fixture, [] { "32 semver merge"_test[s::fresh_sandbox] = [] { // Two consumers of one package with overlapping constraints (exact and range) merge to // one satisfying version instead of erroring out. diff --git a/tests/e2e/deps/tools.cpp b/tests/e2e/deps/tools.cpp index 424289a24..92d28c351 100644 --- a/tests/e2e/deps/tools.cpp +++ b/tests/e2e/deps/tools.cpp @@ -66,7 +66,7 @@ graph make_graph(std::string_view dep_extra, std::string_view dep_dependency, st return g; } -E2E tools{"tools", [] { +E2E tools{"tools", s::home_fixture, [] { "627 one package one version"_test[s::gcc] = [] { // A project and one of its dependencies both name `xim:zoxide`, at different versions: // one package, so one version is installed (the declaration nearer the artifact), the diff --git a/tests/e2e/features/activation.cpp b/tests/e2e/features/activation.cpp index a8a8c60a9..c86e44a88 100644 --- a/tests/e2e/features/activation.cpp +++ b/tests/e2e/features/activation.cpp @@ -26,7 +26,7 @@ e2e::outcome run_app(const s::project& p) { return s::run(*bin); } -E2E activation{"activation", [] { +E2E activation{"activation", s::home_fixture, [] { "67 features strict"_test = [] { // Feature activation, `backend=` sugar, and the --strict schema gate: an unknown feature // or platform warns, and --strict refuses; the six platform names are known. diff --git a/tests/e2e/features/conditional.cpp b/tests/e2e/features/conditional.cpp index 236962493..9a7eb71d5 100644 --- a/tests/e2e/features/conditional.cpp +++ b/tests/e2e/features/conditional.cpp @@ -27,7 +27,7 @@ std::vector ninjas(const s::project& p) { // The text of a file, or nothing when it does not exist (`grep file` then reads no line). std::string text_of(const fs::path& f) { return fs::exists(f) ? s::read_file(f) : std::string{}; } -E2E conditional{"conditional", [] { +E2E conditional{"conditional", s::home_fixture, [] { "85 target cfg build flags"_test = [] { // `[target.'cfg(...)'.build]` applies exactly the host's os/arch predicates. s::project app{s::files{{"mcpp.toml", R"cpp([package] diff --git a/tests/e2e/features/defaults.cpp b/tests/e2e/features/defaults.cpp index 80e4b2839..21cfe8af1 100644 --- a/tests/e2e/features/defaults.cpp +++ b/tests/e2e/features/defaults.cpp @@ -15,7 +15,7 @@ bool failed(const e2e::outcome& r) { return r.started && r.status != 0; } const s::options cdb{.env = {{"MCPP_COMPILE_COMMANDS", "compile_commands.json"}}}; -E2E defaults{"defaults", [] { +E2E defaults{"defaults", s::home_fixture, [] { "126 default features opt out"_test = [] { // `default-features = false` keeps a dependency's default feature (and its define) off; // the default is on otherwise; an explicit request activates it beside the opt-out. diff --git a/tests/e2e/features/layers.cpp b/tests/e2e/features/layers.cpp index 99a8de988..f9a580c6c 100644 --- a/tests/e2e/features/layers.cpp +++ b/tests/e2e/features/layers.cpp @@ -14,7 +14,7 @@ bool failed(const e2e::outcome& r) { return r.started && r.status != 0; } const s::options cdb{.env = {{"MCPP_COMPILE_COMMANDS", "compile_commands.json"}}}; -E2E layers{"layers", [] { +E2E layers{"layers", s::home_fixture, [] { "328 layer predicates apply and unknown ones say so"_test = [] { // `cfg(c-abi = ...)` applies when the resolved c-abi matches and not otherwise, once // beside a triple leg, in a dependency too; an unknown cfg key is named with the diff --git a/tests/e2e/features/sources.cpp b/tests/e2e/features/sources.cpp index 81b15391d..d7c07bd5b 100644 --- a/tests/e2e/features/sources.cpp +++ b/tests/e2e/features/sources.cpp @@ -20,7 +20,7 @@ std::optional newest_graph(const fs::path& target, std::string_view na return best; } -E2E sources{"sources", [] { +E2E sources{"sources", s::home_fixture, [] { "100 feature sources test mode"_test = [] { // A dependency's feature-gated `sources` are compiled under `mcpp test` as under // `mcpp build`, and not without the feature. diff --git a/tests/e2e/golden/protocol-version.json b/tests/e2e/golden/protocol-version.json new file mode 100644 index 000000000..b107e86b5 --- /dev/null +++ b/tests/e2e/golden/protocol-version.json @@ -0,0 +1,148 @@ +{ + "commands": { + "build": { + "effects": [ + "init-mcpp-home", + "read-project", + "write-project", + "network", + "write-global-cache", + "exec-build-script" + ] + }, + "cache list": { + "effects": [] + }, + "emit build-database": { + "effects": [ + "init-mcpp-home", + "read-project", + "network", + "write-global-cache", + "exec-build-script" + ] + }, + "index list": { + "effects": [ + "init-mcpp-home", + "read-project" + ] + }, + "pack": { + "effects": [ + "init-mcpp-home", + "read-project", + "write-project", + "network", + "write-global-cache", + "exec-build-script" + ] + }, + "pack --list-formats": { + "effects": [ + "init-mcpp-home", + "read-project", + "network", + "write-global-cache", + "exec-build-script" + ] + }, + "run --print-launch": { + "effects": [ + "init-mcpp-home", + "read-project", + "write-project", + "network", + "write-global-cache", + "exec-build-script" + ] + }, + "search": { + "effects": [ + "init-mcpp-home", + "network" + ] + }, + "self env": { + "effects": [] + }, + "test --print-launch": { + "effects": [ + "init-mcpp-home", + "read-project", + "write-project", + "network", + "write-global-cache", + "exec-build-script" + ] + }, + "toolchain list": { + "effects": [] + }, + "why sources": { + "effects": [ + "init-mcpp-home", + "read-project", + "network", + "write-global-cache", + "exec-build-script" + ] + }, + "why toolchain": { + "effects": [ + "init-mcpp-home", + "read-project", + "network", + "write-global-cache", + "exec-build-script" + ] + }, + "xpkg parse": { + "effects": [] + } + }, + "envelope": { + "max": 1, + "min": 1 + }, + "features": [ + "target-sets", + "pkg-format-list", + "test-launch", + "launch-stream", + "test-protocol", + "test-parts", + "test-case-stream", + "test-metrics", + "test-launch-records", + "test-frameworks", + "test-checks", + "test-output", + "test-steps", + "test-budget", + "test-jitter", + "test-fixtures" + ], + "kind": "mcpp.protocol", + "kinds": { + "mcpp.build": 1, + "mcpp.build-database": 1, + "mcpp.cache": 1, + "mcpp.env": 1, + "mcpp.index": 1, + "mcpp.launch": 1, + "mcpp.pack": 1, + "mcpp.pack.formats": 1, + "mcpp.search": 1, + "mcpp.toolchain.list": 1, + "mcpp.why.build-programs": 1, + "mcpp.why.compat": 1, + "mcpp.why.sources": 1, + "mcpp.why.toolchain": 1, + "mcpp.xpkg": 1 + }, + "mcpp": { + "version": "" + }, + "schemaVersion": 1 +} diff --git a/tests/e2e/golden/surface.txt b/tests/e2e/golden/surface.txt new file mode 100644 index 000000000..79bf4a596 --- /dev/null +++ b/tests/e2e/golden/surface.txt @@ -0,0 +1,429 @@ +command add +command bmi-await +command bmi-compile +command bmi-supervise +command build +command cache +command cache clean +command cache dir +command cache gc +command cache info +command cache list +command cache prune +command cache verify +command clean +command coff-def +command dyndep +command emit +command emit build-database +command emit sbom +command emit surface +command emit xpkg +command explain +command index +command index add +command index list +command index pin +command index remove +command index status +command index unpin +command index update +command link-output +command new +command pack +command place-dlls +command platform-step +command publish +command remove +command resolve +command run +command search +command self +command self config +command self doctor +command self env +command self explain +command self init +command self store +command self store dedupe +command self version +command stage +command test +command toolchain +command toolchain default +command toolchain install +command toolchain list +command toolchain remove +command update +command version +command why +command write +command xpkg +command xpkg parse +flag --deny +flag --frozen +flag --locked +flag --managed-only +flag --no-color +flag --offline +flag --protocol-version +flag --quiet +flag --system +flag --verbose +flag add --dev +flag bmi-await --object +flag bmi-await --slot +flag bmi-compile --bmi +flag bmi-compile --cap +flag bmi-compile --command-file +flag bmi-compile --dep-from +flag bmi-compile --dep-to +flag bmi-compile --object +flag bmi-compile --relocate +flag bmi-compile --self +flag bmi-compile --sem +flag bmi-compile --slot +flag bmi-supervise --cap +flag bmi-supervise --command-file +flag bmi-supervise --object +flag bmi-supervise --relocation +flag bmi-supervise --sem +flag bmi-supervise --slot +flag build --accel +flag build --cache +flag build --cap +flag build --configure-only +flag build --dev +flag build --exclude +flag build --features +flag build --host-platform +flag build --jobs +flag build --message-format +flag build --no-accel +flag build --no-cache +flag build --package +flag build --platform +flag build --play-game +flag build --print-fingerprint +flag build --profile +flag build --release +flag build --static +flag build --strict +flag build --sysroot +flag build --target +flag build --toolchain +flag build --workspace +flag cache clean --all +flag cache clean --deps +flag cache clean --legacy +flag cache clean --std +flag cache clean --tools +flag cache gc --max-size +flag cache gc --older-than +flag cache list --format +flag cache list --json +flag cache prune --older-than +flag clean --bmi-cache +flag clean --dry-run +flag clean --older-than +flag clean --report +flag clean --stale +flag coff-def --exports-file +flag coff-def --llvm-cxx +flag coff-def --llvm-nm +flag coff-def --llvm-target +flag coff-def --name +flag coff-def --output +flag coff-def --required +flag dyndep --bmi-dir +flag dyndep --bmi-ext +flag dyndep --expect-imports +flag dyndep --expect-none +flag dyndep --expect-provides +flag dyndep --module-map +flag dyndep --output +flag dyndep --single +flag dyndep --split-module +flag emit build-database --accel +flag emit build-database --cap +flag emit build-database --dev +flag emit build-database --exclude +flag emit build-database --features +flag emit build-database --format +flag emit build-database --host-platform +flag emit build-database --no-accel +flag emit build-database --output +flag emit build-database --package +flag emit build-database --platform +flag emit build-database --profile +flag emit build-database --release +flag emit build-database --spec +flag emit build-database --static +flag emit build-database --strict +flag emit build-database --sysroot +flag emit build-database --target +flag emit build-database --toolchain +flag emit build-database --workspace +flag emit sbom --output +flag emit surface --output +flag emit xpkg --namespace +flag emit xpkg --output +flag emit xpkg --package +flag emit xpkg --url +flag emit xpkg --version +flag index list --format +flag link-output --aside +flag link-output --companions +flag link-output --output +flag link-output --temporary +flag link-output --text +flag new --list-templates +flag new --template +flag pack --all-programs +flag pack --compress +flag pack --debug-symbols +flag pack --dev +flag pack --exclude +flag pack --features +flag pack --format +flag pack --host-platform +flag pack --linkage +flag pack --list-formats +flag pack --merge +flag pack --message-format +flag pack --mode +flag pack --no-strip +flag pack --no-universal +flag pack --output +flag pack --package +flag pack --pkg-format +flag pack --platform +flag pack --profile +flag pack --release +flag pack --sysroot +flag pack --systems +flag pack --target +flag pack --toolchain +flag pack --workspace +flag place-dlls --crt +flag place-dlls --depfile +flag place-dlls --output +flag place-dlls --toolset-crt +flag platform-step --out +flag platform-step --steps-hex +flag publish --allow-dirty +flag publish --dry-run +flag publish --exclude +flag publish --package +flag publish --tag +flag publish --workspace +flag resolve --explain +flag run --accel +flag run --cache +flag run --dev +flag run --features +flag run --file +flag run --format +flag run --host-platform +flag run --list-runners +flag run --message-format +flag run --no-accel +flag run --no-cache +flag run --no-runner +flag run --package +flag run --pkg-format +flag run --platform +flag run --play-game +flag run --print-launch +flag run --profile +flag run --release +flag run --runner +flag run --sysroot +flag run --target +flag run --target-triple +flag run --toolchain +flag search --all-versions +flag search --format +flag self config --cache-unused-days +flag self config --keep-days +flag self config --mirror +flag self env --format +flag self env --host +flag self env --host-platform +flag self env --platform +flag self env --target +flag self init --force +flag self init --home +flag self store dedupe --apply +flag stage --list +flag stage --output +flag stage --verify +flag test --accel +flag test --archive +flag test --build-timeout +flag test --cache +flag test --cap +flag test --case +flag test --cases +flag test --changed +flag test --exclude +flag test --failed +flag test --features +flag test --guide +flag test --host-platform +flag test --list +flag test --message-format +flag test --no-accel +flag test --no-cache +flag test --no-programs +flag test --no-run +flag test --no-runner +flag test --package +flag test --platform +flag test --play-game +flag test --print-launch +flag test --profile +flag test --runner +flag test --seed +flag test --shard +flag test --since +flag test --split +flag test --strict +flag test --sysroot +flag test --target +flag test --timeout +flag test --toolchain +flag test --workspace +flag test --workspace-timeout +flag toolchain default --target +flag toolchain install --platform +flag toolchain install --target +flag toolchain list --format +flag toolchain remove --target +flag why --features +flag why --format +flag why --host-platform +flag why --platform +flag why --sysroot +flag why --target +flag why --toolchain +flag write --append +flag write --output +flag xpkg parse --all-os +flag xpkg parse --allow-split-name +flag xpkg parse --allow-unknown +flag xpkg parse --format +flag xpkg parse --json +key build-dependencies +key build.accel +key build.allow_host_libs +key build.bmi_schedule +key build.build_program_timeout +key build.c_standard +key build.cache +key build.cflags +key build.compile_commands +key build.compiler_runtime_dir +key build.cxx_runtime +key build.cxxflags +key build.default-profile +key build.defines +key build.dependency_linkage +key build.dialect_cxxflags +key build.flags +key build.include_dirs +key build.include_dirs_after +key build.ios_deployment_target +key build.jobs +key build.ldflags +key build.macos_deployment_target +key build.memory_threshold +key build.module_extensions +key build.platform-dependencies +key build.private_fragment +key build.private_include_dirs +key build.profile +key build.sources +key build.static_stdlib +key build.std-compat-module +key build.std-module +key build.std-module-flags +key build.target +key build.target_budget +key dependencies +key dev-dependencies +key features +key hooks +key host +key indices +key package +key platform +key profile. +key resources +key runtime +key target..abi +key target..carrier +key target..cxx_runtime +key target..dependencies +key target..layers +key target..link +key target..linkage +key target..min_api_level +key target..platform +key target..run_under +key target..runner +key target..runtime +key target..sysroot +key target..targets +key target..toolchain +key target..tools +key target..xlings +key targets +key test +key toolchain +key xlings +reason accel-backend-undeclared +reason accel-mismatch +reason ambiguous-request +reason apple-sdk-absent +reason c-env-unrealisable +reason c-env-verification-mismatch +reason capability-pin +reason compiler-requirement-conflict +reason convention-unreplaced +reason crt-declared-under-host-coupled +reason device-source-unconsumed +reason exclusive-capability +reason host-cannot-serve +reason host-module-missing +reason host-tool-toolchain +reason interface-not-provided +reason layer-cycle +reason layer-ordering +reason layer-requirement +reason lld-required-absent +reason local-toolchain +reason lto-export-discovery +reason managed-only +reason msvc-redist-unavailable +reason offline-download-required +reason os-mismatch +reason other +reason package-cycle +reason payload-override +reason payload-request +reason platform-dependency +reason platform-not-a-platform +reason platform-phase-directive +reason platform-target-unserved +reason platform-tool-absent +reason platform-version-group +reason program-cxx-runtime-split +reason shared-library-cxx-runtime +reason static-package-in-two-images +reason std-module-precompile +reason target-base +reason tier-planned +reason tool-version-conflict +reason unknown-target +reason version-floor-unmet diff --git a/tests/e2e/index/listing.cpp b/tests/e2e/index/listing.cpp index 6ff3a2ffd..f96194566 100644 --- a/tests/e2e/index/listing.cpp +++ b/tests/e2e/index/listing.cpp @@ -52,7 +52,7 @@ void pin_and_unpin(const s::project& p, const s::options& o) { expect(fatal(any_of_words(out, {"Unpinned", "unpinned", "Unpin", "no rev"}))) << "unpin output unexpected: " << out; } -E2E listing{"listing", [] { +E2E listing{"listing", s::home_fixture, [] { "06 emit xpkg"_test = [] { // `emit xpkg` writes a Lua entry from the manifest: the header, the name and the // pointer to the manifest in the archive (SPEC-001 §3.5); `--output` writes the file, diff --git a/tests/e2e/index/local.cpp b/tests/e2e/index/local.cpp index 06fc8b3c4..fe9ae5207 100644 --- a/tests/e2e/index/local.cpp +++ b/tests/e2e/index/local.cpp @@ -72,7 +72,7 @@ std::string c_lib_segment(std::string_view source, std::string_view target) { }},)", source, target); } -E2E local{"local", [] { +E2E local{"local", s::home_fixture, [] { "52 local path namespaced index"_test[s::gcc][s::fresh_sandbox] = [] { // A local path index finds pkgs/c/compat.cfg.lua; the project builds and runs. A clean // project installs it through the NDJSON interface with the qualified name diff --git a/tests/e2e/index/refresh.cpp b/tests/e2e/index/refresh.cpp index 0f73d1030..a2f608495 100644 --- a/tests/e2e/index/refresh.cpp +++ b/tests/e2e/index/refresh.cpp @@ -57,7 +57,7 @@ bool has_word(const nlohmann::json& list, std::string_view word) { return false; } -E2E refresh{"refresh", [] { +E2E refresh{"refresh", s::home_fixture, [] { "173 index refresh policy"_test[s::gcc][s::fresh_sandbox] = [] { // A 30-day-old marker refreshes nothing when the index answers (an exact version and a // constraint); offline, a miss says so and names the revision and `mcpp index update`; diff --git a/tests/e2e/index/routes.cpp b/tests/e2e/index/routes.cpp index 10b9deace..1e099d52b 100644 --- a/tests/e2e/index/routes.cpp +++ b/tests/e2e/index/routes.cpp @@ -39,7 +39,7 @@ std::string lib_package(std::string_view ns_line, std::string_view name, std::st )", ns_line, name, description, source, target); } -E2E routes{"routes", [] { +E2E routes{"routes", s::home_fixture, [] { "120 ws root indices"_test[s::gcc][s::fresh_sandbox] = [] { // [indices] and [workspace.dependencies] at a workspace root reach a member by their // position, with a warning naming [workspace.indices]; a member's own relative path is diff --git a/tests/e2e/index/xlings.cpp b/tests/e2e/index/xlings.cpp index a62939d36..16c3daf4e 100644 --- a/tests/e2e/index/xlings.cpp +++ b/tests/e2e/index/xlings.cpp @@ -16,7 +16,7 @@ const s::options no_auto{.env = {{"MCPP_NO_AUTO_INSTALL", "1"}}}; bool failed(const e2e::outcome& r) { return r.started && r.status != 0; } -E2E xlings{"xlings", [] { +E2E xlings{"xlings", s::home_fixture, [] { "327 declared xlings deps report their own failure"_test[s::unix_shell] = [] { // An unprovisionable [xlings] dependency: MCPP_NO_AUTO_INSTALL refuses naming the knob // and the package (announcing no provisioning); MCPP_OFFLINE refuses naming the offline @@ -223,7 +223,7 @@ hardware = {} // The cases that hold the machine (P7.1): a suite of their own, which the runner runs with no // other test beside it. -E2E xlings_machine{"xlings-machine", resource{"machine"}, [] { +E2E xlings_machine{"xlings-machine", s::home_fixture, resource{"machine"}, [] { "614 feature xlings reaches xpkg dir"_test[s::gcc][s::unix_shell] = [] { // A [feature-xlings.] tool's MCPP_XPKG___DIR is its payload with the feature // on and empty with it off. diff --git a/tests/e2e/macos/rows.cpp b/tests/e2e/macos/rows.cpp index d1d0dc40c..a7b70f2e9 100644 --- a/tests/e2e/macos/rows.cpp +++ b/tests/e2e/macos/rows.cpp @@ -70,7 +70,7 @@ std::vector section_words(const std::string& dump) { return words; } -E2E rows{"rows", [] { +E2E rows{"rows", s::home_fixture, [] { "259 shared library macho"_test[s::macos] = [] { // A Mach-O shared library's install name is @rpath/, not the build directory, so // it loads where it is extracted: packed, the build tree removed, a consumer builds and diff --git a/tests/e2e/modules/basics.cpp b/tests/e2e/modules/basics.cpp index dbc8f454c..e9a9a11f6 100644 --- a/tests/e2e/modules/basics.cpp +++ b/tests/e2e/modules/basics.cpp @@ -61,7 +61,7 @@ kind = "bin" main = "src/main.cpp" )"; -E2E basics{"basics", [] { +E2E basics{"basics", s::home_fixture, [] { "03 multi module"_test[s::elf][s::gcc] = [] { // A package's module and main build and run; the graph names its BMI and the std BMI. auto p = s::project::scaffold("multi"); diff --git a/tests/e2e/modules/bmi.cpp b/tests/e2e/modules/bmi.cpp index df404e62e..bffa0f8c3 100644 --- a/tests/e2e/modules/bmi.cpp +++ b/tests/e2e/modules/bmi.cpp @@ -100,7 +100,7 @@ std::string lua_xpm(std::string_view name) { return out; } -E2E bmi{"bmi", [] { +E2E bmi{"bmi", s::home_fixture, [] { "19 bmi cache reuse"_test = [] { // A path dependency outside the project, direct or transitive, is kept in the build // cache under the digest of its tree (mcpp#974): a clean build takes it from there, an @@ -396,7 +396,7 @@ int main() { // The cases that hold the machine (P7.1): a suite of their own, which the runner runs with no // other test beside it. -E2E bmi_machine{"bmi-machine", resource{"machine"}, [] { +E2E bmi_machine{"bmi-machine", s::home_fixture, resource{"machine"}, [] { "170 bmi staging no cascade"_test = [] { // The std BMI is staged from the home's cache; a re-staged, unchanged BMI does not // recompile its importer, and a settled staging edge stays clean. diff --git a/tests/e2e/modules/extensions.cpp b/tests/e2e/modules/extensions.cpp index 0a6eba845..2a2f22478 100644 --- a/tests/e2e/modules/extensions.cpp +++ b/tests/e2e/modules/extensions.cpp @@ -66,7 +66,7 @@ s::project extfix(const fs::path& dir, std::string_view extra) { {"src/d.ixx", "export module extfix.d;\nimport std;\nexport auto d() -> void { std::println(\"from .ixx\"); }\n"}}); } -E2E extensions{"extensions", [] { +E2E extensions{"extensions", s::home_fixture, [] { "217 module extensions"_test[s::gcc] = [] { // Undeclared, .ccm/.cxxm/.ixx are not compiled and the default glob is unchanged; // declared, they join the glob, take the module rule and their own object names, and @@ -191,11 +191,13 @@ module_extensions = [".ixx"] } // namespace -// The cases that hold the machine (P7.1): a suite of their own, which the runner runs with no -// other test beside it. 218 asserts a replay: an install into the shared home by a program beside -// it changes the runtime's description, and the build then plans again (design 2026-11-14 §5.2 E). -E2E extensions_machine{"extensions-machine", resource{"machine"}, [] { +// 218 asserts a replay: an install into the shared home by a program beside it changes the +// runtime's description, and the build then plans again (design 2026-11-14 §5.2 E). It runs mcpp +// in a home of its own whose payloads are the user's, as upstreams (`own_home`, design 2026-11-20 +// §5.10), instead of holding the machine. +E2E extensions_own_home{"extensions-own-home", s::home_fixture, [] { "218 module extensions graph shape"_test[s::gcc] = [] { + s::own_home(); // A no-change build replays; a new import in a .ixx plans again and the edge is real; // module_extensions changes the graph in the same directory; a dead entry is reported; a // reserved extension is refused naming the key. diff --git a/tests/e2e/modules/programs.cpp b/tests/e2e/modules/programs.cpp index c4d5d0cfd..ccc0c99a3 100644 --- a/tests/e2e/modules/programs.cpp +++ b/tests/e2e/modules/programs.cpp @@ -49,7 +49,7 @@ bool places(const fs::path& dir, std::string_view obj, std::string_view entry) { return static_cast(s::has_line(host_graph(dir), std::format(R"(^build obj/[^ ]*/{} : stage_file [^ ]*/build-cache/v1/pkg/{})", obj, entry))); } -E2E programs{"programs", [] { +E2E programs{"programs", s::home_fixture, [] { "807 a lib root imports its own package"_test[s::gcc] = [] { // A host-module dependency whose lib root, explicit or by convention, imports a unit of // its own package builds, and the build program runs. diff --git a/tests/e2e/modules/scanner.cpp b/tests/e2e/modules/scanner.cpp index 8a3a775fc..7307f28a9 100644 --- a/tests/e2e/modules/scanner.cpp +++ b/tests/e2e/modules/scanner.cpp @@ -31,7 +31,7 @@ std::optional program(const s::project& p, std::string_view name) { return std::nullopt; } -E2E scanner{"scanner", [] { +E2E scanner{"scanner", s::home_fixture, [] { "113 scanner mcpp dir prune"_test[s::gcc] = [] { // `.mcpp` is pruned from a glob walk: a file reachable only through its index link is not // compiled; a file whose name has no narrow spelling does not abort the build. diff --git a/tests/e2e/openkal/bases.cpp b/tests/e2e/openkal/bases.cpp index 5d97f42f3..42fe5ac4a 100644 --- a/tests/e2e/openkal/bases.cpp +++ b/tests/e2e/openkal/bases.cpp @@ -89,7 +89,7 @@ std::optional artifact(const s::project& p) { const s::options plain{.env = {{"NO_COLOR", "1"}}}; -E2E bases{"bases", [] { +E2E bases{"bases", s::home_fixture, [] { "925 a stated sysroot is the c library"_test[s::gcc][s::elf][s::unix_shell][s::python3] = [] { // S2: a sysroot named by a relative path is the base the artifact is built against and // runs on (PT_INTERP and RUNPATH the sysroot's, not the SubOS farm), in a directory of diff --git a/tests/e2e/openkal/images.cpp b/tests/e2e/openkal/images.cpp index 14cbb5386..6893e1a9b 100644 --- a/tests/e2e/openkal/images.cpp +++ b/tests/e2e/openkal/images.cpp @@ -163,7 +163,7 @@ __attribute__((no_stack_protector)) int ok_main(void) { constexpr std::string_view caught_main = "import std;\nint main(int argc, char** argv) {\n try { throw std::runtime_error(\"caught\"); }\n catch (const std::exception& e) { std::println(\"image: {} ({} args)\", e.what(), argc); }\n return 0;\n}\n"; constexpr std::string_view smoke_test = "import std;\nint main() { std::vector v{2, 1}; std::ranges::sort(v); return v.front() == 1 ? 0 : 1; }\n"; -E2E images{"images", [] { +E2E images{"images", s::home_fixture, [] { "942 an ok image is built once and run by okrun"_test[s::llvm][s::python3] = [] { // A both rows publish bin/app.oke, an ET_DYN of their architecture, and no unchecked // image; B okrun runs this machine's image with its arguments, environment, a callback, diff --git a/tests/e2e/openkal/layers.cpp b/tests/e2e/openkal/layers.cpp index 3a7f8cdfc..d456b70c8 100644 --- a/tests/e2e/openkal/layers.cpp +++ b/tests/e2e/openkal/layers.cpp @@ -27,7 +27,7 @@ std::vector words_of(const nlohmann::json& entry) { return out; } -E2E layers{"layers", [] { +E2E layers{"layers", s::home_fixture, [] { "305 built from is not published"_test[s::unix_shell][s::jq] = [] { // `private_include_dirs` (a subset of `include_dirs`, globs expanded before the filter) // build the provider and do not reach the consumer: the provider's units carry its diff --git a/tests/e2e/openkal/machine.cpp b/tests/e2e/openkal/machine.cpp index 117702733..b26247860 100644 --- a/tests/e2e/openkal/machine.cpp +++ b/tests/e2e/openkal/machine.cpp @@ -26,7 +26,7 @@ bool locked_as_platforms(std::string_view lock, std::string_view name) { return false; } -E2E machine{"machine", resource{"machine"}, [] { +E2E machine{"machine", s::home_fixture, resource{"machine"}, [] { "951 an ok image needs no declaration"_test[s::llvm][s::python3] = [] { // The target table pins openkal-platform for the ok-image rows: A a project naming only // its sources runs as an image; B the report names the platform and what it added; C diff --git a/tests/e2e/openkal/platforms.cpp b/tests/e2e/openkal/platforms.cpp index 5148f52d7..de83cf0f8 100644 --- a/tests/e2e/openkal/platforms.cpp +++ b/tests/e2e/openkal/platforms.cpp @@ -52,7 +52,7 @@ std::string tail_bytes(const fs::path& f, std::size_t n) { return b.size() <= n ? b : b.substr(b.size() - n); } -E2E platforms{"platforms", [] { +E2E platforms{"platforms", s::home_fixture, [] { "952 a platform package states a class of targets"_test[s::unix_shell][s::python3] = [] { // A local platform for this machine's triples: A its flag, program name, seal step and // runner reach the program; B the report names it and its source; C an override under diff --git a/tests/e2e/openkal/sets.cpp b/tests/e2e/openkal/sets.cpp index 442f8234a..e929bbe6d 100644 --- a/tests/e2e/openkal/sets.cpp +++ b/tests/e2e/openkal/sets.cpp @@ -74,7 +74,7 @@ constexpr std::string_view smoke_test = "import std;\nint main() { std::vector/ with one Finished; B the // JSON envelope is the first leg's record with both legs, each artifact event naming its diff --git a/tests/e2e/openkal/stack.cpp b/tests/e2e/openkal/stack.cpp index 5bd5b7522..ef81f532a 100644 --- a/tests/e2e/openkal/stack.cpp +++ b/tests/e2e/openkal/stack.cpp @@ -37,7 +37,7 @@ bool packages_unreachable(std::string_view out) { return out.contains("not found in the synced index") || out.contains("install_packages failed"); } -E2E stack{"stack", [] { +E2E stack{"stack", s::home_fixture, [] { "269 openkal llvm spelling still resolves"_test[s::import_std_libcxx] = [] { // `openkal-llvm@` resolves to the driver `llvm@` does, and decides nothing of the // target side: with an empty graph every layer is the payload's. diff --git a/tests/e2e/openkal/trees.cpp b/tests/e2e/openkal/trees.cpp index 624b9dbb1..263495c35 100644 --- a/tests/e2e/openkal/trees.cpp +++ b/tests/e2e/openkal/trees.cpp @@ -102,7 +102,7 @@ constexpr std::string_view caught_main = "import std;\nint main(int argc, char** // The openkal set built for every architecture: about 20 minutes beside three other programs on a // Linux runner, measured 2026-11-14; stated, so the run's limit is not raised for every test. -E2E trees{"trees", timeout{std::chrono::minutes{40}}, [] { +E2E trees{"trees", s::home_fixture, timeout{std::chrono::minutes{40}}, [] { "955 the openkal set is one tree of every architecture"_test[s::llvm][s::python3][s::unix_shell] = [] { // A `pack --target openkal` writes --openkal/ with both images, the // launchers and metadata naming legs, runner and entries; B the launcher runs this diff --git a/tests/e2e/pack/formats.cpp b/tests/e2e/pack/formats.cpp index 2af090510..2a2085cd9 100644 --- a/tests/e2e/pack/formats.cpp +++ b/tests/e2e/pack/formats.cpp @@ -61,7 +61,7 @@ std::string trimmed(std::string s) { return s; } -E2E formats{"formats", [] { +E2E formats{"formats", s::home_fixture, [] { "638 pack format dispatch"_test[s::gcc] = [] { // `--pkg-format ` dispatches to a format the resolved graph provides; a plain // build's graph is dist=none and makes no distributable; an unknown format is refused diff --git a/tests/e2e/pack/library.cpp b/tests/e2e/pack/library.cpp index 81dc23795..1ec2358a1 100644 --- a/tests/e2e/pack/library.cpp +++ b/tests/e2e/pack/library.cpp @@ -67,7 +67,7 @@ bool has_symtab(const fs::path& f) { return false; } -E2E library{"library", [] { +E2E library{"library", s::home_fixture, [] { "242 pack library interface and headers"_test = [] { // A library package carries its module interface and its headers; a consumer by // header, by module and by both at once builds and runs against it. --compress zstd diff --git a/tests/e2e/pack/members.cpp b/tests/e2e/pack/members.cpp index 36732023f..7cc804901 100644 --- a/tests/e2e/pack/members.cpp +++ b/tests/e2e/pack/members.cpp @@ -84,7 +84,7 @@ bool compiled_nothing(const fs::path& dir) { return true; } -E2E members{"members", [] { +E2E members{"members", s::home_fixture, [] { "249 pack workspace root unchanged"_test = [] { // `mcpp pack` in a virtual workspace root still packs the member's program. s::project ws{s::files{{"mcpp.toml", R"cpp([workspace] diff --git a/tests/e2e/pack/modes.cpp b/tests/e2e/pack/modes.cpp index 00ee4d939..4824be036 100644 --- a/tests/e2e/pack/modes.cpp +++ b/tests/e2e/pack/modes.cpp @@ -79,7 +79,7 @@ bool has_exact_line(const fs::path& f, std::string_view line) { return std::rang // A program's standard output and error, run with no injected environment. std::string output_of(const fs::path& program) { return s::text(s::run(program)); } -E2E modes{"modes", [] { +E2E modes{"modes", s::home_fixture, [] { "30 pack modes"_test[s::pack][s::patchelf][s::elf] = [] { // `mcpp pack` in each mode, read from the extracted archive: the default (vendored) // repoints PT_INTERP and bundles nothing for a dependency-free program; static (where diff --git a/tests/e2e/run/contract.cpp b/tests/e2e/run/contract.cpp index e1c24ebe6..e1fa8bf5f 100644 --- a/tests/e2e/run/contract.cpp +++ b/tests/e2e/run/contract.cpp @@ -55,7 +55,7 @@ std::string list_of(const fs::path& rec) { bool received(const fs::path& rec) { return fs::exists(rec) && s::no_line(s::read_file(rec), "^UNSET$"); } -E2E contract{"contract", [] { +E2E contract{"contract", s::home_fixture, [] { "672 a runner receives the files the artifact carries"_test[s::unix_shell] = [] { // A runner receives MCPP_RUNTIME_FILES, a list of `\t` // naming the deployed files and the dependency's shared library, for `run` and for each @@ -265,12 +265,13 @@ int main() {{ }; }}; -// The cases that hold the machine (P7.1): a suite of their own, which the runner runs with no -// other test beside it. -E2E contract_machine{"contract-machine", resource{"machine"}, [] { +// The cases that run mcpp in a home of their own whose payloads are the user's, as upstreams +// (`own_home`, design 2026-11-20 §5.10), instead of holding the machine. +E2E contract_own_home{"contract-own-home", s::home_fixture, [] { // It asserts a replay: an install into the shared home by a program beside it changes the // runtime's description, and the full path then plans again (design 2026-11-14 §5.2 E). "1024 a launch description takes the fast path"_test[s::python3] = [] { + s::own_home(); // A: `run --print-launch` of an unchanged project takes the fast path where `run` does, // and describes what the full path does (plus the program's arguments). B: after a // change, the description is the same. C: a launch record carrying a foreign field is not @@ -348,6 +349,7 @@ E2E contract_machine{"contract-machine", resource{"machine"}, [] { }; "1104 a run's files are in its working directory"_test[s::unix_shell] = [] { + s::own_home(); // Design 2026-11-16 §5: `--file FROM[=TO]` copies FROM into a directory of the build tree, // where the program runs; the directory is removed after it; without --file the program // runs where mcpp was started. Through a runner, the files are listed in @@ -388,6 +390,7 @@ E2E contract_machine{"contract-machine", resource{"machine"}, [] { }; "927 the runner contract"_test[s::gcc][s::unix_shell] = [] { + s::own_home(); // run_under composes in front of the artifact with the contract variables and a closure // manifest, returns the program's status and states where it was declared; the // environment's `..._RUN_UNDER=none` and --no-runner drop it; config.toml's run_under is @@ -440,7 +443,8 @@ exec "$@" expect(fatal(e2e::exited(r, 7) && !fs::exists(record))) << "--no-runner did not drop run_under"; manifest(""); - const auto config = s::shared_home() / "config.toml"; + // The case's own home (own_home): the machine's configuration is not the case's to edit. + const auto config = s::case_home(s::home::inherited) / "config.toml"; const auto saved = s::read_file(config); { std::ofstream out(config, std::ios::app); diff --git a/tests/e2e/run/environment.cpp b/tests/e2e/run/environment.cpp index cf0038113..d28c9b970 100644 --- a/tests/e2e/run/environment.cpp +++ b/tests/e2e/run/environment.cpp @@ -59,7 +59,7 @@ int main() {{ )", base, base + 1, base + 2); } -E2E environment{"environment", [] { +E2E environment{"environment", s::home_fixture, [] { "62 runtime library dirs"_test[s::gcc][s::unix_shell] = [] { // [runtime] library_dirs is a run path of the program (-Wl,-rpath), not a link search // directory (-L), and a library dlopen()ed from it is found by `run` and `test`. diff --git a/tests/e2e/run/program.cpp b/tests/e2e/run/program.cpp index 56498a3c5..a8b957b5a 100644 --- a/tests/e2e/run/program.cpp +++ b/tests/e2e/run/program.cpp @@ -30,7 +30,7 @@ std::string host_triple(const s::project& p) { return triples.front(); } -E2E program{"program", [] { +E2E program{"program", s::home_fixture, [] { "339 run reports the program status"_test[s::gcc][s::unix_shell] = [] { // `run` exits with the program's status (127 too, without an error line of its own); a // runner that cannot be executed exits 126 saying why; a missing runner exits 2 or 127. diff --git a/tests/e2e/run/runners.cpp b/tests/e2e/run/runners.cpp index d1f737199..f4978bd66 100644 --- a/tests/e2e/run/runners.cpp +++ b/tests/e2e/run/runners.cpp @@ -33,7 +33,7 @@ bool spoiled(const fs::path& file) { return bytes.size() > 19 && static_cast(bytes[18]) == 0xff && static_cast(bytes[19]) == 0xff; } -E2E runners{"runners", [] { +E2E runners{"runners", s::home_fixture, [] { "330 runner hosted targets"_test[s::unix_shell] = [] { // A runner declared under [target.] runs the artifact (argv[0] the artifact), on // the fast path too; --no-runner bypasses it with a note; a missing runner exits 2 naming diff --git a/tests/e2e/runtime/binding.cpp b/tests/e2e/runtime/binding.cpp index cd3c99dca..24aef6ebd 100644 --- a/tests/e2e/runtime/binding.cpp +++ b/tests/e2e/runtime/binding.cpp @@ -55,7 +55,7 @@ std::string with_following(std::string_view text, std::string_view pattern, std: return out; } -E2E binding{"binding", [] { +E2E binding{"binding", s::home_fixture, [] { "206 runtime binding physics"_test[s::elf][s::gcc] = [] { // A host library linked under `allow_host_libs` builds and its verdict is what the // artifact's own search path decides; a hot no-op neither relinks nor re-probes; the @@ -382,12 +382,14 @@ int main() { }; }}; -// The cases that hold the machine (P7.1): a suite of their own, which the runner runs with no -// other test beside it. Each states that a build is replayed, and a build is planned again when -// the runtime environment description changed: another program installing into the shared home -// beside it rewrote that description (measured on Linux CI, 2026-11-14). -E2E binding_machine{"binding-machine", resource{"machine"}, [] { +// The cases whose assertion holds only while nothing else writes their home (design 2026-11-20 +// §5.10): each states that a build is replayed, and a build is planned again when the runtime +// environment description changed -- another program installing into the shared home beside it +// rewrote that description (measured on Linux CI, 2026-11-14). Each runs mcpp in a home of its own +// whose payloads are the user's, as upstreams (`own_home`), instead of holding the machine. +E2E binding_own_home{"binding-own-home", s::home_fixture, [] { "200 subos env reaches program"_test[s::elf] = [] { + s::own_home(); // One selected RuntimeBinding reaches the full-path and the cached `run`; the legacy // MCPP_SUBOS_DIR is ignored; a changed contract, an aged cache and no selection are // each taken as they are; a SubOS that cannot answer the contract is refused. diff --git a/tests/e2e/runtime/contract.cpp b/tests/e2e/runtime/contract.cpp index f1f53e1af..b884abe99 100644 --- a/tests/e2e/runtime/contract.cpp +++ b/tests/e2e/runtime/contract.cpp @@ -78,7 +78,7 @@ bool defines(const fs::path& f, std::string_view sym) { return std::ranges::any_of(dynsyms(f), [&](const auto& e) { return e.defined && e.name == sym; }); } -E2E contract{"contract", [] { +E2E contract{"contract", s::home_fixture, [] { "183 cxx runtime contract"_test = [] { // A global object touching std::cout during static initialisation runs, as a program // and as a test; `host-coupled` (and `static_stdlib = false`) reaches both roles; an diff --git a/tests/e2e/runtime/deploy.cpp b/tests/e2e/runtime/deploy.cpp index d294d2a15..e225e8aa6 100644 --- a/tests/e2e/runtime/deploy.cpp +++ b/tests/e2e/runtime/deploy.cpp @@ -34,7 +34,7 @@ std::vector matching(std::string_view text, std::string_view patter return out; } -E2E deploy{"deploy", [] { +E2E deploy{"deploy", s::home_fixture, [] { "646 runtime deploy places files in a directory"_test = [] { // The root's entries land at bin// (`.` beside the executable), a dependency's // `from` is its own, a test binary finds the layout, two different sources for one diff --git a/tests/e2e/runtime/machine.cpp b/tests/e2e/runtime/machine.cpp index 9fb7117e9..3a5c14f0a 100644 --- a/tests/e2e/runtime/machine.cpp +++ b/tests/e2e/runtime/machine.cpp @@ -24,7 +24,7 @@ std::vector split(std::string_view text, char sep) { return out; } -E2E machine{"machine", resource{"machine"}, [] { +E2E machine{"machine", s::home_fixture, resource{"machine"}, [] { "219 runtime search farm is last"_test[s::gcc][s::elf][s::python3] = [] { const auto tmp = s::case_dir(); // A dependency that ships a shared library, so that the program carries `$ORIGIN`. diff --git a/tests/e2e/runtime/packages.cpp b/tests/e2e/runtime/packages.cpp index c274d821e..01274df1c 100644 --- a/tests/e2e/runtime/packages.cpp +++ b/tests/e2e/runtime/packages.cpp @@ -80,7 +80,7 @@ std::string captured(const e2e::outcome& r) { return t; } -E2E packages{"packages", [] { +E2E packages{"packages", s::home_fixture, [] { "55 dependency shared artifact"_test[s::elf] = [] { // A dependency's shared library is built as its own artifact and linked by the consumer, // not flattened into it as objects. diff --git a/tests/e2e/runtime/records.cpp b/tests/e2e/runtime/records.cpp index cbf3db0d1..f0cdfe1a9 100644 --- a/tests/e2e/runtime/records.cpp +++ b/tests/e2e/runtime/records.cpp @@ -63,7 +63,7 @@ std::string paths_in(const fs::path& file, std::string_view name) { return out; } -E2E records{"records", [] { +E2E records{"records", s::home_fixture, [] { "66 runtime provides"_test = [] { // Only an explicit `provides` owns a capability; requiring it never makes the requester // its provider, and an override cannot promote it. diff --git a/tests/e2e/self/config.cpp b/tests/e2e/self/config.cpp index 7d1f87512..ef42fcca2 100644 --- a/tests/e2e/self/config.cpp +++ b/tests/e2e/self/config.cpp @@ -25,7 +25,7 @@ nlohmann::json mcpplibs_entry(const fs::path& xj) { fail("no mcpplibs entry in " + xj.string()); } -E2E config{"config", [] { +E2E config{"config", s::home_fixture, [] { "10 env command"_test = [] { // `self env` does not initialise a cold home; `self init --offline` lays it out (bin, // registry, build-cache/v1, cache, config.toml, a seeded .xlings.json with mcpplibs and diff --git a/tests/e2e/self/homes.cpp b/tests/e2e/self/homes.cpp index 051569b37..9ddc9e429 100644 --- a/tests/e2e/self/homes.cpp +++ b/tests/e2e/self/homes.cpp @@ -24,7 +24,7 @@ std::map> snapshot(const fs::path& root) return out; } -E2E homes{"homes", [] { +E2E homes{"homes", s::home_fixture, [] { "687 the vendored xlings probe is an argument vector"_test = [] { // A: the vendored xlings's version probe runs without a shell. B: a vendored binary // answering an older version (ninja stands in) is replaced from MCPP_VENDORED_XLINGS. diff --git a/tests/e2e/self/machine.cpp b/tests/e2e/self/machine.cpp index 9ef4edaca..b20dc9175 100644 --- a/tests/e2e/self/machine.cpp +++ b/tests/e2e/self/machine.cpp @@ -23,7 +23,7 @@ std::string field11(std::string_view out) { return last; } -E2E machine{"machine", resource{"machine"}, [] { +E2E machine{"machine", s::home_fixture, resource{"machine"}, [] { "205 root local subos"_test[s::elf][s::gcc] = [] { // The default SubOS's binding ignores XLINGS_ACTIVE_SUBOS, and an explicit "default" is // not its absence; a workspace root owns the members' SubOS (materialised at the root diff --git a/tests/e2e/self/subos.cpp b/tests/e2e/self/subos.cpp index 562d59d9d..2cb1fa113 100644 --- a/tests/e2e/self/subos.cpp +++ b/tests/e2e/self/subos.cpp @@ -27,7 +27,7 @@ void rewrite(const fs::path& file, const std::string& pattern) { dir.write(name, std::regex_replace(dir.read(name), std::regex{pattern}, "\n")); } -E2E subos{"subos", [] { +E2E subos{"subos", s::home_fixture, [] { "221 subos without info still builds"_test = [] { // A project SubOS that does not describe itself costs at most hermeticity (never "cannot // provide a RuntimeBinding"), builds with allow_host_libs and reports the degradation; a diff --git a/tests/e2e/targets/abi.cpp b/tests/e2e/targets/abi.cpp index 529680283..d11c4c284 100644 --- a/tests/e2e/targets/abi.cpp +++ b/tests/e2e/targets/abi.cpp @@ -40,7 +40,7 @@ std::string emscripten_abi_table(std::string_view member) { return std::format("[target.'cfg(os = \"emscripten\")'.abi]\n{} = true\n", member); } -E2E abi{"abi", [] { +E2E abi{"abi", s::home_fixture, [] { "647 abi threads is one switch for the graph"_test[s::elf] = [] { // `[target..abi] threads = true` in the root is one switch: -pthread reaches a // dependency's compile command and the fingerprint moves; a table scoped to another diff --git a/tests/e2e/targets/graph.cpp b/tests/e2e/targets/graph.cpp index 461774ad0..97670e945 100644 --- a/tests/e2e/targets/graph.cpp +++ b/tests/e2e/targets/graph.cpp @@ -37,7 +37,7 @@ bool unreachable(std::string_view out) { return out.contains("not found in the synced index") || out.contains("install_packages failed"); } -E2E graph{"graph", [] { +E2E graph{"graph", s::home_fixture, [] { "291 dynamic linkage is refused only when the c library is the graphs"_test[s::gcc][s::llvm][s::elf][s::unix_shell] = [] { // `linkage = "dynamic"` is honoured without a word when the C library is the payload's // (the artifact has DT_NEEDED entries), and reported as having no effect when the graph diff --git a/tests/e2e/targets/machine.cpp b/tests/e2e/targets/machine.cpp index 176c063f0..c597bfaca 100644 --- a/tests/e2e/targets/machine.cpp +++ b/tests/e2e/targets/machine.cpp @@ -11,7 +11,7 @@ namespace fs = std::filesystem; namespace { -E2E machine{"machine", resource{"machine"}, [] { +E2E machine{"machine", s::home_fixture, resource{"machine"}, [] { "28 target static"_test[s::musl][s::elf] = [] { // The musl toolchain is the one this home has, whichever release that is. const auto musl_version = s::musl_gcc_version(); diff --git a/tests/e2e/targets/requests.cpp b/tests/e2e/targets/requests.cpp index 0553d8ab1..d3dc9a2da 100644 --- a/tests/e2e/targets/requests.cpp +++ b/tests/e2e/targets/requests.cpp @@ -70,7 +70,7 @@ std::string uname_m() { return s::on_macos() ? "arm64" : "aarch64"; } -E2E requests{"requests", [] { +E2E requests{"requests", s::home_fixture, [] { "282 target is a request"_test[s::gcc] = [] { // A triple that declines to name a C library is no contradiction of the graph's; one // that names another is reported, with what resolved and a spelling to use instead. diff --git a/tests/e2e/targets/rows.cpp b/tests/e2e/targets/rows.cpp index 427d4d1b2..652aa58f0 100644 --- a/tests/e2e/targets/rows.cpp +++ b/tests/e2e/targets/rows.cpp @@ -56,7 +56,7 @@ int main() { } )"; -E2E rows{"rows", [] { +E2E rows{"rows", s::home_fixture, [] { "91 target bare alias"_test = [] { // `[target.linux.build]` is `[target.'cfg(linux)'.build]`: exactly one OS alias applies // (the host's), and each bare alias agrees with its cfg() form on whichever host runs. diff --git a/tests/e2e/targets/selection.cpp b/tests/e2e/targets/selection.cpp index d56828d8c..c3e852847 100644 --- a/tests/e2e/targets/selection.cpp +++ b/tests/e2e/targets/selection.cpp @@ -91,7 +91,7 @@ void targets_in(const nlohmann::json& j, std::vector& out) { } } -E2E selection{"selection", [] { +E2E selection{"selection", s::home_fixture, [] { "296 what the report names is what the link line uses"_test[s::gcc][s::unix_shell][s::jq] = [] { // Every toolchain the list names round-trips through --toolchain; the C library the query // names is on the link line (or, when it names none, the link line reaches nothing diff --git a/tests/e2e/targets/side.cpp b/tests/e2e/targets/side.cpp index a24f8b3fa..5a075cf68 100644 --- a/tests/e2e/targets/side.cpp +++ b/tests/e2e/targets/side.cpp @@ -36,7 +36,7 @@ bool any_word_contains(const nlohmann::json& entry, std::string_view needle) { return std::ranges::any_of(words_of(entry), [&](const std::string& w) { return w.contains(needle); }); } -E2E side{"side", [] { +E2E side{"side", s::home_fixture, [] { "268 target side resolution"_test[s::unix_shell] = [] { // A layer a dependency declares is reported with its interface, its supplier and its // origin; a package that declares nothing fills nothing; a misspelt name in the mcpp: diff --git a/tests/e2e/test/affected.cpp b/tests/e2e/test/affected.cpp index 94c64b1c6..f17680ec9 100644 --- a/tests/e2e/test/affected.cpp +++ b/tests/e2e/test/affected.cpp @@ -10,7 +10,7 @@ using nlohmann::json; namespace { -E2E affected{"affected", [] { +E2E affected{"affected", s::home_fixture, [] { "1038 test programs of a directory and the tests a change affects"_test[s::gcc] = [] { auto p = s::project::scaffold("app"); diff --git a/tests/e2e/test/archive.cpp b/tests/e2e/test/archive.cpp index 2304135ce..2ee6af449 100644 --- a/tests/e2e/test/archive.cpp +++ b/tests/e2e/test/archive.cpp @@ -18,7 +18,7 @@ std::string trim(std::string v) { // Every line of a file that is a JSON object. std::vector records(const fs::path& file) { return s::json_lines(s::read_file(file)); } -E2E archive{"archive", [] { +E2E archive{"archive", s::home_fixture, [] { "745 no run builds the tests and says so"_test[s::gcc] = [] { // `--no-run` builds the tests and reports that, distinguishably from a run that could not // run them. The runner is a name that is not a program: tests built, nothing run, on every diff --git a/tests/e2e/test/fast.cpp b/tests/e2e/test/fast.cpp index e6b8a6a78..714fad101 100644 --- a/tests/e2e/test/fast.cpp +++ b/tests/e2e/test/fast.cpp @@ -13,7 +13,7 @@ namespace { void touch(const fs::path& f) { std::ofstream(f) << ""; } -E2E fast{"fast", [] { +E2E fast{"fast", s::home_fixture, [] { "1068 a running test holds no build directory"_test = [] { // `mcpp test` releases the build directory's lock once its tests are built (mcpp#915): a // build and a second test of the same configuration proceed while a test runs. The first @@ -97,7 +97,7 @@ int main() {{ return std::system(WHOLE("\"{}\" build > \"{}\" 2>&1")) == 0 ? 0 : // The cases that hold the machine (P7.1): a suite of their own, which the runner runs with no // other test beside it. -E2E fast_machine{"fast-machine", resource{"machine"}, [] { +E2E fast_machine{"fast-machine", s::home_fixture, resource{"machine"}, [] { "1069 an unchanged project tests without planning"_test = [] { // A full `mcpp test` records what it planned beside test.ninja; the next one runs the // selected tests from it, resolving no dev-dependency. An edited test, a test added or @@ -129,8 +129,8 @@ UTest two{{"two", [] {{ expect(eq(2, {})); }}}}; expect(s::lacks(s::text(one), "Resolved mcpplibs.")) << "a selection was planned again"; expect(e2e::contains(s::text(one), "cases: 1 passed, 0 failed")); // An edited test is built, and its result is the edit's. - std::this_thread::sleep_for(1s); write_cases(3); + s::touch(p.at("tests/cases.cpp")); auto edited = p.mcpp({"test"}); expect(edited.started && edited.status != 0) << "the edited test passed"; expect(e2e::contains(s::text(edited), "cases: 1 passed, 1 failed")); @@ -139,8 +139,8 @@ UTest two{{"two", [] {{ expect(eq(2, {})); }}}}; expect(s::lacks(s::text(again), "Resolved mcpplibs.")) << "an unchanged failing project was planned again"; expect(e2e::contains(s::text(again), "cases: 1 passed, 1 failed")) << "the record's run lost the failure"; // A test added is planned and run; one removed is no longer run. - std::this_thread::sleep_for(1s); write_cases(2); + s::touch(p.at("tests/cases.cpp")); p.write("tests/extra.cpp", "int main() { return 0; }\n"); auto added = p.mcpp({"test"}); expect(fatal(e2e::succeeded(added))); diff --git a/tests/e2e/test/frameworks.cpp b/tests/e2e/test/frameworks.cpp index 74ee94a40..a1d844f26 100644 --- a/tests/e2e/test/frameworks.cpp +++ b/tests/e2e/test/frameworks.cpp @@ -10,7 +10,7 @@ namespace fs = std::filesystem; namespace { -E2E frameworks{"frameworks", [] { +E2E frameworks{"frameworks", s::home_fixture, [] { "1052 a framework that uses the package under test is built against it"_test[s::gcc] = [] { // mtest uses mcpplibs.os, which is tested with mtest: the package being built stands for // itself in its own graph, whatever version the framework's edge asked for. diff --git a/tests/e2e/test/granularity.cpp b/tests/e2e/test/granularity.cpp index 3b60fe51f..8a45540be 100644 --- a/tests/e2e/test/granularity.cpp +++ b/tests/e2e/test/granularity.cpp @@ -24,7 +24,7 @@ std::string unit(std::string_view id) { "UTest case_{}{{\"{}\", [] {{ expect(eq(1 + 1, 2)); }}}};\n", id, id); } -E2E granularity{"granularity", [] { +E2E granularity{"granularity", s::home_fixture, [] { "1081 a package's tests are one program"_test[s::gcc] = [] { auto p = s::project::scaffold("app"); p.write("tests/b.cpp", unit("top")); diff --git a/tests/e2e/test/graph.cpp b/tests/e2e/test/graph.cpp index ab2b8dc48..c55fc6dd2 100644 --- a/tests/e2e/test/graph.cpp +++ b/tests/e2e/test/graph.cpp @@ -20,7 +20,7 @@ std::optional test_graph(const s::project& p) { return std::nullopt; } -E2E graph{"graph", [] { +E2E graph{"graph", s::home_fixture, [] { "18 devdeps isolation"_test = [] { // dev-dependencies are not resolved by `mcpp build` and are by `mcpp test`. auto p = s::project::scaffold("myapp"); diff --git a/tests/e2e/test/launch.cpp b/tests/e2e/test/launch.cpp index 86e5fe330..3f3b64342 100644 --- a/tests/e2e/test/launch.cpp +++ b/tests/e2e/test/launch.cpp @@ -42,7 +42,7 @@ s::project app_with_tests() { return p; } -E2E launch{"launch", [] { +E2E launch{"launch", s::home_fixture, [] { "993 a test describes its launch"_test = [] { auto p = app_with_tests(); // A: the envelope names the test, its program, env and cwd, ends its argv with the arguments diff --git a/tests/e2e/test/protocol.cpp b/tests/e2e/test/protocol.cpp index 1ed728f2e..d8e72f14b 100644 --- a/tests/e2e/test/protocol.cpp +++ b/tests/e2e/test/protocol.cpp @@ -28,7 +28,7 @@ std::vector ids(const json& rec) { std::string generic(const fs::path& p) { return p.generic_string(); } -E2E protocol{"protocol", [] { +E2E protocol{"protocol", s::home_fixture, [] { "1035 a test program reports its cases"_test[s::gcc] = [] { // mcpp runs the runner's side of the mtest protocol, version 1; every other program is one // case judged by its exit status and is told none of the variables. diff --git a/tests/e2e/test/report.cpp b/tests/e2e/test/report.cpp index b86bc937b..2fdf897bd 100644 --- a/tests/e2e/test/report.cpp +++ b/tests/e2e/test/report.cpp @@ -13,7 +13,7 @@ std::string manifest(std::string_view name) { return std::format("[package]\nname = \"{}\"\nversion = \"0.1.0\"\nstandard = \"c++23\"\n", name); } -E2E report{"report", [] { +E2E report{"report", s::home_fixture, [] { "15 test passing"_test = [] { // `mcpp test` discovers tests/**/*.cpp and runs each as a separate binary; all passing: // exit 0 and the summary. diff --git a/tests/e2e/test/selection.cpp b/tests/e2e/test/selection.cpp index f7853366b..a7eea94dd 100644 --- a/tests/e2e/test/selection.cpp +++ b/tests/e2e/test/selection.cpp @@ -42,7 +42,7 @@ extra = {{ sources = ["src/extra/*.cppm"] }} {1})", name, test); } -E2E selection{"selection", [] { +E2E selection{"selection", s::home_fixture, [] { "152 test subdir names"_test = [] { // The same stem in two directories of tests/ coexists; a test is named by its path. s::project p{{"mcpp.toml", manifest("subdirs")}, diff --git a/tests/e2e/test/skeleton.cpp b/tests/e2e/test/skeleton.cpp index deba406a3..2491e513d 100644 --- a/tests/e2e/test/skeleton.cpp +++ b/tests/e2e/test/skeleton.cpp @@ -21,7 +21,7 @@ std::string following(std::string_view text, std::string_view anchor, std::size_ return out; } -E2E skeleton{"skeleton", [] { +E2E skeleton{"skeleton", s::home_fixture, [] { "1037 a skeleton tests with mtest and a test needs no main"_test[s::gcc] = [] { // `mcpp new` scaffolds a program or a library whose test is mtest's, and a test file with no diff --git a/tests/e2e/test/streams.cpp b/tests/e2e/test/streams.cpp index 9c4c8951b..c6ee208b1 100644 --- a/tests/e2e/test/streams.cpp +++ b/tests/e2e/test/streams.cpp @@ -13,7 +13,7 @@ namespace { constexpr std::string_view hang_cpp = "#include \n#include \nint main() { std::this_thread::sleep_for(std::chrono::seconds(30)); return 0; }\n"; -E2E streams{"streams", [] { +E2E streams{"streams", s::home_fixture, [] { "865 mcpp test reports its steps on stderr and its results on stdout"_test[s::unix_shell] = [] { s::project p{{"mcpp.toml", R"([package] name = "p" diff --git a/tests/e2e/test/workspace.cpp b/tests/e2e/test/workspace.cpp index 51e57db6e..2dba0c1e4 100644 --- a/tests/e2e/test/workspace.cpp +++ b/tests/e2e/test/workspace.cpp @@ -46,7 +46,7 @@ s::project libraries(const fs::path& dir, std::string_view tag, std::initializer return ws; } -E2E workspace{"workspace", [] { +E2E workspace{"workspace", s::home_fixture, [] { "90 workspace test"_test = [] { // `-p `, `--workspace` and a bare `mcpp test` at a virtual root; discovery is scoped // to a member, so two members may each have tests/main.cpp. (The script's `requires: gcc` diff --git a/tests/e2e/toolchain/custom.cpp b/tests/e2e/toolchain/custom.cpp index 333b7e0df..ba02c7e34 100644 --- a/tests/e2e/toolchain/custom.cpp +++ b/tests/e2e/toolchain/custom.cpp @@ -34,7 +34,7 @@ fs::path llvm_tree(const fs::path& at) { return root; } -E2E custom{"custom", [] { +E2E custom{"custom", s::home_fixture, [] { "875 toolchain by path"_test[s::llvm][s::unix_shell][s::symlink] = [] { // `[toolchain] default = { path = ... }` (mcpp#755): a toolchain tree named by path builds // with its launcher and its stated linker, is reported as custom, writes nothing into the diff --git a/tests/e2e/toolchain/layers.cpp b/tests/e2e/toolchain/layers.cpp index 7a26d22e2..74477792e 100644 --- a/tests/e2e/toolchain/layers.cpp +++ b/tests/e2e/toolchain/layers.cpp @@ -80,7 +80,7 @@ std::optional std_tokens_in(const nlohmann::json& j, std::string_vi return std::nullopt; } -E2E layers{"layers", [] { +E2E layers{"layers", s::home_fixture, [] { "201 gcc no specs pollution"_test[s::gcc][s::elf] = [] { // A gcc artifact carries only the RUNPATH the build asked for (no other home's temporary // directory, no dead entry), names the payload's glibc as its interpreter, and loads diff --git a/tests/e2e/toolchain/llvm.cpp b/tests/e2e/toolchain/llvm.cpp index a1fe13c54..0724c3ccc 100644 --- a/tests/e2e/toolchain/llvm.cpp +++ b/tests/e2e/toolchain/llvm.cpp @@ -67,7 +67,7 @@ s::project llvm_project(std::string_view name, s::files sources, std::string_vie return p; } -E2E llvm{"llvm", [] { +E2E llvm{"llvm", s::home_fixture, [] { "36 llvm toolchain"_test = [] { // A non-module C/C++ package built with the LLVM payload: the list names it, the build // finishes, and the program runs. diff --git a/tests/e2e/toolchain/management.cpp b/tests/e2e/toolchain/management.cpp index 9520ab7cf..bb3e5a600 100644 --- a/tests/e2e/toolchain/management.cpp +++ b/tests/e2e/toolchain/management.cpp @@ -35,7 +35,7 @@ void configure_e2e_mirror(const s::options& o) { expect(fatal(e2e::succeeded(r))) << "failed to configure e2e mirror: " << s::text(r); } -E2E management{"management", [] { +E2E management{"management", s::home_fixture, [] { "13 toolchain pin"_test = [] { // A project's `[toolchain]` pin resolves to the payload's absolute path in mcpp's home. const auto gcc = s::gcc_version(); diff --git a/tests/e2e/toolchain/standard.cpp b/tests/e2e/toolchain/standard.cpp index 5f70ebf18..ea78044b8 100644 --- a/tests/e2e/toolchain/standard.cpp +++ b/tests/e2e/toolchain/standard.cpp @@ -86,7 +86,7 @@ constexpr std::string_view print_struct = R"(void print_struct(auto &&value) { struct Point { double x{}; double y{}; }; )"; -E2E standard{"standard", [] { +E2E standard{"standard", s::home_fixture, [] { "59 cpp standard config"_test[s::gcc] = [] { // `[package].standard` drives the build flags, the compile database and the std BMI's // metadata, through the P1689 scanner too. diff --git a/tests/e2e/web/emscripten.cpp b/tests/e2e/web/emscripten.cpp index 209021eb7..63014a538 100644 --- a/tests/e2e/web/emscripten.cpp +++ b/tests/e2e/web/emscripten.cpp @@ -56,7 +56,7 @@ std::string recorded(const s::project& p, std::string_view key) { return line.empty() ? std::string() : line.substr(key.size() + 1); } -E2E emscripten{"emscripten", [] { +E2E emscripten{"emscripten", s::home_fixture, [] { "650 wasm row names its launcher"_test[s::elf] = [] { // `kind = "shared"` is refused on the row before a toolchain is resolved, and an ordinary // bin is not; a program is bin/.js with .wasm, runs, is staged with both, diff --git a/tests/e2e/windows/crt.cpp b/tests/e2e/windows/crt.cpp index 4840b1afc..6689783cb 100644 --- a/tests/e2e/windows/crt.cpp +++ b/tests/e2e/windows/crt.cpp @@ -71,7 +71,7 @@ std::size_t std_cache_files_with(std::string_view word) { return n; } -E2E crt{"crt", [] { +E2E crt{"crt", s::home_fixture, [] { "703 the llvm row on the msvc abi records its static crt"_test[s::windows] = [] { // What resolution.json records for the llvm row's C++ runtime contract: toolchain-coupled // by default, and an explicit host-coupled or self-contained delivered as recorded, with diff --git a/tests/e2e/windows/deploy.cpp b/tests/e2e/windows/deploy.cpp index f0c708f73..c85850443 100644 --- a/tests/e2e/windows/deploy.cpp +++ b/tests/e2e/windows/deploy.cpp @@ -104,7 +104,7 @@ int main() {{ expect(fatal(s::lines_with(s::text(r), placed).empty())) << "6: a build in which the placement edge did not run stated its difference again: " << s::text(r); } -E2E deploy{"deploy", [] { +E2E deploy{"deploy", s::home_fixture, [] { "811 place dlls never overwrites a declared deploy"_test[s::windows] = [] { declared_deploy_outranks({}); }; diff --git a/tests/e2e/windows/exports.cpp b/tests/e2e/windows/exports.cpp index 5d95434a5..c9ccb1ef2 100644 --- a/tests/e2e/windows/exports.cpp +++ b/tests/e2e/windows/exports.cpp @@ -56,7 +56,7 @@ fs::path compiler_of(const fs::path& db) { return path; } -E2E exports{"exports", [] { +E2E exports{"exports", s::home_fixture, [] { "881 pe auto exports accept llvm bitcode"_test[s::msvc][s::python3] = [] { const auto o = registry_home(); const auto tmp = s::case_dir(); diff --git a/tests/e2e/windows/images.cpp b/tests/e2e/windows/images.cpp index 277bef2fc..bce3385fd 100644 --- a/tests/e2e/windows/images.cpp +++ b/tests/e2e/windows/images.cpp @@ -75,7 +75,7 @@ void subsystems(const s::project& p, std::vector build_args, std::o expect(fatal(e2e::succeeded(ran))) << "wide.exe did not exit 0: " << s::text(ran); } -E2E images{"images", [] { +E2E images{"images", s::home_fixture, [] { "84 runtime dll deploy"_test = [] { // A dependency's [runtime] library_dirs: each *.dll is placed beside the program, byte // for byte, and nothing else of that directory is. diff --git a/tests/e2e/windows/libraries.cpp b/tests/e2e/windows/libraries.cpp index 9fae25c9d..9789d3ff0 100644 --- a/tests/e2e/windows/libraries.cpp +++ b/tests/e2e/windows/libraries.cpp @@ -43,7 +43,7 @@ std::size_t exported_lines(const fs::path& def) { return n; } -E2E libraries{"libraries", [] { +E2E libraries{"libraries", s::home_fixture, [] { "255 pack library msvc archiver"_test[s::msvc] = [] { // The packer drops the published interface's objects from the archive; lib.exe takes // `/REMOVE:` once per member and the archive last, and a wrong spelling makes the pack refuse, diff --git a/tests/e2e/windows/mingw.cpp b/tests/e2e/windows/mingw.cpp index 278a59d3d..db3e65b6e 100644 --- a/tests/e2e/windows/mingw.cpp +++ b/tests/e2e/windows/mingw.cpp @@ -40,7 +40,7 @@ void self_contained(const std::string& imports, std::string_view what) { expect(fatal(!s::contains_nocase(imports, dll))) << what << " imports a toolchain DLL (" << dll << "): " << imports; } -E2E mingw{"mingw", [] { +E2E mingw{"mingw", s::home_fixture, [] { "97 mingw toolchain"_test[s::windows] = [] { // `toolchain install mingw 16.1.0` installs the payload; `toolchain default mingw@16.1.0` // persists gcc@16.1.0 and x86_64-windows-gnu, both starred by `list`; a project with a named diff --git a/tests/e2e/windows/msvc.cpp b/tests/e2e/windows/msvc.cpp index 7a4e9649a..d79b69526 100644 --- a/tests/e2e/windows/msvc.cpp +++ b/tests/e2e/windows/msvc.cpp @@ -56,7 +56,7 @@ std::set ms_versions(const fs::path& ninja) { return words; } -E2E msvc{"msvc", [] { +E2E msvc{"msvc", s::home_fixture, [] { "95 msvc system toolchain"_test[s::msvc] = [] { // `toolchain default msvc` locates and identifies the machine's MSVC and persists // msvc@system; `toolchain list` shows it under System, starred; a versioned spec is a diff --git a/tests/e2e/windows/pack.cpp b/tests/e2e/windows/pack.cpp index 57ad2f90f..fa787039a 100644 --- a/tests/e2e/windows/pack.cpp +++ b/tests/e2e/windows/pack.cpp @@ -56,7 +56,7 @@ names = zipfile.ZipFile(sys.argv[1]).namelist() assert not any(n.lower().endswith('.dll') for n in names), f"--mode system bundled a DLL: {names}" )py"; -E2E pack{"pack", [] { +E2E pack{"pack", s::home_fixture, [] { "240 pack pe zip cross"_test[s::mingw_cross][s::python3] = [] { // A x86_64-windows-gnu program packs as a zip: one wrapper directory, the program with its // executable bit, the DLL its import table and `force_bundle` name flat beside it, and no diff --git a/tests/e2e/windows/placement.cpp b/tests/e2e/windows/placement.cpp index 3ff051f56..bbaf9517b 100644 --- a/tests/e2e/windows/placement.cpp +++ b/tests/e2e/windows/placement.cpp @@ -152,7 +152,7 @@ e2e::outcome run_clean(const fs::path& dir) { return s::run(dir / "app.exe", {}, dir, {{"PATH", root + "\\System32"}}); } -E2E placement{"placement", [] { +E2E placement{"placement", s::home_fixture, [] { "819 the crt is placed by its rule not by search order"_test = [] { // L1 host-coupled (--crt system) places no copy of the runtime; L2 toolchain-coupled with the // set already placed by the plan: the dependency's older copy is neither placed nor stated per diff --git a/tests/e2e/windows/resources.cpp b/tests/e2e/windows/resources.cpp index b148096c3..287b73b5d 100644 --- a/tests/e2e/windows/resources.cpp +++ b/tests/e2e/windows/resources.cpp @@ -287,7 +287,7 @@ void dependency_images(std::vector target) { expect(fatal(e2e::contains(s::text(r), "owner.manifest"))) << "the missing resource was not attributed: " << s::text(r); } -E2E resources{"resources", [] { +E2E resources{"resources", s::home_fixture, [] { "197 windows resources"_test[s::windows] = [] { // mcpp#365 on a Windows host: see resources_reach_the_image. resources_reach_the_image({}); diff --git a/tests/e2e/windows/rows.cpp b/tests/e2e/windows/rows.cpp index 61ee42c9c..3cb2ff801 100644 --- a/tests/e2e/windows/rows.cpp +++ b/tests/e2e/windows/rows.cpp @@ -31,7 +31,7 @@ void carries_versioninfo(const s::project& p, std::string_view triple, std::stri expect(fatal(e2e::succeeded(r) && e2e::contains(s::text(r), "VERSIONINFO"))) << "the program carries no VERSIONINFO: " << s::text(r); } -E2E rows{"rows", [] { +E2E rows{"rows", s::home_fixture, [] { "1058 a gnu abi windows row compiles its resources"_test[s::mingw_cross][s::llvm] = [] { // llvm paired with MinGW-w64 finds the payload's windres (or llvm-rc beside clang) and the // payload's "winres.h"; the cross gcc of the `mingw` provider reads "winres.h" from its own diff --git a/tests/e2e/workspace/graph.cpp b/tests/e2e/workspace/graph.cpp index 59588be1e..6d4ec475d 100644 --- a/tests/e2e/workspace/graph.cpp +++ b/tests/e2e/workspace/graph.cpp @@ -153,7 +153,7 @@ int hide_dual_dll(const fs::path& root, const fs::path& hidden, std::string_view return n; } -E2E graph{"graph", [] { +E2E graph{"graph", s::home_fixture, [] { "833 a workspace is one graph per configuration"_test[s::unix_shell] = [] { // G1 a shared member compiled once; G2 a selection replays the workspace's graph; G3 a // member's flag edit rebuilds that member alone; G4 one build directory per diff --git a/tests/e2e/workspace/inheritance.cpp b/tests/e2e/workspace/inheritance.cpp index eb5cf74ab..6f8646172 100644 --- a/tests/e2e/workspace/inheritance.cpp +++ b/tests/e2e/workspace/inheritance.cpp @@ -73,7 +73,7 @@ std::string archived(const fs::path& archive, std::string_view member) { return s::read_file(into / member); } -E2E inheritance{"inheritance", [] { +E2E inheritance{"inheritance", s::home_fixture, [] { "321 workspace inheritance"_test[s::gcc] = [] { // [workspace.package] and [workspace.build] (#527), asserted on the compile command: a // member that declares nothing inherits the standard and the flags; a member that pins diff --git a/tests/e2e/workspace/members.cpp b/tests/e2e/workspace/members.cpp index 654fcef90..a82b4cea4 100644 --- a/tests/e2e/workspace/members.cpp +++ b/tests/e2e/workspace/members.cpp @@ -59,7 +59,7 @@ std::size_t graphs(const fs::path& root) { return n; } -E2E members{"members", [] { +E2E members{"members", s::home_fixture, [] { "35 workspace"_test = [] { // A virtual workspace of two libraries and a program: the root's build builds every // member, path edges between members hold, and the program is bin/hello/hello in the diff --git a/tests/e2e/workspace/statements.cpp b/tests/e2e/workspace/statements.cpp index 51c49e7cd..a844d0ae9 100644 --- a/tests/e2e/workspace/statements.cpp +++ b/tests/e2e/workspace/statements.cpp @@ -34,7 +34,7 @@ std::vector graphs(const fs::path& root) { return out; } -E2E statements{"statements", [] { +E2E statements{"statements", s::home_fixture, [] { "836 a members own statements reach its build"_test = [] { // M1 a member whose tests import std is tested, from the root and from its directory; // M2 a member's unknown `mcpp:` capability is refused naming its manifest; M3 a member's diff --git a/tests/support/src/support.cpp b/tests/support/src/support.cpp index 723c35c33..cb5394f02 100644 --- a/tests/support/src/support.cpp +++ b/tests/support/src/support.cpp @@ -284,17 +284,89 @@ e2e::outcome as_text(e2e::outcome r) { return r; } +namespace { +// THE SURFACE A COMMAND USED (design 2026-11-20 §5.16, SPEC-017 P5.6). mcpp appends the items it +// used -- the command and flags it was given, the manifest keys it read, the reasons it recorded -- +// to the file MCPP_SURFACE_RECORD names; the case states them as its `exercise`, which the ledger +// of the surface reads. Placed first, so that a case that names its own record keeps it. +fs::path surface_record() { + static std::atomic n{0}; + return case_dir() / std::format(".surface-{}", n++); +} +void report_surface(const fs::path& record) { + std::error_code ec; + if (!fs::exists(record, ec)) return; + std::set items; + for (auto part : std::views::split(read_file(record), '\n')) { + std::string line(part.begin(), part.end()); + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (line.find(' ') != std::string::npos) items.insert(std::move(line)); + } + fs::remove(record, ec); + if (!items.empty()) mcpplibs::mtest::exercise({items.begin(), items.end()}); +} +} // namespace + +// THE SHARED HOME IS A RUN FIXTURE (SPEC-017 P11, design 2026-11-20 §5.10). Its contents -- the +// index, the default toolchain, the standard library's modules -- were filled by whichever case ran +// first, and that case's time held them. Served once per run, the filling is the fixture's own +// (`fixtures.ndjson`): a project made and built in the home. A filling that fails is printed and +// the home is served as it is, since a case that needs no toolchain still runs in it. +mcpplibs::mtest::run_fixture shared_home_fixture{"home", [](mcpplibs::mtest::serving& serve) { + const auto home = home_dir(); + const auto scratch = fs::temp_directory_path() / std::format("mcpp-e2e-home-{:08x}", std::random_device{}()); + std::error_code ec; + fs::create_directories(scratch, ec); + std::vector>> env{{"MCPP_HOME", home.string()}, + {"MCPP_VERBOSE", std::nullopt}}; + auto made = e2e::run({.argv = {mcpp_path(), "new", "filling"}, .cwd = scratch, .env = env, + .timeout = std::chrono::minutes{10}}); + auto built = made ? e2e::run({.argv = {mcpp_path(), "build"}, .cwd = scratch / "filling", .env = env, + .timeout = std::chrono::minutes{30}}) + : made; + if (!built) + std::println(std::cerr, "the shared home {} was not filled (status {}):\n{}{}", home.string(), + built.status, built.out, built.err); + fs::remove_all(scratch, ec); + serve.ready(home.string()); + serve.stopped(); +}}; + +// THE DIAGNOSTIC LINES (design 2026-11-20 §5.18, mtest 0.11.0): a line mcpp writes as a warning or +// an error -- a degradation is written as a warning -- in the output of a program a case ran is the +// case's to declare (`e2e::expect_diagnostic`). Under `report`, the policy the cases migrate under, +// an undeclared line is printed and counted as the case's metric `diagnostics.undeclared`, and the +// case is not failed for it. Declared at the first command, after mtest's own state exists. +void declare_diagnostics() { + static std::once_flag once; + std::call_once(once, [] { + e2e::diagnostics({"^(\\x1b\\[[0-9;]*m)*warning:", "^(\\x1b\\[[0-9;]*m)*error:"}); + e2e::diagnostics_policy(e2e::diagnostic_policy::report); + }); +} + e2e::outcome mcpp(std::vector args, const fs::path& cwd, const options& o) { - return as_text(e2e::run(mcpp_command(std::move(args), cwd, o))); + auto c = mcpp_command(std::move(args), cwd, o); + declare_diagnostics(); + const auto record = surface_record(); + c.env.insert(c.env.begin(), {"MCPP_SURFACE_RECORD", record.string()}); + auto r = as_text(e2e::run(std::move(c))); + report_surface(record); + return r; } +std::optional own_home_of_case(); // below, with the case's directories + e2e::command mcpp_command(std::vector args, const fs::path& cwd, const options& o) { e2e::command c; c.argv.push_back(mcpp_path()); for (auto& a : args) c.argv.push_back(std::move(a)); c.cwd = cwd; - if (o.where != home::shared) { - c.env.emplace_back("MCPP_HOME", case_home(o.where).string()); + const auto where = o.where == home::shared ? own_home_of_case().value_or(home::shared) : o.where; + if (where != home::shared) { + c.env.emplace_back("MCPP_HOME", case_home(where).string()); + } else if (const auto served = mcpplibs::mtest::fixture_endpoint("home"); !served.empty()) { + c.env.emplace_back("MCPP_HOME", served); } else if (!env("MCPP_HOME")) { if (auto u = env(on_windows() ? "USERPROFILE" : "HOME")) c.env.emplace_back("MCPP_HOME", (fs::path(*u) / ".mcpp").string()); @@ -332,48 +404,52 @@ void link_or_copy(const fs::path& from, const fs::path& to) { if (ec) fs::copy(from, to, fs::copy_options::recursive, ec); } -// _inherit_toolchain.sh's link_xpkg_payloads: each package directory of `from` is a directory of -// `to` whose versions are links; anything else is linked whole. -void link_payloads(const fs::path& from, const fs::path& to) { +// THE USER'S PAYLOADS ARE UPSTREAMS OF THE CASE'S HOME, NEVER LINKS INTO IT (mcpp#293, design +// 2026-11-20 §5.10). A link to each payload made the user's store part of the case's: a hook the +// case's home ran again (gcc's specs, an ELF patch) wrote through the link into the user's +// toolchain, with paths of the case's temporary home. An upstream is read by link at install and +// never written (docs/25 §3): the case's home installs what it needs from the user's homes. +void name_upstreams(const fs::path& h) { + std::vector ups; std::error_code ec; - if (!fs::is_directory(from, ec)) - return; - fs::create_directories(to); - for (const auto& e : fs::directory_iterator(from, ec)) { - const auto there = to / e.path().filename(); - if (!e.is_directory(ec) || fs::is_symlink(fs::symlink_status(there, ec))) { - link_or_copy(e.path(), there); - continue; + for (const auto& u : {home_dir(), user_dir() / ".xlings"}) + if (fs::is_directory(u, ec)) ups.push_back(std::format("\"{}\"", u.generic_string())); + if (ups.empty()) return; + std::string line = "upstream = ["; + for (std::size_t i = 0; i < ups.size(); ++i) line += (i ? ", " : "") + ups[i]; + line += "]"; + std::string text = fs::exists(h / "config.toml", ec) ? read_file(h / "config.toml") : std::string{}; + std::string out; + bool inXlings = false, placed = false; + for (auto part : std::views::split(text, '\n')) { + std::string l(part.begin(), part.end()); + if (l.starts_with('[')) { + if (inXlings && !placed) { out += line + "\n"; placed = true; } + inXlings = l.starts_with("[xlings]"); } - fs::create_directories(there); - for (const auto& v : fs::directory_iterator(e.path(), ec)) - link_or_copy(v.path(), there / v.path().filename()); + if (inXlings && l.starts_with("upstream")) continue; + out += l + "\n"; } + while (out.ends_with("\n\n")) out.pop_back(); + if (inXlings && !placed) { out += line + "\n"; placed = true; } + if (!placed) out += "\n[xlings]\n" + line + "\n"; + std::ofstream(h / "config.toml", std::ios::binary) << out; } + void inherit(const fs::path& h, bool whole) { const auto user = home_dir(); - const auto xlings = user_dir() / ".xlings/data/xpkgs"; - link_payloads(user / "registry/data/xpkgs", h / "registry/data/xpkgs"); - link_payloads(user / "xpkgs", h / "xpkgs"); - link_payloads(xlings, h / "registry/data/xpkgs"); std::error_code ec; for (auto name : {"xim-pkgindex", "xim-index-repos"}) if (fs::is_directory(user / "registry/data" / name, ec)) { fs::create_directories(h / "registry/data"); link_or_copy(user / "registry/data" / name, h / "registry/data" / name); } - // THE SUBOS IS THE CASE'S OWN COPY, ITS LINKS KEPT AS LINKS. A link to the user's subos made - // it shared state: an install by the case wrote into the user's subos, and an install in the - // user's home by a case beside it appeared in this one's. The subos is a farm of links (about - // 1600 on a developer's home), so the copy costs what the link did. - if (whole && env("MCPP_INHERIT_SUBOS").value_or("1") != "0" && fs::is_directory(user / "registry/subos", ec) - && !fs::exists(fs::symlink_status(h / "registry/subos", ec))) { - fs::create_directories(h / "registry"); - fs::copy(user / "registry/subos", h / "registry/subos", - fs::copy_options::recursive | fs::copy_options::copy_symlinks, ec); - } + // NO SUBOS IS COPIED: an install by link registers its programs in the case's own subos, and + // a copy of the user's, whose links already name them, made that registration fail (measured: + // `[glibc] failed: declared programs were not registered`). if (whole && env("MCPP_INHERIT_CONFIG").value_or("1") != "0" && fs::is_regular_file(user / "config.toml", ec)) fs::copy_file(user / "config.toml", h / "config.toml", fs::copy_options::overwrite_existing, ec); + name_upstreams(h); if (fs::is_directory(user / "bin", ec)) link_or_copy(user / "bin", h / "bin"); } @@ -394,6 +470,7 @@ struct case_dirs { std::vector> made; // (the case's runner directory, the directory) std::map own; // the case's own directory, by its runner directory std::map home; // the case's home, by its runner directory + std::map instead; // the home a case runs mcpp in instead of the shared one ~case_dirs() { std::error_code ec; for (const auto& [_, d] : made) fs::remove_all(d, ec); @@ -406,6 +483,7 @@ struct case_dirs { fs::remove_all(e.second, ec); own.erase(e.first); home.erase(e.first); + instead.erase(e.first); return true; }); } @@ -461,6 +539,21 @@ fs::path case_home(home kind) { return h; } +void own_home(home kind) { + auto& all = registry_of_cases(); + const auto key = mcpplibs::mtest::temp_dir(); + std::scoped_lock lock(all.m); + all.sweep(); + all.instead[key] = kind; +} + +std::optional own_home_of_case() { + auto& all = registry_of_cases(); + std::scoped_lock lock(all.m); + if (auto it = all.instead.find(mcpplibs::mtest::temp_dir()); it != all.instead.end()) return it->second; + return std::nullopt; +} + fs::path another_home(home kind) { auto& all = registry_of_cases(); std::scoped_lock lock(all.m); diff --git a/tests/support/src/support.cppm b/tests/support/src/support.cppm index 28cec1a6b..686603785 100644 --- a/tests/support/src/support.cppm +++ b/tests/support/src/support.cppm @@ -57,6 +57,9 @@ using mcpplibs::mtest::platform; const only elf{platform::linux}; // a host whose programs are ELF const only unix_shell{platform::linux, platform::macos}; // a POSIX shell and its tools const only fresh_sandbox{platform::linux, platform::macos}; +// The shared home, served once per run before its first user (SPEC-017 P11): each group of cases +// states it, and the time that fills the home is the fixture's, not its first case's. +const mcpplibs::mtest::uses home_fixture{"home"}; const only symlink{platform::linux, platform::macos}; const only macos{platform::macos}; const only windows{platform::windows}; @@ -156,6 +159,12 @@ fs::path case_home(home kind = home::fresh); // that exported MCPP_HOME again did); removed with the case's other homes. mcpp runs in it with // `options::env` setting MCPP_HOME. fs::path another_home(home kind = home::fresh); +// THE CASE RUNS MCPP IN A HOME OF ITS OWN instead of the shared one (design 2026-11-20 §5.10): each +// later `mcpp` of the case that names no home uses `case_home(kind)`. A case whose assertion holds +// only while nothing else writes its home -- a replayed build, which plans again when an install +// beside it rewrote the runtime environment's description -- states this instead of holding the +// machine; `inherited` reads the user's payloads as upstreams. +void own_home(home kind = home::inherited); // The shared home: `MCPP_HOME`, else `~/.mcpp`. fs::path shared_home(); From 6e9ff7fbeed5a81274b74eb8b09bd407a2eeece9 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 05:48:02 +0900 Subject: [PATCH 21/38] docs: memory_threshold, dependency_debug, the run's records, one budget, run fixtures, header selection, the home's caches, the build record, emit surface and the records an invocation appends; the e2e rules of the kit; the spread of functions measured in e2e-coverage --- .agents/skills/mcpp-contributing/SKILL.md | 13 ++++++ .github/workflows/e2e-coverage.yml | 9 +++++ docs/04-mcpp-toml.md | 30 ++++++++++++++ docs/08-testing.md | 48 ++++++++++++++++++++++- docs/25-home-indices-and-store.md | 2 +- docs/26-build-optimizations.md | 33 ++++++++++++++++ docs/50-machine-output.md | 34 ++++++++++++++++ docs/specs/manifest-semantics.md | 2 +- docs/zh/04-mcpp-toml.md | 20 ++++++++++ docs/zh/08-testing.md | 27 ++++++++++++- docs/zh/25-home-indices-and-store.md | 2 +- docs/zh/26-build-optimizations.md | 23 +++++++++++ docs/zh/50-machine-output.md | 30 ++++++++++++++ 13 files changed, 267 insertions(+), 6 deletions(-) diff --git a/.agents/skills/mcpp-contributing/SKILL.md b/.agents/skills/mcpp-contributing/SKILL.md index 0f45b1288..04709297e 100644 --- a/.agents/skills/mcpp-contributing/SKILL.md +++ b/.agents/skills/mcpp-contributing/SKILL.md @@ -150,6 +150,19 @@ E2E 并不保证完全离线:部分脚本需要工具链、索引或 capabilit 按 CI 等价方式设置 `MCPP_HOME`、镜像和其他 capability 后再运行;不要让缓存命中 或空 workspace 选择冒充行为覆盖。 +C++ 端到端用例的写法(`tests/support`,设计 2026-11-20): + +- 等待被测程序给出的事件(一行输出、一个文件、退出),不等待时间;`sleep_for` 与 shell `sleep ` 的次数只减不增 + (`.github/tools/check_waits.sh`、`waits.baseline`)。只为改变修改时间的等待改用 `s::touch(file)`。 +- 断言只在其 home 不被别人改写时成立的用例(回放的构建、快速路径)调用 `s::own_home()`,在自己的 home 中运行 mcpp, + 而不持有 `machine`;改写 home 配置的用例写自己 home 的 `config.toml`,不写共享 home 的。 +- 继承的 home(`home::inherited`、`_inherit_toolchain.sh`)以用户的 home 为 upstream,不链接其载荷,不复制 subos。 +- 每个 E2E 组带 `s::home_fixture`:共享的 home 是运行夹具,其填充记在夹具上。 +- 形式本身是契约的输出(`--protocol-version`、`emit surface`)以 golden 文件整段比较;改变它的拉取请求以 + `MTEST_GOLDEN=update` 更新文件,审阅读取更新给出的改动。 +- 程序写出的 `warning:` 与 `error:` 行由用例以 `e2e::expect_diagnostic` 声明;kit 以 `report` 方式统计未声明的行。 +- 不变量(无改动的重建、第二个工程)以构建记录(`MCPP_BUILD_RECORD`)计数,以指标与界限陈述。 + ### 4. 提交 PR **提交信息前缀**:`feat:` / `fix:` / `refactor:` / `test:` / `docs:` / `chore:` diff --git a/.github/workflows/e2e-coverage.yml b/.github/workflows/e2e-coverage.yml index 0ba383919..2fc09968e 100644 --- a/.github/workflows/e2e-coverage.yml +++ b/.github/workflows/e2e-coverage.yml @@ -64,6 +64,15 @@ jobs: "$MCPP" self config bash tests/e2e/run_all.sh || true (cd tests/e2e && "$E2E_MTEST" coverage) + # SELECTION BY FUNCTION IS DECIDED BY MEASUREMENT (design 2026-11-20 §5.11, K8): how widely the + # cases share each file and each function they executed. Below a median share of one half, a + # selection by function selects fewer cases than one by file; the figures go to the summary + # and the design record. + - name: How widely the cases share files and functions + run: | + set -euo pipefail + (cd tests/e2e && "$E2E_MTEST" coverage spread) | tee spread.txt + { echo '## How widely the e2e cases share what they executed'; echo '```'; cat spread.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" - uses: actions/upload-artifact@v4 with: name: e2e-coverage-linux diff --git a/docs/04-mcpp-toml.md b/docs/04-mcpp-toml.md index 48c19739d..a5b01b7ea 100644 --- a/docs/04-mcpp-toml.md +++ b/docs/04-mcpp-toml.md @@ -1225,6 +1225,24 @@ its final path with `O_TRUNC` and a reader could observe a half-written file. MSVC is left alone — neither `/ifcOnly`'s cost nor `.ifc` atomicity has been measured, and guessing either wrong is silent. +### Available memory (`memory_threshold`) *(@NEXT@+)* + +```toml +[build] +memory_threshold = "2GiB" # a size ("512MiB", "1.5GB") or bytes; "0" waits for nothing +``` + +The compiler executor reads the system's available memory after it holds its +slot and before it starts a compiler. Below this value it waits for a compiler +already running to end; it ends none. The engine's value is 1 GiB, since one +module interface compile peaks at 0.5–1.0 GB (above). The value is taken from +`MCPP_MEMORY_THRESHOLD`, this key, `[build] memory_threshold` in +`~/.mcpp/config.toml`, and the engine, in that order; a value that is not a size +is reported and the next one holds. After a minute below it the executor says so +on standard error and starts the compiler: memory that no compiler of this machine +holds would be waited for forever ([SPEC-004 §9](specs/manifest-semantics.md) +item 19). + ### Module interface extensions (`module_extensions`) mcpp treats `.cppm` as a module interface unit. The C++ ecosystem has not @@ -1786,6 +1804,7 @@ ndebug = true # -DNDEBUG for every unit of the graph (default: on fo trim_paths = true # -ffile-prefix-map per unit (default: on for release/dist) split_debuginfo = "off" # "unpacked": debug information beside the objects (default: dev/debug) bmi_cutoff = "exact" # "ignore-locations" (default): a GCC BMI whose line maps alone moved is kept +dependency_debug = "lines" # the store's and git's dependencies: true | false | "lines" (default under dev/debug) # passthrough escape hatch (fixed keys, open values): cflags = ["-fno-plt"] cxxflags = ["-fno-plt"] @@ -1829,6 +1848,17 @@ anything compiles. into the program (`__builtin_LINE()`, `std::source_location::current()` in a template's body) and a graph compiled with `-fcontracts` are compared exactly under either value. Clang's BMI has no such part and the key has no effect on it. +- **`dependency_debug` (@NEXT@+)** states the debug information of the dependencies + that come from the package store and from git: `true` (the profile's `debug`), + `false` (none), `"lines"` (the line tables alone: `-g1` on GCC, + `-gline-tables-only` on Clang, from the toolchain's row). Omitted, `dev` and `debug` + take `"lines"` while their `debug` is on, and every other profile takes its + `debug`. The members of the workspace and path dependencies keep the profile's + `debug`. A backtrace through a dependency still names its files and lines; + measured on a project whose only dependency is mtest, its test program fell from + 43.7 MB to 8.3 MB and its build from 15.9 s to 7.2 s. The flag enters the + dependency's configuration, so each value has its own entry in the cache + ([SPEC-004 §9](specs/manifest-semantics.md) item 20). - **`ndebug` and `trim_paths` (2026.10.24.1+)** follow the industry's convention: on for `release` and `dist`, off for `dev`, `debug` and a custom profile that does not state them. `ndebug` defines `NDEBUG` for every unit, so `assert` compiles out, as in diff --git a/docs/08-testing.md b/docs/08-testing.md index 805971bb5..2e915b5c5 100644 --- a/docs/08-testing.md +++ b/docs/08-testing.md @@ -159,6 +159,29 @@ the slowest cases, the critical path and the schedule). In CI, `commit` and `group` are `GITHUB_SHA` and `GITHUB_RUN_ID`. `[test] store = ""` names another directory, relative to the project; `"none"` keeps nothing. +**What a run records** (@NEXT@+, [SPEC-017](specs/test.md) B4, P5.6, P12.4). A run's +`run.json` and its line in `index.ndjson` name the machine's class (`machine`: +the processor's model), its `processors` and `memory`, the monitoring level the +run stated (`MTEST_MONITOR`, `basic` unless the environment states another: each +case's processor time) and the jitter, when one was stated. A case's line adds +its processor time (`cpu_ms`), its steps (each command an end-to-end case ran, +with its time and status), the items of the product's surface it used +(`exercised`) and the limit it ran within (`limit_ms`). `builds.ndjson` holds the +builds of the run: a line per edge that ran, with its package, its time, the +processor time and the largest resident set of the compiler, and whether the unit +cache placed it (`served`), and a line per package the cache served. The +estimates of the order prefer the runs of the same class of machine. + +**One budget for the machine** (@NEXT@+, SPEC-017 O11). `mcpp test` makes a +directory of one slot file per job and states it to its programs +(`MTEST_BUDGET`, and `MTEST_BUDGET_HELD`, the slot a worker holds); a compiler a +test starts through mcpp holds a slot of it before it starts, so a test that +builds a project takes from the run's jobs rather than adding its own. A run +inside another runner's budget uses that one. + +**A case's time is the one its scopes state** (@NEXT@+). A case selected with +`--case` runs within the `timeout` its listed case states, not the run's. + A suite compiled from several sources of its own is also a package: a workspace member whose `[build] sources` carries the suite and whose one test program drives it, selected with `mcpp test -p `. @@ -335,8 +358,13 @@ the tests a change affects, read from the plan before anything is built: - a test is affected by a change of a file it is built from: its own files, the module units it imports and the units those import in turn, and every unit of its link that belongs to no module; -- a change of a manifest, a lock, a build program, a header, or a source the - plan does not hold affects every test; +- a change of a manifest, a lock, a build program, or a source the plan does + not hold affects every test; +- a header (@NEXT@+) affects the tests built from a unit that read it when + the build directory last built it, as ninja's deps log and the scans' + depfiles record it, and the tests that run a program of the package when + such a unit is not a test's own; while a unit of the project has no such + record (a build directory that never built it), a header affects every test; - a test that runs a program of the package (a file it is built from names `MCPP_BIN_`) is affected by a change of any source of the package that is not a test's own (2026.11.13.1+): an end-to-end test runs the program it @@ -612,6 +640,22 @@ The records of `--message-format json` carry the protocol, the cases and their failures ([50](50-machine-output.md)); `--print-launch` states the protocol for a debugger that starts the program. +**Run fixtures** (@NEXT@+, SPEC-017 O6). A program may declare a run fixture -- +one object the whole run shares: a sandboxed home, a server, an index -- and its +cases may name it in `uses` (mtest's `run_fixture` and `uses{}`). `mcpp test` +reads both from the programs' listings. Before the first program whose selected +cases use a fixture starts, it starts a program that declares it with +`MTEST_SERVE` naming it and its input a pipe, and waits, at most `--timeout`, for +its line `mtest-ready `; each user receives the endpoint as +`MTEST_FIXTURE_`. After the last user ends, its input is closed, and it is +ended after the same time if it has not ended. `fixtures.ndjson` in the run's +store states each fixture's start, readiness and end, so its preparation is not +counted in the first case that used it; `programs.ndjson` names the fixtures each +program used. A fixture that does not become ready is the machine's failure: the +programs that use it do not run, their records say `infra`, and the run exits +with 2 as for a test that was not run. A program run behind a runner is not +listed and receives no fixture. + ## Writing a test ```bash diff --git a/docs/25-home-indices-and-store.md b/docs/25-home-indices-and-store.md index 51e3ed7bf..875bb9040 100644 --- a/docs/25-home-indices-and-store.md +++ b/docs/25-home-indices-and-store.md @@ -119,7 +119,7 @@ a project's `target/` is its own. Each has a retention, in config.toml: | key | default | removed | |---|---|---| -| `[cache] unused_days` | 30 | after a build, at most once a day: cache entries (packages, `std` modules, host tools) no build used for that many days | +| `[cache] unused_days` | 30 | after a build, at most once a day: cache entries (packages, `std` modules, host tools, units, build programs' outputs) no build used for that many days | | `[build] keep_days` | 3 | after `mcpp build` or `mcpp test`, at most once a day per project: build directories no configuration used for that many days | | `[build] target_budget` | none | after every `mcpp build` or `mcpp test` whose `target/` holds more than this size (`"20GiB"`, `"512MiB"`): every build directory no configuration records as current, whatever its age (2026.11.8.1+) | diff --git a/docs/26-build-optimizations.md b/docs/26-build-optimizations.md index e8035fb0a..fee726ba2 100644 --- a/docs/26-build-optimizations.md +++ b/docs/26-build-optimizations.md @@ -86,6 +86,39 @@ The note is given when the rebuild started at a module of one file with at least importing units, once a build directory. Beside it, a build that replaced or kept BMIs states which ([04](04-mcpp-toml.md), `bmi_cutoff`). +## What the home's cache places (@NEXT@+) + +Beside the entries, the home's build cache (`build-cache/v1/`, [25](25-home-indices-and-store.md)) +keeps what one build made for the next, in any project of the home, whenever `[build] cache` +is the home's (the default): + +- **Units.** Before it starts a compiler, the compiler executor looks up the compile by its + command, its compiler and the content of its source. An entry names the files an earlier + compile of that key read (its depfile, or the scan's depfile and the BMIs it imported) with + their contents; when they all still hold those contents, its object and BMI are placed and + the compiler does not start. After a compile it keeps the outputs and the files read. A BMI + is named by the key of the compile that made it, since GCC writes a different BMI on each + compile of the same source; a file of a package store is named by its path, since the store + does not change. A fresh checkout of mcpp on a filled cache built in 4.1 s instead of 164 s, + its 288 compiles all placed. cl.exe's compiles are not cached. +- **The outputs of build programs.** A package whose source is named -- an index's version, a + git commit, a path dependency's content digest -- runs its build program into + `build-cache/v1/out/@//out` (`MCPP_OUT_DIR`), the key being its + configuration without that directory: a second project takes the outputs, and runs nothing + ([SPEC-004 §9](specs/manifest-semantics.md) item 18). + +Entries no build used within `[cache] unused_days` are removed as other entries are. + +## The record of a build (@NEXT@+) + +A build records each edge it ran: its package, its time, and, for a compile, the compiler's +processor time and largest resident set and whether the unit cache placed it; and the units +the cache served each package. `MCPP_BUILD_RECORD=` appends the record of every build of +the invocation to the file, a line per edge ([SPEC-017](specs/test.md) B4, `builds.ndjson`); +`mcpp test` keeps the builds of a run in its store. Under `--verbose` a build names its five +slowest edges, each on a `Slowest` line with its processor time and memory. An edge that has +run for a minute is named once on a `Running` line, with the package and source it compiles. + ## Stating an entry otherwise ```toml diff --git a/docs/50-machine-output.md b/docs/50-machine-output.md index 804d65b15..6a0ea4fc8 100644 --- a/docs/50-machine-output.md +++ b/docs/50-machine-output.md @@ -254,6 +254,10 @@ envelope says when either has changed since. | `test-frameworks` | 2026.11.10.1 | `mcpp test --guide` names each framework's `capabilities` and `session` (SPEC-017 B6) | an IDE choosing between a session and processes | | `test-checks` | 2026.11.10.1 | with `test-protocol`: a failure's `source` and `fatal`, and a case's `checks` (`held`, `failed`), as the program reported them (SPEC-017 O3) | an IDE's list of checks | | `test-output` | 2026.11.10.1 | with `test-case-stream`: a case's own output, `output`: the program's `output` events, or all the output of a program that ran that case alone, in its program's `cases` and, from events, in its `end` record (SPEC-017 O8) | an IDE's output of one case | +| `test-steps` | @NEXT@ | with `test-protocol`: a case's `steps` and the surface items it used (`exercised`), its `cpu_ms` and `limit_ms`, and the run's `builds.ndjson` (SPEC-017 P5.6, B4) | a run's report | +| `test-budget` | @NEXT@ | `mcpp test` states one budget of slots (`MTEST_BUDGET`) its programs' compilers take, and uses a budget its environment states (SPEC-017 O11) | `mtest run`, a runner that runs `mcpp test` | +| `test-jitter` | @NEXT@ | the jitter a run states (`MTEST_JITTER`, `MTEST_JITTER_SEED`) reaches its programs and its record (SPEC-017 O12) | the jitter workflow | +| `test-fixtures` | @NEXT@ | run fixtures are served for the programs whose cases use them (`MTEST_SERVE`, `MTEST_FIXTURE_`), and a run's `fixtures.ndjson` states them (SPEC-017 O6) | a framework's shared fixtures | ### How a tool runs mcpp for a project @@ -971,6 +975,36 @@ or `null`, `guide` or `null`, `repo`), `commands` (`list`, `one`, `records`, `debug`), `docs`, and `suggest` (the dev-dependency line and repository of mtest) when the package has no framework. +### `mcpp emit surface` and the records an invocation appends *(mcpp @NEXT@+)* + +`mcpp emit surface [-o FILE]` writes mcpp's surface, one item per line, ` `, in +the order `command`, `flag`, `key`, `reason`, each sorted by name: + +``` +command emit surface +flag build --target +flag --quiet +key build.cflags +key toolchain +reason unknown-target +``` + +The commands and flags are those of the definition the command line is parsed with (a global +flag is named without a command); the keys are the manifest's key registry (a whole table by +its name); the reasons are the tokens of the refusals' table (`mcpp.why.toolchain`, above). `--help` and `--version`, which the parser +adds, are not items. + +Two variables name a file an invocation appends to; each process of the invocation appends, +so a build's own sub-builds and the compilers it starts through mcpp add to the same file: + +| variable | each line | +|---|---| +| `MCPP_SURFACE_RECORD` | an item of the surface the process used, once per process: the command and flags it was given, the registry keys its manifests state, the reasons it recorded | +| `MCPP_BUILD_RECORD` | an edge a build ran (`build`, `output`, `package`, `program`, `started_ms`, `ms`, and for a compile `cpu_ms`, `peak_bytes` and `served`), or a package the cache served (`build`, `package`, `cached`, `units`): SPEC-017 B4's `builds.ndjson` | + +A test kit turns the first into a case's `exercise` events (SPEC-017 P5.6), and the surface's +ledger compares the two. + ## 9. The build stream and the result record *(mcpp 2026.10.26.1+)* Every chain that builds a project — a person at a terminal, an agent with a diff --git a/docs/specs/manifest-semantics.md b/docs/specs/manifest-semantics.md index 70dfa1166..a6403516c 100644 --- a/docs/specs/manifest-semantics.md +++ b/docs/specs/manifest-semantics.md @@ -692,7 +692,7 @@ mcpp 2026.9.26.2,#703)。** 身份的输出由所有工程共用,运行时持有该目录的文件锁。构建程序收到 `MCPP_OUT_DIR` 并可以把它 写进生成的文件,因此位置是内容的一部分,由身份决定。 19. **`[build] memory_threshold`。** 编译执行器启动编译器之前读取系统的可用内存:低于该值时等待 - 已运行的编译器结束,不结束它们。值为大小(`"2GiB"`、`"512MiB"`、`"1.5GB"`)或字节数,`0` + 已运行的编译器结束,不结束它们;低于该值满一分钟时说明并启动编译器。值为大小(`"2GiB"`、`"512MiB"`、`"1.5GB"`)或字节数,`0` 不等待。取值按 §2 的四层:`MCPP_MEMORY_THRESHOLD`、清单的 `[build] memory_threshold`、机器的 `config.toml` 的 `[build] memory_threshold`、引擎的 1 GiB。它是配置键(作用域 C),不进入包的 配置。无法读出的值被报告,并取下一层。 diff --git a/docs/zh/04-mcpp-toml.md b/docs/zh/04-mcpp-toml.md index e3946f3cb..e6f402d28 100644 --- a/docs/zh/04-mcpp-toml.md +++ b/docs/zh/04-mcpp-toml.md @@ -1101,6 +1101,19 @@ CI 上都跑通过。 一半的文件。MSVC 被留在一边不动——`/ifcOnly` 的开销与 `.ifc` 的 原子性都未经测量,两者任何一个猜错都是无声的。 +### 可用内存(`memory_threshold`)*(@NEXT@+)* + +```toml +[build] +memory_threshold = "2GiB" # 大小("512MiB"、"1.5GB")或字节数;"0" 不等待 +``` + +编译执行器取得槽位之后、启动编译器之前读取系统的可用内存:低于该值时等待已运行的编译器结束,不结束它们。 +引擎的取值为 1 GiB,因为一次模块接口的编译峰值为 0.5–1.0 GB(见上)。取值依次来自 `MCPP_MEMORY_THRESHOLD`、本键、 +`~/.mcpp/config.toml` 的 `[build] memory_threshold` 与引擎;不是大小的值被报告,由下一层取值。低于该值满一分钟时,执行器在 +标准错误上说明并启动编译器:本机没有编译器占用的内存永远等不到([SPEC-004 §9](../specs/manifest-semantics.md) +第 19 项)。 + ### 模块接口扩展名(`module_extensions`) mcpp 把 `.cppm` 当作一个模块接口单元。C++ 生态还没有在这件事上收敛到 @@ -1612,6 +1625,7 @@ ndebug = true # 整个图的每个单元 -DNDEBUG(release/dist 默 trim_paths = true # 按单元的 -ffile-prefix-map(release/dist 默认开) bmi_cutoff = "exact" # "ignore-locations"(默认):只有行映射变化的 GCC BMI 被保留 split_debuginfo = "off" # "unpacked":调试信息留在对象旁(dev/debug 默认) +dependency_debug = "lines" # 来自存储与 git 的依赖:true | false | "lines"(dev/debug 的默认) # passthrough escape hatch (fixed keys, open values): cflags = ["-fno-plt"] cxxflags = ["-fno-plt"] @@ -1640,6 +1654,12 @@ ldflags = [] - 内置 profile:`release`(-O2)/ `dev`、`debug`(-O0 -g)/ `dist` (-O3 + strip;**默认不启用 LTO**)。`[profile.<内置名>]` 可以整体 覆盖一个内置定义。 +- **`dependency_debug`(@NEXT@+)** 陈述来自包存储与 git 的依赖的调试信息:`true`(与 profile 的 `debug` 相同)、 + `false`(不带)、`"lines"`(只带行表:GCC 的 `-g1`、Clang 的 `-gline-tables-only`,由工具链行给出)。省略时,`dev` + 与 `debug` 在其 `debug` 打开时取 `"lines"`,其余 profile 取其 `debug`。工作空间的成员与路径依赖保持 profile 的 + `debug`。经由依赖的回溯仍给出文件与行号;在只依赖 mtest 的工程上实测,测试程序由 43.7 MB 降为 8.3 MB,构建由 + 15.9 s 降为 7.2 s。标志进入依赖的配置,因此每种取值在缓存中各有条目([SPEC-004 §9](../specs/manifest-semantics.md) + 第 20 项)。 - **`bmi_cutoff`(2026.11.17.2+)** 规定导入者何时跟随模块的接口 ([SPEC-018](../specs/module-rebuilds.md))。所导入的 BMI 变化时,导入者重新编译。GCC 下,在 其后没有已记录实体的位置插入一行,只改变 BMI 的行映射;默认值 `"ignore-locations"` 保留 diff --git a/docs/zh/08-testing.md b/docs/zh/08-testing.md index 33f8b356d..3c478c49c 100644 --- a/docs/zh/08-testing.md +++ b/docs/zh/08-testing.md @@ -114,6 +114,20 @@ note,说明有多少文件仍各自是一个程序;它不形成任何组时 mtest 自身的运行一样读取它们(Markdown 报告、最慢的用例、关键路径与调度)。在 CI 中 `commit` 与 `group` 取 `GITHUB_SHA` 与 `GITHUB_RUN_ID`。`[test] store = ""` 另指一个目录,相对项目;`"none"` 不保存。 +**运行记录的内容**(@NEXT@+,[SPEC-017](../specs/test.md) B4、P5.6、P12.4)。运行的 `run.json` 及其在 `index.ndjson` +中的一行写出机器的类别(`machine`:处理器的型号)、`processors` 与 `memory`、运行陈述的监测程度(`MTEST_MONITOR`, +环境未陈述时为 `basic`:每个用例的处理器时间)与陈述了的抖动。用例的一行另有其处理器时间(`cpu_ms`)、步骤(端到端 +用例运行的每条命令及其用时与状态)、它用到的产品面的项(`exercised`)与它所受的时限(`limit_ms`)。`builds.ndjson` +保存运行中的构建:每条运行的边一行,带包、用时、编译器的处理器时间与最大驻留集,以及是否由单元缓存放置 +(`served`);缓存提供的每个包一行。调度的估计优先取同类机器的运行。 + +**机器的一份预算**(@NEXT@+,SPEC-017 O11)。`mcpp test` 建立每个作业一个槽位文件的目录,并告知其程序 +(`MTEST_BUDGET`,以及工作者持有的槽位 `MTEST_BUDGET_HELD`);测试经由 mcpp 启动的编译器在启动前持有其中一个槽位, +因此构建工程的测试占用本次运行的作业数,而不另加自己的。运行在另一个运行器的预算之内时使用那一份。 + +**用例的时限是其作用域陈述的时限**(@NEXT@+)。以 `--case` 选出的用例在其列出用例陈述的 `timeout` 之内运行, +而不是运行的缺省时限。 + 由自身多个源文件编译而成的测试套件也可以是一个包:一个工作区成员,它的 `[build] sources` 承载整套套件,其唯一的测试程序驱动套件,用 `mcpp test -p ` 选中。 @@ -252,7 +266,10 @@ test result ok. 7 passed; 0 failed; finished in 7.77s (plan 2.10s · programs 1. - 测试受其构建所用文件的变更影响:它自己的文件、它导入的模块单元及这些单元再导入的 单元,以及链接中不属于任何模块的单元; -- 清单、锁文件、构建程序、头文件或计划中没有的源文件的变更影响所有测试; +- 清单、锁文件、构建程序或计划中没有的源文件的变更影响所有测试; +- 头文件(@NEXT@+)影响由构建目录上次构建时读取它的单元构建的测试(ninja 的依赖日志与扫描的依赖文件所记录), + 这样的单元不属于测试自身时还影响运行本包程序的测试;工程的某个单元尚无此记录时(构建目录从未构建过它), + 头文件影响所有测试; - 运行本包程序的测试(其构建所用的某个文件写出 `MCPP_BIN_`)受本包中任何不属于测试自身的源文件的 变更影响(2026.11.13.1+):端到端测试运行它所测的程序,却不由该程序的任何源文件构建; - 其他文件只影响自己的文件中写出该文件的测试(测试数据),不影响其他测试。 @@ -460,6 +477,14 @@ mcpp test --seed 42 # 用例中随机性的种子 `--message-format json` 的记录带有协议、用例及其失败([50](50-machine-output.md));`--print-launch` 为启动程序的调试器给出协议。 +**运行夹具**(@NEXT@+,SPEC-017 O6)。程序可以声明运行夹具——整次运行共享的一个对象:沙箱化的 home、服务器、 +索引——其用例在 `uses` 中写出它(mtest 的 `run_fixture` 与 `uses{}`)。`mcpp test` 从程序的列出中读取二者。在所选 +用例使用某个夹具的第一个程序启动之前,它以 `MTEST_SERVE` 指明该夹具、以管道为输入启动一个声明它的程序,并至多等待 +`--timeout` 读到 `mtest-ready <端点>` 一行;每个使用者以 `MTEST_FIXTURE_` 收到端点。最后一个使用者结束后关闭 +其输入,同样的时间之后仍未结束则结束它。运行存储的 `fixtures.ndjson` 写出每个夹具的启动、就绪与结束,夹具的准备 +因此不计入第一个使用它的用例;`programs.ndjson` 写出每个程序用到的夹具。未就绪的夹具是机器的失败:使用它的程序 +不运行,其记录为 `infra`,运行如同有测试未运行一样以 2 结束。运行器之后运行的程序不被列出,不取得夹具。 + ## 编写测试 ```bash diff --git a/docs/zh/25-home-indices-and-store.md b/docs/zh/25-home-indices-and-store.md index d36049e9f..0fe1a3b46 100644 --- a/docs/zh/25-home-indices-and-store.md +++ b/docs/zh/25-home-indices-and-store.md @@ -94,7 +94,7 @@ Windows 首次显式 `self init` 时,若 D 是可写的本地固定磁盘, | 键 | 默认 | 移除的内容 | |---|---|---| -| `[cache] unused_days` | 30 | 构建后每天至多一次:该天数内没有构建使用过的缓存条目(包、`std` 模块、host 工具) | +| `[cache] unused_days` | 30 | 构建后每天至多一次:该天数内没有构建使用过的缓存条目(包、`std` 模块、host 工具、单元、构建程序的输出) | | `[build] keep_days` | 3 | `mcpp build` 或 `mcpp test` 后,每个工程每天至多一次:该天数内没有配置使用过的构建目录 | | `[build] target_budget` | 无 | 每次 `mcpp build` 或 `mcpp test` 后,`target/` 超过该大小(`"20GiB"`、`"512MiB"`)时:没有任何配置记录为当前的构建目录,不论其年龄(2026.11.8.1+) | diff --git a/docs/zh/26-build-optimizations.md b/docs/zh/26-build-optimizations.md index b64f375b8..18a9ed9c5 100644 --- a/docs/zh/26-build-optimizations.md +++ b/docs/zh/26-build-optimizations.md @@ -75,6 +75,29 @@ note hub.cppm is the whole of module hub, which 46 units import: its function b 重编译的起点是一个只有一个文件的模块、且导入它的单元不少于 5 个时给出这条说明,每个构建目录一次。 此外,替换或保留了 BMI 的构建说明是哪些([04](04-mcpp-toml.md),`bmi_cutoff`)。 +## home 的缓存放置的内容(@NEXT@+) + +在各条目之外,`[build] cache` 为 home 的(缺省)时,home 的构建缓存(`build-cache/v1/`,[25](25-home-indices-and-store.md)) +为 home 中任何工程的下一次构建保存一次构建的产物: + +- **单元。** 编译执行器启动编译器之前,以编译的命令、编译器与源文件的内容查找这次编译。条目写出该键此前一次编译 + 读取的文件(其依赖文件,或扫描的依赖文件与它导入的 BMI)及其内容;这些文件仍是那些内容时,放置其对象与 BMI, + 不启动编译器。编译之后保存输出与读取的文件。BMI 以产生它的编译的键命名,因为 GCC 对同一源文件的每次编译写出 + 不同的 BMI;包存储中的文件以路径命名,因为存储不变。mcpp 的新检出在已填充的缓存上 4.1 s 构建完成(原为 164 s), + 288 次编译全部被放置。cl.exe 的编译不进入缓存。 +- **构建程序的输出。** 来源被命名的包——索引的版本、git 的提交、路径依赖的内容摘要——其构建程序在 + `build-cache/v1/out/<包>@<版本>/<键>/out`(`MCPP_OUT_DIR`)中运行,键为不含该目录的配置:第二个工程取用这些输出, + 不再运行([SPEC-004 §9](../specs/manifest-semantics.md) 第 18 项)。 + +在 `[cache] unused_days` 之内没有构建使用过的条目与其他条目一样被移除。 + +## 构建的记录(@NEXT@+) + +构建记录它运行的每条边:所属的包、用时,编译另有编译器的处理器时间与最大驻留集,以及是否由单元缓存放置; +以及缓存为每个包提供的单元数。`MCPP_BUILD_RECORD=<文件>` 把该次调用每次构建的记录追加到该文件,每条边一行 +([SPEC-017](../specs/test.md) B4,`builds.ndjson`);`mcpp test` 在其存储中保存一次运行的构建。`--verbose` 下构建以 +`Slowest` 行写出最慢的五条边及其处理器时间与内存。运行满一分钟的边以一行 `Running` 被点名一次,写出它编译的包与源文件。 + ## 改变一个条目的取值 ```toml diff --git a/docs/zh/50-machine-output.md b/docs/zh/50-machine-output.md index ae40d3b49..d14243250 100644 --- a/docs/zh/50-machine-output.md +++ b/docs/zh/50-machine-output.md @@ -217,6 +217,10 @@ effects 采用具名的方式而不是一个 `destructive: true|false`,因为 | `test-frameworks` | 2026.11.10.1 | `mcpp test --guide` 写出每个框架的 `capabilities` 与 `session`(SPEC-017 B6) | 在会话与进程之间选择的 IDE | | `test-checks` | 2026.11.10.1 | 在 `test-protocol` 之上:失败的 `source` 与 `fatal`,用例的 `checks`(`held`、`failed`),按程序的报告(SPEC-017 O3) | IDE 的检查点列表 | | `test-output` | 2026.11.10.1 | 在 `test-case-stream` 之上:用例自己的输出 `output`:程序的 `output` 事件,或只运行该用例的程序的全部输出;出现在程序的 `cases` 中,来自事件时也出现在其 `end` 记录中(SPEC-017 O8) | IDE 中单个用例的输出 | +| `test-steps` | @NEXT@ | 在 `test-protocol` 之上:用例的 `steps` 与其用到的面的项(`exercised`)、其 `cpu_ms` 与 `limit_ms`,以及运行的 `builds.ndjson`(SPEC-017 P5.6、B4) | 运行的报告 | +| `test-budget` | @NEXT@ | `mcpp test` 陈述一份槽位的预算(`MTEST_BUDGET`),其程序的编译器从中取槽位;环境陈述了预算时使用那一份(SPEC-017 O11) | `mtest run`,运行 `mcpp test` 的运行器 | +| `test-jitter` | @NEXT@ | 运行陈述的抖动(`MTEST_JITTER`、`MTEST_JITTER_SEED`)到达其程序与记录(SPEC-017 O12) | 抖动的工作流 | +| `test-fixtures` | @NEXT@ | 为用例使用运行夹具的程序服务夹具(`MTEST_SERVE`、`MTEST_FIXTURE_`),运行的 `fixtures.ndjson` 写出它们(SPEC-017 O6) | 框架共享的夹具 | ### 工具为项目运行 mcpp 的方式 @@ -862,6 +866,32 @@ mcpp test [pattern] [--workspace] --message-format json (每项 `package`、`version`、`entry`、`protocol` 或 `null`、`guide` 或 `null`、`repo`)、`commands`(`list`、 `one`、`records`、`debug`)、`docs`,以及包没有框架时的 `suggest`(mtest 的 dev-dependency 写法与仓库)。 +### `mcpp emit surface` 与一次调用追加的记录 *(mcpp @NEXT@+)* + +`mcpp emit surface [-o FILE]` 写出 mcpp 的面,每行一项 `<种类> <名字>`,按 `command`、`flag`、`key`、`reason` 的次序, +各自按名字排序: + +``` +command emit surface +flag build --target +flag --quiet +key build.cflags +key toolchain +reason unknown-target +``` + +命令与选项取自解析命令行所用的定义(全局选项不带命令);键取自清单的键登记表(整张表以其名字);原因取自拒绝的原因标记表(上文 `mcpp.why.toolchain`)中的 +标记。解析器自己加入的 `--help` 与 `--version` 不是项。 + +两个变量各指明一个调用追加的文件;调用的每个进程都追加,因此构建自己的子构建与它经由 mcpp 启动的编译器写入同一文件: + +| 变量 | 每行 | +|---|---| +| `MCPP_SURFACE_RECORD` | 进程用到的面的一项,每进程一次:所给的命令与选项、其清单陈述的登记表中的键、它记录的原因 | +| `MCPP_BUILD_RECORD` | 构建运行的一条边(`build`、`output`、`package`、`program`、`started_ms`、`ms`,编译另有 `cpu_ms`、`peak_bytes` 与 `served`),或缓存提供的一个包(`build`、`package`、`cached`、`units`):SPEC-017 B4 的 `builds.ndjson` | + +测试 kit 把前者转为用例的 `exercise` 事件(SPEC-017 P5.6),面的账本比较二者。 + ## 9. 构建事件流与结果记录 *(mcpp 2026.10.26.1+)* 构建一个工程的每条调用链 —— 终端里的人、只有 shell 的 agent、IDE、CI 任务 —— From e397c961daca8702495719228c869cadb3562f25 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 06:05:18 +0900 Subject: [PATCH 22/38] The dependency_debug comment is shorter: toolchain.cpp stays within 2,500 lines --- src/build/prepare/toolchain.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/build/prepare/toolchain.cpp b/src/build/prepare/toolchain.cpp index d77d325f0..effaec564 100644 --- a/src/build/prepare/toolchain.cpp +++ b/src/build/prepare/toolchain.cpp @@ -271,12 +271,10 @@ static std::expected step1_define_early_toolchain_closures(Pr "[profile.{}] debug = \"{}\": expected true, false or \"lines\"", pname, pr.debugWritten)); state.m->buildConfig.debug = pr.debug; state.m->buildConfig.debugLines = pr.debugLines; - // THE DEPENDENCIES' DEBUG INFORMATION (design 2026-11-20 §5.15): the - // packages from the store and git get the line tables under the - // profiles a developer debugs, unless the profile states otherwise. - // Their debug information made 8.5 of a test program's 11.5 MB, and - // 15.4 of a workspace's 22 GB of test programs (mcpp#927); a backtrace - // through them keeps its files and lines. + // THE DEPENDENCIES' DEBUG INFORMATION (design 2026-11-20 §5.15): the packages from the + // store and git get the line tables under the profiles a developer debugs, unless the + // profile states otherwise. Their debug information made 8.5 of a test program's 11.5 MB + // and 15.4 of a workspace's 22 GB (mcpp#927); a backtrace through them keeps its lines. { if (pr.dependencyDebugDeclared && pr.dependencyDebug != "true" && pr.dependencyDebug != "false" && pr.dependencyDebug != "lines") From 9d6806a47522cfb4d272abc87f654a383023d29e Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 06:05:37 +0900 Subject: [PATCH 23/38] =?UTF-8?q?CI:=20a=20tree=20is=20verified=20once,=20?= =?UTF-8?q?and=20a=20release=20promotes=20the=20archives=20a=20run=20teste?= =?UTF-8?q?d=20(design=202026-11-20=20=C2=A75.22,=20=C2=A77.2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ci.yml: the run's tree; the tiers reach the jobs (bootstrap, cross, the target matrix, the upgrade, openkal, the iOS and MSVC-xlings jobs, the second macOS image and Linux's other toolchains run in `full`); a push to main whose tree a run verified registers that run and tests nothing; `verdict` judges every job and leaves `verified-` after a `full` pass; `merge_group` is accepted. - release-build.yml: the four release builds, moved from release.yml. ci.yml's `full` tier runs them, and its Linux shards test the program of the Linux archive (use-built-mcpp `form`). - release.yml: `find` reads the tag's tree; `promote` takes the verifying run's archives, checked against their checksums and version; without such a run the four builds run, and the summary says which. - The unit shards come from the measured work of main's last run; build.yml restores main's unit cache and, on main, keeps the entries its build used. - The report's sections and `mtest gate`; jitter.yml (weekly, by hand, and on a flip in main's run); the downstream action project-test; main_run.sh resolves main's registered run. - docs/92 and the mcpp-release skill: the release pull request passes `ci:full` before it is merged, main's run is not cancelled, and the release promotes. --- .agents/skills/mcpp-release/SKILL.md | 26 +- .github/actions/e2e-cases/action.yml | 3 +- .github/actions/e2e-mtest/action.yml | 3 +- .github/actions/mcpp-unit/action.yml | 26 +- .github/actions/project-test/action.yml | 128 ++++ .github/actions/use-built-mcpp/action.yml | 19 +- .github/actions/use-built-mcpp/use.sh | 22 +- .github/tools/main_run.sh | 23 + .github/workflows/build.yml | 38 + .github/workflows/ci-linux.yml | 9 + .github/workflows/ci-macos.yml | 5 +- .github/workflows/ci-target-matrix.yml | 4 +- .github/workflows/ci-unit.yml | 58 +- .github/workflows/ci.yml | 186 ++++- .github/workflows/jitter.yml | 139 ++++ .github/workflows/release-build.yml | 762 +++++++++++++++++++ .github/workflows/release.yml | 855 +++------------------- docs/92-release.md | 24 +- docs/zh/92-release.md | 20 +- tests/e2e/mtest.toml | 14 + 20 files changed, 1571 insertions(+), 793 deletions(-) create mode 100644 .github/actions/project-test/action.yml create mode 100755 .github/tools/main_run.sh create mode 100644 .github/workflows/jitter.yml create mode 100644 .github/workflows/release-build.yml diff --git a/.agents/skills/mcpp-release/SKILL.md b/.agents/skills/mcpp-release/SKILL.md index d4df8a3d3..886788674 100644 --- a/.agents/skills/mcpp-release/SKILL.md +++ b/.agents/skills/mcpp-release/SKILL.md @@ -108,6 +108,11 @@ gh pr create --title "chore: bump version to $NEW_VERSION" --body "..." 然后推送(标签在 push 启动运行时读取)。这样 PR 自己的运行就是全集,结果在 PR 的检查中可见。不要以 `gh workflow run ci.yml --ref <分支>` 另开一次运行,也不要取消 PR 的运行(见 mcpp-contributing 第 5 节)。 +全集运行通过时留下 `verified-`(设计 2026-11-20 §5.22),它的 `release-form` 作业留下四个发布归档。 +版本号的两处、CHANGELOG 与文档中 `@NEXT@` 的替换都在这次运行之前提交:之后的任何改动都改变树,标记随之失效。 +合入(squash)之后 main 的树与被验证的树相同,main 的运行只登记、保存缓存,不再运行测试,**不取消它**。 +合入与打 tag 之间 main 移动过时,发布照常构建,摘要写明原因。 + ### 3. 创建并推送 tag ```bash @@ -115,21 +120,26 @@ git tag "v$NEW_VERSION" git push origin "v$NEW_VERSION" ``` -Tag push 会自动触发 `release.yml` workflow。 +Tag push 会自动触发 `release.yml` workflow。它的 `find` 作业由 tag 得到树:有运行验证过这棵树、且仍保留 +四个 `release-*` 归档(七天)时,`promote` 提升它们(核对 `.sha256` 与 Linux 程序报告的版本),不再构建; +否则运行 `release-build.yml` 的四个构建。运行摘要写明 "Promoted" 或 "Built" 及原因。 ### 4. 监控 Release CI -Release workflow 的四个平台构建**同时开始**,各自把归档留作 artifact;`create-release` -在四个构建都通过后一次创建带全部归档的 GitHub release(设计 2026-11-14): +Release workflow 先由 `find` 决定提升还是构建。构建时四个平台构建(`release-build.yml`)**同时开始**, +各自把归档留作 artifact;`create-release` 在四个构建或 `promote` 通过后一次创建带全部归档的 GitHub +release(设计 2026-11-14、2026-11-20 §5.22): | Job | 平台 | 产物 | 依赖 | |-----|------|------|------| | `canaries` | — | 生态项目(xlings、mcppls)用候选版本构建 | 仅手动发布且勾选 `canaries` 时运行 | -| `build-release` | Linux x86_64 | `mcpp-X.Y.Z-linux-x86_64.tar.gz`、源码包、`mcpp.lua`、`install.sh` | 无 | -| `build-linux-aarch64` | Linux aarch64(交叉) | `mcpp-X.Y.Z-linux-aarch64.tar.gz` | 无 | -| `build-macos` | macOS ARM64 | `mcpp-X.Y.Z-macosx-arm64.tar.gz` | 无 | -| `build-windows` | Windows x86_64 | `mcpp-X.Y.Z-windows-x86_64.zip` | 无 | -| `create-release` | — | GitHub release、覆盖全部归档的 `SHA256SUMS` | 四个构建(及运行时的 canaries) | +| `find` | — | tag、提交、树与验证它的运行 | 无 | +| `promote` | — | 被验证运行的四个归档,作为 `release-promoted` | `find` 找到运行时 | +| `build-release` | Linux x86_64 | `mcpp-X.Y.Z-linux-x86_64.tar.gz`、源码包、`mcpp.lua`、`install.sh` | `find` 未找到运行时(`build`) | +| `build-linux-aarch64` | Linux aarch64(交叉) | `mcpp-X.Y.Z-linux-aarch64.tar.gz` | `find` 未找到运行时(`build`) | +| `build-macos` | macOS ARM64 | `mcpp-X.Y.Z-macosx-arm64.tar.gz` | `find` 未找到运行时(`build`) | +| `build-windows` | Windows x86_64 | `mcpp-X.Y.Z-windows-x86_64.zip` | `find` 未找到运行时(`build`) | +| `create-release` | — | GitHub release、覆盖全部归档的 `SHA256SUMS` | 四个构建或 `promote`(及运行时的 canaries) | | `release-manifest` | — | `mcpp-release.json` | `create-release` | | `publish-ecosystem` | — | 镜像到 xlings-res 双端 + 开索引 bump PR | `release-manifest` | diff --git a/.github/actions/e2e-cases/action.yml b/.github/actions/e2e-cases/action.yml index 4ee4bc4a3..fc072e44b 100644 --- a/.github/actions/e2e-cases/action.yml +++ b/.github/actions/e2e-cases/action.yml @@ -53,8 +53,7 @@ runs: HOST: ${{ inputs.host }} run: | set -euo pipefail - run=$(gh run list -R "$GITHUB_REPOSITORY" --workflow ci.yml --branch main --status success --limit 1 \ - --json databaseId -q '.[0].databaseId') + run=$(bash .github/tools/main_run.sh) # main's, or the run it registered (design 2026-11-20 §5.22) [ -n "$run" ] || { echo "::error::no successful CI run of main holds the tests"; exit 1; } rm -rf "$RUNNER_TEMP/mcpp-tests" gh run download "$run" -R "$GITHUB_REPOSITORY" -n "mcpp-tests-$HOST" -D "$RUNNER_TEMP/mcpp-tests" diff --git a/.github/actions/e2e-mtest/action.yml b/.github/actions/e2e-mtest/action.yml index 34c95c5e4..57e445cdb 100644 --- a/.github/actions/e2e-mtest/action.yml +++ b/.github/actions/e2e-mtest/action.yml @@ -41,8 +41,7 @@ runs: HOST: ${{ inputs.host }} run: | set -euo pipefail - run=$(gh run list -R "$GITHUB_REPOSITORY" --workflow ci.yml --branch main --status success --limit 1 \ - --json databaseId -q '.[0].databaseId' 2>/dev/null || true) + run=$(bash .github/tools/main_run.sh) # main's, or the run it registered (design 2026-11-20 §5.22) if [ -z "$run" ]; then echo "no successful run of main: the suite is divided by count"; exit 0; fi rm -rf "$RUNNER_TEMP/e2e-stores" if ! gh run download "$run" -R "$GITHUB_REPOSITORY" -p "e2e-store-$HOST-*" -D "$RUNNER_TEMP/e2e-stores" 2>/dev/null; then diff --git a/.github/actions/mcpp-unit/action.yml b/.github/actions/mcpp-unit/action.yml index eec47323b..2ab02cdfa 100644 --- a/.github/actions/mcpp-unit/action.yml +++ b/.github/actions/mcpp-unit/action.yml @@ -41,6 +41,20 @@ inputs: description: the tests to run, as `mcpp test` patterns separated by spaces; empty for all required: false default: '' + cases: + description: the cases to run, one `mcpp test --case` pattern per line; empty for all + required: false + default: '' + jitter: + description: > + the most a contention point waits, in milliseconds (MTEST_JITTER, SPEC-017 O12; design + 2026-11-20 §5.19), with `jitter-seed`; empty for none + required: false + default: '' + jitter-seed: + description: the seed of the jitter's waits (MTEST_JITTER_SEED) + required: false + default: '' mcpp: description: the mcpp that runs the tests; empty for this commit's (MCPP_FRESH) required: false @@ -70,8 +84,7 @@ runs: run: | set -uo pipefail py=python3; command -v python3 >/dev/null 2>&1 || py=python - run=$(gh run list -R "$GITHUB_REPOSITORY" --workflow ci.yml --branch main --status success --limit 1 \ - --json databaseId -q '.[0].databaseId' 2>/dev/null || true) + run=$(bash .github/tools/main_run.sh) # main's, or the run it registered (design 2026-11-20 §5.22) if [ -z "$run" ]; then echo "no successful run of main: the tests run in the order they are listed"; exit 0; fi d="$RUNNER_TEMP/unit-history" rm -rf "$d" @@ -114,6 +127,9 @@ runs: PATTERNS: ${{ inputs.patterns }} TIMEOUT: ${{ inputs.timeout }} RUNS: ${{ inputs.mcpp }} + CASES: ${{ inputs.cases }} + JITTER: ${{ inputs.jitter }} + JITTER_SEED: ${{ inputs.jitter-seed }} run: | set -euo pipefail args=(test --timeout "$TIMEOUT" --split auto) @@ -137,6 +153,12 @@ runs: [ -z "$SINCE" ] || args+=(--since "$SINCE") # shellcheck disable=SC2206 -- the patterns are words [ -z "$PATTERNS" ] || args+=($PATTERNS) + while IFS= read -r c; do [ -z "$c" ] || args+=(--case "$c"); done <<< "$CASES" + fi + # A search for an order defect, never a verdict of its own (design 2026-11-20 §5.19). + if [ -n "$JITTER" ]; then + export MTEST_JITTER="$JITTER" MTEST_JITTER_SEED="${JITTER_SEED:-0}" + echo "jitter: up to $JITTER ms, seed $MTEST_JITTER_SEED" fi echo "mcpp ${args[*]}" "${RUNS:-$MCPP_FRESH}" "${args[@]}" diff --git a/.github/actions/project-test/action.yml b/.github/actions/project-test/action.yml new file mode 100644 index 000000000..1f0762fd7 --- /dev/null +++ b/.github/actions/project-test/action.yml @@ -0,0 +1,128 @@ +name: mcpp project test +description: > + The tests of a project built with mcpp, as a downstream repository's CI runs them (design + 2026-11-20 §7.4): the mcpp the project pins, its home restored by the project's identity (read + only on a pull request, saved by a push to the default branch), `mcpp test`, the run's store + kept as an artifact, and a summary of the run: its result, its slowest cases and the slowest + edges of its builds (SPEC-017 B4). + + Use: `- uses: mcpp-community/mcpp/.github/actions/project-test@main` after `actions/checkout`. + +inputs: + mcpp: + description: > + the mcpp version to install; empty for the one the project's .xlings.json names, else the + newest release + required: false + default: '' + args: + description: the arguments of `mcpp test`, as words (`-p member --profile release`) + required: false + default: '' + home-key: + description: > + what identifies the project's home beside the system and the mcpp version (its manifests and + lock, by default) + required: false + default: '' + name: + description: the name of the store's artifact + required: false + default: mcpp-test-store + +runs: + using: composite + steps: + - name: The mcpp the project pins + id: version + shell: bash + env: + WANTED: ${{ inputs.mcpp }} + run: | + set -euo pipefail + v="$WANTED" + if [ -z "$v" ] && [ -f .xlings.json ]; then + v=$(sed -n 's/.*"mcpp": *"\([^"]*\)".*/\1/p' .xlings.json | head -1) + fi + echo "version=${v:-latest}" >> "$GITHUB_OUTPUT" + echo "mcpp ${v:-latest}" + + # The home is the tests' run fixture (design 2026-11-20 §5.10): restored by its identity, the + # system, the mcpp version and the project's manifests. A pull request reads it and adds + # nothing; a push to the default branch saves what its run filled. + - name: Restore the project's home + id: home + uses: actions/cache/restore@v4 + with: + path: ~/.mcpp + key: mcpp-project-home-${{ runner.os }}-${{ runner.arch }}-${{ steps.version.outputs.version }}-${{ inputs.home-key || hashFiles('**/mcpp.toml', '**/mcpp.lock', '.xlings.json') }} + restore-keys: | + mcpp-project-home-${{ runner.os }}-${{ runner.arch }}-${{ steps.version.outputs.version }}- + + - name: Install mcpp + shell: bash + env: + VERSION: ${{ steps.version.outputs.version }} + XLINGS_NON_INTERACTIVE: '1' + run: | + set -euo pipefail + if ! command -v xlings > /dev/null 2>&1; then + curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.sh | bash + echo "$HOME/.xlings/subos/current/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.xlings/subos/current/bin:$PATH" + fi + spec=mcpp; [ "$VERSION" = latest ] || spec="mcpp@$VERSION" + for attempt in 1 2 3; do + xlings update > /dev/null 2>&1 || true + if xlings install "$spec" -y -g; then break; fi + [ "$attempt" = 3 ] && { echo "::error::$spec could not be installed"; exit 1; } + sleep 30 + done + mcpp --version + + - name: mcpp test + shell: bash + env: + ARGS: ${{ inputs.args }} + run: | + set -euo pipefail + # shellcheck disable=SC2086 -- the arguments are words + mcpp test $ARGS + + - name: Save the project's home + if: ${{ always() && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && steps.home.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: ~/.mcpp + key: mcpp-project-home-${{ runner.os }}-${{ runner.arch }}-${{ steps.version.outputs.version }}-${{ inputs.home-key || hashFiles('**/mcpp.toml', '**/mcpp.lock', '.xlings.json') }} + + - name: Keep the run's store + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.name }} + path: target/mtest + if-no-files-found: ignore + include-hidden-files: true + retention-days: 14 + + - name: The run in the summary + if: always() + shell: bash + run: | + set -uo pipefail + run=$(ls -t target/mtest/runs 2>/dev/null | head -1) + [ -n "$run" ] || { echo "no run was kept" >> "$GITHUB_STEP_SUMMARY"; exit 0; } + d="target/mtest/runs/$run" + { + echo "## mcpp test" + jq -r '"\(.status // "?"): \(.counts // {} | to_entries | map("\(.value) \(.key)") | join(", ")) in \((.ms // 0) / 1000 | floor) s on \(.machine // "?")"' "$d/run.json" 2>/dev/null + echo + echo "| slowest case | ms |"; echo "|---|---|" + jq -rs 'sort_by(-(.ms // 0)) | .[:10][] | "| \(.program)/\(.id) | \(.ms // 0 | floor) |"' "$d/cases.ndjson" 2>/dev/null + if [ -s "$d/builds.ndjson" ]; then + echo + echo "| slowest edge | package | ms |"; echo "|---|---|---|" + jq -rs 'map(select(.output)) | sort_by(-(.ms // 0)) | .[:10][] | "| \(.output) | \(.package) | \(.ms // 0 | floor) |"' "$d/builds.ndjson" + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/use-built-mcpp/action.yml b/.github/actions/use-built-mcpp/action.yml index 7a87ff6fc..eb45b463a 100644 --- a/.github/actions/use-built-mcpp/action.yml +++ b/.github/actions/use-built-mcpp/action.yml @@ -29,16 +29,33 @@ inputs: description: The mirror xlings and mcpp use in this job. required: false default: GLOBAL + form: + description: > + `host`: the binary of build.yml (`mcpp-built-`). `release`: the + program of the archive release-build.yml kept for the host + (`release-`), as a release ships it -- ci.yml's `full` tier + tests it, so that a release promotes what a run tested (design + 2026-11-20 §5.22). + required: false + default: host runs: using: composite steps: - name: Download this commit's mcpp (${{ inputs.host }}) + if: ${{ inputs.form != 'release' }} uses: actions/download-artifact@v4 with: name: mcpp-built-${{ inputs.host }} path: ${{ runner.temp }}/mcpp-built + - name: Download this commit's release archive (${{ inputs.host }}) + if: ${{ inputs.form == 'release' }} + uses: actions/download-artifact@v4 + with: + name: release-${{ inputs.host == 'macos-arm64' && 'macosx-arm64' || inputs.host }} + path: ${{ runner.temp }}/mcpp-release + - name: Use this commit's mcpp shell: bash - run: bash "$GITHUB_ACTION_PATH/use.sh" "${{ inputs.host }}" "${{ inputs.mirror }}" + run: bash "$GITHUB_ACTION_PATH/use.sh" "${{ inputs.host }}" "${{ inputs.mirror }}" "${{ inputs.form }}" diff --git a/.github/actions/use-built-mcpp/use.sh b/.github/actions/use-built-mcpp/use.sh index 5602dd018..3d8d275b7 100644 --- a/.github/actions/use-built-mcpp/use.sh +++ b/.github/actions/use-built-mcpp/use.sh @@ -1,17 +1,37 @@ #!/usr/bin/env bash # The body of the use-built-mcpp action; see action.yml for what it provides. # -# Usage: use.sh +# Usage: use.sh [host|release] set -euo pipefail host="$1" mirror="$2" +form="${3:-host}" dir="$RUNNER_TEMP/mcpp-built" case "$host" in windows-*) exe=mcpp.exe; dir="$(cygpath -u "$dir")" ;; *) exe=mcpp ;; esac + +# The release form: the program of the host's versioned archive, as install.sh +# unpacks it, placed where the built binary would be. +if [ "$form" = release ]; then + rel="$RUNNER_TEMP/mcpp-release" + [ "${host#windows-}" != "$host" ] && rel="$(cygpath -u "$rel")" + archive=$(find "$rel" -maxdepth 1 -type f \( -name 'mcpp-*-*.tar.gz' -o -name 'mcpp-*-*.zip' \) \ + | grep -E '/mcpp-[0-9][^/]*-[a-z]' | head -1 || true) + [ -n "$archive" ] || { echo "::error::the release archive of $host is not in the artifact"; ls -la "$rel" || true; exit 1; } + unpacked="$RUNNER_TEMP/mcpp-release-root" + mkdir -p "$unpacked" "$dir" + case "$archive" in + *.zip) (cd "$unpacked" && unzip -q "$archive") ;; + *) tar -xzf "$archive" -C "$unpacked" ;; + esac + wrapper="$(basename "$archive")"; wrapper="${wrapper%.tar.gz}"; wrapper="${wrapper%.zip}" + cp "$unpacked/$wrapper/bin/$exe" "$dir/$exe" + echo "the program under test is the release form: $(basename "$archive")" +fi bin="$dir/$exe" if [ ! -f "$bin" ]; then echo "::error::the artifact mcpp-built-$host holds no $exe" diff --git a/.github/tools/main_run.sh b/.github/tools/main_run.sh new file mode 100755 index 000000000..34d9f492f --- /dev/null +++ b/.github/tools/main_run.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# main_run.sh -- print the run whose test artifacts are main's: main's last +# successful ci.yml run, or, when that run registered a verified run instead of +# testing (design 2026-11-20 §5.22), the run it registered. Prints nothing when +# main has no successful run. Reads GITHUB_REPOSITORY and GH_TOKEN. +# +# A push to main whose tree a `full` run of its pull request verified runs no +# test: its `register` job uploads `registered`, a file holding the verifying +# run's id. The stores, the unit archives and the build directories later runs +# read from main are that run's. +set -uo pipefail +repo="${GITHUB_REPOSITORY:?main_run.sh: GITHUB_REPOSITORY is not set}" +run=$(gh run list -R "$repo" --workflow ci.yml --branch main --status success --limit 1 \ + --json databaseId -q '.[0].databaseId' 2>/dev/null || true) +[ -n "$run" ] || exit 0 +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +if gh run download "$run" -R "$repo" -n registered -D "$tmp" > /dev/null 2>&1 && [ -s "$tmp/run" ]; then + tr -d '[:space:]' < "$tmp/run" + echo +else + echo "$run" +fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 124433ac4..34fede409 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,6 +69,23 @@ jobs: with: image: ${{ inputs.runs-on }} + # THE UNITS MAIN'S BUILDS COMPILED (design 2026-11-20 §5.3, CI5): a compile whose command, + # compiler and the contents of the files it read are those of a compile main kept is placed + # from the home's unit cache. Restored on every run; saved by this job on a push to main + # alone (R3), with the entries no build of this job used removed first, so the cache holds + # what main's build reads. A bootstrap mcpp without the unit cache leaves it as it was. + - name: Restore main's unit cache + uses: actions/cache/restore@v4 + with: + path: ~/.mcpp/build-cache/v1/unit + key: mcpp-units-${{ inputs.host }}-${{ github.sha }} + restore-keys: | + mcpp-units-${{ inputs.host }}- + + - name: The start of this job's use of the units + shell: bash + run: touch "$RUNNER_TEMP/units.mark" + - name: Build mcpp from source (self-host) shell: bash run: | @@ -179,6 +196,27 @@ jobs: path: ~/.xlings key: ${{ steps.boot.outputs.xlings-key }} + - name: Keep the units this job used + if: ${{ github.event_name == 'push' }} + shell: bash + run: | + set -euo pipefail + root="$HOME/.mcpp/build-cache/v1/unit" + [ -d "$root" ] || { echo "no unit cache: this build's mcpp keeps none"; exit 0; } + kept=0 dropped=0 + # An entry is //; a use stamps its manifest.tsv. + while IFS= read -r entry; do + if [ "$entry/manifest.tsv" -nt "$RUNNER_TEMP/units.mark" ]; then kept=$((kept + 1)); else rm -rf "$entry"; dropped=$((dropped + 1)); fi + done < <(find "$root" -mindepth 3 -maxdepth 3 -type d) + echo "units: $kept used by this job and kept, $dropped unused and removed" + + - name: Save main's unit cache + if: ${{ github.event_name == 'push' }} + uses: actions/cache/save@v4 + with: + path: ~/.mcpp/build-cache/v1/unit + key: mcpp-units-${{ inputs.host }}-${{ github.sha }} + # macOS keeps no sandbox cache: setup-macos-llvm builds the sandbox from # the freshly resolved LLVM on every run, which is what the macOS jobs # exist to prove. Its xlings cache has this job as its writer. diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml index 0ae71c16a..b3643585a 100644 --- a/.github/workflows/ci-linux.yml +++ b/.github/workflows/ci-linux.yml @@ -128,6 +128,9 @@ jobs: # be the same suite, toolchain and binary as `build-test`. toolchain-gcc: name: "toolchain: gcc (cold self-host)" + # Another toolchain's build of mcpp, or another project's: a checkpoint's (`full`), not each + # push's (design 2026-11-20 §7.2). + if: inputs.set == 'full' runs-on: ubuntu-24.04 timeout-minutes: 45 steps: @@ -150,6 +153,9 @@ jobs: # builder: a runner apiece would cost more in setup than it saves. toolchain-cross: name: "toolchain: musl + llvm" + # Another toolchain's build of mcpp, or another project's: a checkpoint's (`full`), not each + # push's (design 2026-11-20 §7.2). + if: inputs.set == 'full' runs-on: ubuntu-24.04 timeout-minutes: 45 steps: @@ -197,6 +203,9 @@ jobs: # resolves deps through. integration-xlings: name: "integration: mcpp builds & runs xlings" + # Another toolchain's build of mcpp, or another project's: a checkpoint's (`full`), not each + # push's (design 2026-11-20 §7.2). + if: inputs.set == 'full' runs-on: ubuntu-24.04 timeout-minutes: 45 env: diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index e869bbeb9..9ebeb454f 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -35,12 +35,11 @@ jobs: # behind an input so a PR that could not fix it held no green slot) — the # 23.1.3 line move removed that cause, and a change can change the outcome # here again. + # The newest image is a checkpoint's (`full`), not each push's (design 2026-11-20 §7.2). strategy: fail-fast: false matrix: - include: - - image: macos-15 - - image: xcode-27 + image: ${{ fromJSON(inputs.set == 'full' && '["macos-15", "xcode-27"]' || '["macos-15"]') }} runs-on: ${{ matrix.image }} # The C++ e2e programs run in this job (tests/e2e/*/, design 2026-11-12 # section 5.9); a change to tests/support runs them all, measured at 40 diff --git a/.github/workflows/ci-target-matrix.yml b/.github/workflows/ci-target-matrix.yml index dce57833e..f320eaa1c 100644 --- a/.github/workflows/ci-target-matrix.yml +++ b/.github/workflows/ci-target-matrix.yml @@ -463,11 +463,11 @@ jobs: # a claim nobody checked; a host that is scanned and never shipped is # coverage spent on a machine no user has. # - # Derived from release.yml's asset names rather than restated here, so + # Derived from release-build.yml's asset names rather than restated here, so # adding a fifth host to the release fails this step until the matrix # covers it. ship=$(grep -oE 'mcpp-\$\{?[A-Za-z_{}. ]*\}?-(linux|macosx|windows)-(x86_64|aarch64|arm64)' \ - .github/workflows/release.yml \ + .github/workflows/release-build.yml \ | sed -E 's/.*-(linux|macosx|windows)-/\1-/' \ | sed 's/^macosx-/macos-/' | sort -u) scanned=$(awk -F'\t' 'NF>=11 {print $2}' tests/matrix/expected.tsv | sort -u) diff --git a/.github/workflows/ci-unit.yml b/.github/workflows/ci-unit.yml index 5fdebc24a..9a74cf083 100644 --- a/.github/workflows/ci-unit.yml +++ b/.github/workflows/ci-unit.yml @@ -58,10 +58,29 @@ on: description: the run stage's cases of `mcpp-tests-` that need the MinGW cross toolchain, one per line type: string default: '' + select: + description: the run stage's cases, one `mcpp test --case` pattern per line; empty for every case + type: string + default: '' + jitter: + description: the jitter of the run stage's shards, in milliseconds (jitter.yml); empty for none + type: string + default: '' + jitter-seed: + description: the seed of the jitter + type: string + default: '' + form: + description: the run stage's program under test, `host` or `release` (use-built-mcpp `form`) + type: string + default: host outputs: archived: description: the number of tests the archive holds value: ${{ jobs.archive.outputs.archived }} + shards: + description: the run stage's shards, from the measured work of main's last run + value: ${{ jobs.archive.outputs.shards }} env: MCPP_HOME: /home/runner/.mcpp @@ -77,6 +96,7 @@ jobs: timeout-minutes: 60 outputs: archived: ${{ steps.archive.outputs.archived }} + shards: ${{ steps.shards.outputs.shards }} steps: - uses: actions/checkout@v4 @@ -111,8 +131,7 @@ jobs: HOST: ${{ inputs.host }} run: | set -uo pipefail - run=$(gh run list -R "$GITHUB_REPOSITORY" --workflow ci.yml --branch main --status success --limit 1 \ - --json databaseId -q '.[0].databaseId' 2>/dev/null || true) + run=$(bash .github/tools/main_run.sh) # main's, or the run it registered (design 2026-11-20 §5.22) [ -n "$run" ] || { echo "no successful run of main: the tests are built whole"; exit 0; } if ! gh run download "$run" -R "$GITHUB_REPOSITORY" -n "mcpp-unit-build-$HOST" -D "$RUNNER_TEMP/build" 2>/dev/null; then echo "main's run $run kept no test build for $HOST: the tests are built whole"; exit 0 @@ -165,8 +184,7 @@ jobs: run: | set -uo pipefail mkdir -p "$RUNNER_TEMP/unit/history" - run=$(gh run list -R "$GITHUB_REPOSITORY" --workflow ci.yml --branch main --status success --limit 1 \ - --json databaseId -q '.[0].databaseId' 2>/dev/null || true) + run=$(bash .github/tools/main_run.sh) # main's, or the run it registered (design 2026-11-20 §5.22) if [ -z "$run" ]; then echo "no successful run of main: the shards divide by count"; exit 0; fi d="$RUNNER_TEMP/unit-stores" if ! gh run download "$run" -R "$GITHUB_REPOSITORY" -p "mtest-unit-$KEY*" -D "$d" 2>/dev/null; then @@ -174,6 +192,28 @@ jobs: fi python3 .github/tools/test_history.py take "$RUNNER_TEMP/unit/history" "$d" "$d"/*/ + # THE SHARDS FROM THE PLAN (design 2026-11-20 §5.17, CI4): the measured work of main's last + # run, its shards' programs' durations summed (the history holds that run's shards alone, + # test_history.py), over what one shard does in 30 minutes with its four workers; at least + # one, at most eight. With no history, four, as before. + - name: The run stage's shards + id: shards + shell: bash + run: | + set -uo pipefail + work=$(find "$RUNNER_TEMP/unit/history" -name programs.ndjson -print0 2>/dev/null \ + | xargs -0 -r cat | jq -s 'map(.ms // 0) | add // 0') + if [ -z "$work" ] || [ "${work%.*}" = 0 ]; then + echo 'shards=[1,2,3,4]' >> "$GITHUB_OUTPUT"; echo "no measured work: four shards"; exit 0 + fi + per=$((30 * 60 * 1000 * 4)) + n=$(( (${work%.*} + per - 1) / per )) + [ "$n" -ge 1 ] || n=1 + [ "$n" -le 8 ] || n=8 + list=$(seq -s, 1 "$n") + echo "shards=[$list]" >> "$GITHUB_OUTPUT" + echo "measured work ${work%.*} ms: $n shard(s)" + - name: Upload the tests and their history if: steps.archive.outputs.archived != '0' timeout-minutes: 10 @@ -246,6 +286,7 @@ jobs: - uses: ./.github/actions/use-built-mcpp with: host: ${{ inputs.host }} + form: ${{ inputs.form }} - name: The tests of this run uses: actions/download-artifact@v4 @@ -253,8 +294,9 @@ jobs: name: mcpp-unit-${{ inputs.host }} path: ${{ runner.temp }}/unit - # The program the tests run is this commit's, as build.yml built it: an - # archive written with --no-programs takes it from MCPP_BIN_MCPP. + # The program the tests run is this commit's, as build.yml built it or, in + # ci.yml's `full` tier, as its release archive holds it: an archive + # written with --no-programs takes it from MCPP_BIN_MCPP. - name: The program under test shell: bash run: | @@ -300,6 +342,9 @@ jobs: name: ${{ inputs.name }}-${{ matrix.shard }} archive: ${{ runner.temp }}/unit/unit.tar.gz shard: ${{ matrix.shard }}/${{ strategy.job-total }} + cases: ${{ inputs.select }} + jitter: ${{ inputs.jitter }} + jitter-seed: ${{ inputs.jitter-seed }} # A compiler that did not start is a state of the home, not of a test: # the loader its payload is bound through (/registry/abi), the @@ -340,6 +385,7 @@ jobs: - uses: ./.github/actions/use-built-mcpp with: host: ${{ inputs.host }} + form: ${{ inputs.form }} # llvm beside it: the case that pairs clang with the MinGW payload reads # the program's resources with the payload's llvm-readobj (`llvm`). - name: Install mingw-cross toolchain diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb4efa0ca..7cb9b029a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,9 @@ on: branches: [ main ] pull_request: branches: [ main ] + # A merge queue's checks, when the repository enables one (design 2026-11-20 §5.22); without + # one the event never comes. + merge_group: workflow_dispatch: inputs: bench: @@ -56,9 +59,37 @@ jobs: outputs: code: ${{ steps.classify.outputs.code }} set: ${{ steps.set.outputs.set }} + tree: ${{ steps.tree.outputs.tree }} + verified: ${{ steps.tree.outputs.verified }} + verified_run: ${{ steps.tree.outputs.verified_run }} steps: - uses: actions/checkout@v4 + # A TREE IS VERIFIED ONCE (design 2026-11-20 §5.22). A `full` run that passes leaves the + # artifact `verified-` (the `verdict` job). A push to main whose tree has one -- the + # pull request was verified at its checkpoint and main has not moved since -- runs no test: + # it builds, which saves the caches only main's runs write (R3), and registers the verified + # run's stores as main's (`register`). Without one it runs `full`. + - name: The tree of this run, and whether a run verified it + id: tree + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + tree=$(git rev-parse 'HEAD^{tree}') + echo "tree=$tree" >> "$GITHUB_OUTPUT" + verified=false run="" + if [ "$EVENT" = push ]; then + run=$(gh api "repos/$REPO/actions/artifacts?name=verified-$tree&per_page=5" \ + --jq '[.artifacts[] | select(.expired | not)][0].workflow_run.id // empty' 2>/dev/null || true) + [ -n "$run" ] && verified=true + fi + echo "verified=$verified" >> "$GITHUB_OUTPUT" + echo "verified_run=$run" >> "$GITHUB_OUTPUT" + echo "tree $tree; verified by a run: $verified${run:+ ($run)}" + # The set of cases this run runs (design 2026-11-12 §5.11): a pull # request's push runs `pr`, the cases its change affects; a pull request # labelled `ci:full`, a push to main and a run started by hand run `full`. @@ -71,6 +102,9 @@ jobs: run: | set=full if [ "$EVENT" = pull_request ] && [[ " $LABELS " != *" ci:full "* ]]; then set=pr; fi + # The tiers widen to jobs (design 2026-11-20 §7.2): `pr` runs each host's build, gates, + # unit shards and the e2e the change affects; the other toolchains' and targets' jobs, the + # bootstrap and the newest macOS image run at a checkpoint (`full`). echo "set=$set" >> "$GITHUB_OUTPUT" echo "the set of this run: $set" @@ -93,6 +127,8 @@ jobs: gh api --paginate "repos/$REPO/pulls/$PR/files" \ --jq '.[] | .filename, (.previous_filename // empty)' > changed.txt \ || { full "the files of the pull request could not be listed"; exit 0; } ;; + merge_group) + full "a merge queue's group"; exit 0 ;; push) if [ -z "$BEFORE" ] || [ "$BEFORE" = 0000000000000000000000000000000000000000 ]; then full "a push with no previous commit"; exit 0 @@ -118,7 +154,7 @@ jobs: e2e-plan: name: the e2e shards of each host needs: changes - if: needs.changes.outputs.code == 'true' + if: ${{ needs.changes.outputs.code == 'true' && needs.changes.outputs.verified != 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 10 outputs: @@ -311,7 +347,7 @@ jobs: # and so does the newest release while the pin lags it (ci-bootstrap.yml). bootstrap: needs: changes - if: needs.changes.outputs.code == 'true' + if: ${{ needs.changes.outputs.code == 'true' && needs.changes.outputs.verified != 'true' && needs.changes.outputs.set == 'full' }} uses: ./.github/workflows/ci-bootstrap.yml # THE LINUX UNIT AND C++ E2E TESTS, built once beside the build and run in @@ -320,7 +356,7 @@ jobs: # program; the gate and the shards run them against build-linux's mcpp. unit-linux-archive: needs: changes - if: needs.changes.outputs.code == 'true' + if: ${{ needs.changes.outputs.code == 'true' && needs.changes.outputs.verified != 'true' }} # main's measured history is read from its last run. permissions: contents: read @@ -334,8 +370,21 @@ jobs: # The Windows cases the MinGW cross job runs (the run stage's `cases`). tests: e2e/windows + # THE ARCHIVES A RELEASE OF THIS TREE WOULD SHIP (design 2026-11-20 §5.22, CI2), in the `full` + # tier: release.yml's own builds. The Linux shards test the program of the Linux archive, and a + # release of a tree this run verifies promotes the four archives instead of building them. + release-form: + needs: changes + if: ${{ needs.changes.outputs.code == 'true' && needs.changes.outputs.verified != 'true' && needs.changes.outputs.set == 'full' }} + permissions: + contents: read + uses: ./.github/workflows/release-build.yml + unit-linux: - needs: [build-linux, unit-linux-archive, changes] + needs: [build-linux, unit-linux-archive, changes, release-form] + # The release form exists in the `full` tier alone: a skipped `release-form` does not skip the + # shards, a failed one does. + if: ${{ !cancelled() && !failure() && needs.changes.outputs.verified != 'true' && needs.unit-linux-archive.result == 'success' }} # A failed gate cancels the run (ci-unit.yml). permissions: contents: read @@ -347,9 +396,11 @@ jobs: name: linux runs-on: ubuntu-24.04 archived: ${{ needs.unit-linux-archive.outputs.archived }} - # Four shards of four workers: the measured run of 2026-11-13 reached 65 - # of its programs, 3970 s of work, in the 45 minutes of one job. - shards: '[1,2,3,4]' + # The shards the archive stage planned from main's measured work (four + # without a history): the measured run of 2026-11-13 reached 65 of its + # programs, 3970 s of work, in the 45 minutes of one job of four workers. + shards: ${{ needs.unit-linux-archive.outputs.shards || '[1,2,3,4]' }} + form: ${{ needs.release-form.result == 'success' && 'release' || 'host' }} # The Windows cases only a MinGW cross toolchain runs, from Linux: # resources (windres -O coff, since GNU ld cannot consume a .res; # mcpp#365) on the GNU and clang + MinGW rows, a Windows program packed @@ -364,6 +415,7 @@ jobs: linux: needs: [build-linux, changes] + if: ${{ needs.changes.outputs.verified != 'true' }} uses: ./.github/workflows/ci-linux.yml with: # A pull request's label is read when a push starts its run; a run @@ -373,6 +425,7 @@ jobs: linux-e2e: needs: [build-linux, changes, e2e-plan] + if: ${{ needs.changes.outputs.verified != 'true' }} uses: ./.github/workflows/ci-linux-e2e.yml with: set: ${{ needs.changes.outputs.set }} @@ -382,12 +435,14 @@ jobs: macos: needs: [build-macos, changes] + if: ${{ needs.changes.outputs.verified != 'true' }} uses: ./.github/workflows/ci-macos.yml with: set: ${{ needs.changes.outputs.set }} macos-e2e: needs: [build-macos, changes, e2e-plan] + if: ${{ needs.changes.outputs.verified != 'true' }} uses: ./.github/workflows/ci-macos-e2e.yml with: set: ${{ needs.changes.outputs.set }} @@ -396,17 +451,20 @@ jobs: work: ${{ needs.e2e-plan.outputs.macos_work != 'false' }} macos-ios: - needs: build-macos + needs: [build-macos, changes] + if: ${{ needs.changes.outputs.verified != 'true' && needs.changes.outputs.set == 'full' }} uses: ./.github/workflows/ci-macos-ios.yml windows: needs: [build-windows, changes] + if: ${{ needs.changes.outputs.verified != 'true' }} uses: ./.github/workflows/ci-windows.yml with: set: ${{ needs.changes.outputs.set }} windows-e2e: needs: [build-windows, changes, e2e-plan] + if: ${{ needs.changes.outputs.verified != 'true' }} uses: ./.github/workflows/ci-windows-e2e.yml with: set: ${{ needs.changes.outputs.set }} @@ -415,33 +473,37 @@ jobs: work: ${{ needs.e2e-plan.outputs.windows_work != 'false' }} windows-msvc-xlings: - needs: build-windows + needs: [build-windows, changes] + if: ${{ needs.changes.outputs.verified != 'true' && needs.changes.outputs.set == 'full' }} uses: ./.github/workflows/ci-windows-msvc-xlings.yml cross: - needs: [build-linux, build-windows] + needs: [build-linux, build-windows, changes] + if: ${{ needs.changes.outputs.verified != 'true' && needs.changes.outputs.set == 'full' }} uses: ./.github/workflows/cross-build-test.yml target-matrix: - needs: [build-linux, build-linux-arm, build-macos, build-windows] + needs: [build-linux, build-linux-arm, build-macos, build-windows, changes] + if: ${{ needs.changes.outputs.verified != 'true' && needs.changes.outputs.set == 'full' }} uses: ./.github/workflows/ci-target-matrix.yml # A home the previous release made, run by this commit's mcpp (D9 of the # 2026-10-17 upgrade-continuity design). upgrade: - needs: [build-linux, build-linux-arm, build-macos, build-windows] + needs: [build-linux, build-linux-arm, build-macos, build-windows, changes] + if: ${{ needs.changes.outputs.verified != 'true' && needs.changes.outputs.set == 'full' }} uses: ./.github/workflows/ci-upgrade.yml # Pull requests and dispatch only, as when it was a workflow of its own. openkal: - needs: [build-linux, build-linux-arm, build-macos, build-windows] - if: github.event_name != 'push' + needs: [build-linux, build-linux-arm, build-macos, build-windows, changes] + if: ${{ github.event_name != 'push' && needs.changes.outputs.verified != 'true' && needs.changes.outputs.set == 'full' }} uses: ./.github/workflows/openkal-cross.yml e2e-coverage: name: every e2e test runs somewhere needs: [changes, linux-e2e, windows-e2e, macos-e2e] - if: ${{ always() && needs.changes.outputs.code == 'true' && !cancelled() }} + if: ${{ always() && needs.changes.outputs.code == 'true' && !cancelled() && needs.changes.outputs.verified != 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 10 steps: @@ -471,11 +533,12 @@ jobs: # THE WORKFLOW RUN'S TESTS AS ONE REPORT (mtest `ci report`): the unit runs # each host kept (test-report, mtest-unit-*) and the e2e shards' stores # (e2e-store-*), the runs of this workflow run's group together -- the - # summary, the failures, the slowest cases and the metrics. + # summary, the failures, the slowest cases, the builds, the steps, the + # margins, the budget, the ledgers and the flips. test-report: name: the run's tests, reported needs: [changes, linux, unit-linux, macos, windows, linux-e2e, macos-e2e, windows-e2e] - if: ${{ always() && needs.changes.outputs.code == 'true' && !cancelled() }} + if: ${{ always() && needs.changes.outputs.code == 'true' && !cancelled() && needs.changes.outputs.verified != 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 15 steps: @@ -490,11 +553,26 @@ jobs: shell: bash run: | set -euo pipefail - for store in stores/*/; do + # In tests/e2e, whose mtest.toml states the budget and the ledgers the report judges. + cd tests/e2e + for store in "$GITHUB_WORKSPACE"/stores/*/; do [ -f "$store/index.ndjson" ] || continue "$MTEST" store import --from "$store" done - "$MTEST" ci report --group "$GITHUB_RUN_ID" --out test-report.md + # The sections of design 2026-11-20 §7.2: beside the summary and the failures, the builds' + # slowest edges, the steps, the cases near their limit, the budget's excesses, the surface's + # ledger and the cases whose verdict flipped. + "$MTEST" ci report --group "$GITHUB_RUN_ID" --out "$GITHUB_WORKSPACE/test-report.md" \ + --sections summary,failures,slowest,builds,steps,margin,budget,ledgers,flaky,path + # THE GATE OVER THE RUN'S GROUP (design 2026-11-20 §5.16, §5.17): no case over the budget + # without a waiver, every item of the surface used by a passing case or waived, and no waiver + # that excuses nothing. + - name: The gate over the run's group + shell: bash + run: | + set -euo pipefail + cd tests/e2e + "$MTEST" gate - uses: actions/upload-artifact@v4 if: always() with: @@ -502,3 +580,73 @@ jobs: path: test-report.md if-no-files-found: ignore retention-days: 14 + + # A PUSH TO MAIN WHOSE TREE IS VERIFIED (design 2026-11-20 §5.22) runs no test: the run that + # verified it is registered as main's, so that the stores, unit archives and build directories + # later runs read from main (.github/tools/main_run.sh) are that run's. The builds above still + # run, and save the caches only main's runs write (R3). + register: + name: the verified run is main's + needs: changes + if: ${{ needs.changes.outputs.verified == 'true' }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: The run that verified this tree + env: + RUN: ${{ needs.changes.outputs.verified_run }} + TREE: ${{ needs.changes.outputs.tree }} + run: | + set -euo pipefail + mkdir registered + echo "$RUN" > registered/run + { + echo "## Registered, not tested" + echo "The tree \`$TREE\` was verified by run [$RUN](https://github.com/$GITHUB_REPOSITORY/actions/runs/$RUN);" + echo "its stores and archives are main's, and this run builds and saves the caches." + } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 + with: + name: registered + path: registered/ + retention-days: 90 + + # THE RUN'S VERDICT, one job a ruleset can require: every job it needs succeeded or was skipped + # by the run's tier. A `full` run that passes leaves `verified-` (design 2026-11-20 + # §5.22): a push to main of the same tree registers this run instead of testing again, and a + # release of it promotes this run's archives (release.yml). + verdict: + name: verdict + needs: [changes, docs, build-linux, build-linux-arm, build-macos, build-windows, bootstrap, + unit-linux-archive, unit-linux, linux, linux-e2e, macos, macos-e2e, macos-ios, windows, + windows-e2e, windows-msvc-xlings, cross, target-matrix, upgrade, openkal, e2e-coverage, + test-report, release-form, register] + if: ${{ always() }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Every job passed or was not this tier's + env: + NEEDS: ${{ toJSON(needs) }} + run: | + set -euo pipefail + bad=$(jq -r 'to_entries[] | select(.value.result == "failure" or .value.result == "cancelled") | "\(.key): \(.value.result)"' <<< "$NEEDS") + if [ -n "$bad" ]; then + echo "::error::the run did not pass:"; echo "$bad"; exit 1 + fi + echo "every job passed or was not this run's" + - name: The tree's mark + if: ${{ needs.changes.outputs.set == 'full' && needs.changes.outputs.verified != 'true' }} + env: + TREE: ${{ needs.changes.outputs.tree }} + run: | + set -euo pipefail + mkdir verified + printf 'tree %s\ncommit %s\nrun %s\n' "$TREE" "$GITHUB_SHA" "$GITHUB_RUN_ID" > verified/mark + echo "The tree \`$TREE\` is verified by this run." >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 + if: ${{ needs.changes.outputs.set == 'full' && needs.changes.outputs.verified != 'true' }} + with: + name: verified-${{ needs.changes.outputs.tree }} + path: verified/ + retention-days: 90 diff --git a/.github/workflows/jitter.yml b/.github/workflows/jitter.yml new file mode 100644 index 000000000..bcca21f82 --- /dev/null +++ b/.github/workflows/jitter.yml @@ -0,0 +1,139 @@ +name: jitter + +# THE SEARCH FOR AN ORDER DEFECT (design 2026-11-20 §5.19, SPEC-017 O12). A case whose verdict +# depends on an order nothing fixes passes most of the time; jitter makes the other orders likely: +# each program waits a random time, up to the jitter, at its contention points -- mtest's at a +# case's start and at a shared object, mcpp's where it takes a lock and where it publishes. A run +# with jitter is a search, never a verdict of the tree: it runs on main's tree, apart from the pull +# requests' runs and the full verification. +# +# weekly every unit and C++ e2e case of Linux, three seeds. +# a flip a CI run of main ended with a case whose verdict changed within it (failed, then +# passed when its shard ran again, or `flaky`): that case alone, five seeds. +# by hand the cases, the jitter and the seeds given. +# +# The summary names each seed's result: a seed that fails repeats the order with +# `MTEST_JITTER= MTEST_JITTER_SEED= mcpp test --case `. + +on: + schedule: + - cron: "41 3 * * 6" + workflow_run: + workflows: [ci] + types: [completed] + branches: [main] + workflow_dispatch: + inputs: + cases: + description: the cases, one `mcpp test --case` pattern per line; empty for every case + type: string + default: '' + jitter: + description: the most a contention point waits, in milliseconds + type: string + default: '50' + seeds: + description: the seeds, a JSON list + type: string + default: '[1, 2, 3]' + +permissions: + contents: read + actions: read + +jobs: + # The cases whose verdict changed within the CI run that triggered this one, from its stores: + # a case with both a failure and a pass in the run's group, or one recorded `flaky`. + plan: + name: what the jitter runs + runs-on: ubuntu-24.04 + timeout-minutes: 10 + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion != 'cancelled' }} + outputs: + cases: ${{ steps.plan.outputs.cases }} + seeds: ${{ steps.plan.outputs.seeds }} + jitter: ${{ steps.plan.outputs.jitter }} + run: ${{ steps.plan.outputs.run }} + steps: + - uses: actions/checkout@v4 + - name: The cases, the seeds and the jitter + id: plan + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + RUN: ${{ github.event.workflow_run.id }} + CASES: ${{ inputs.cases }} + JITTER: ${{ inputs.jitter }} + SEEDS: ${{ inputs.seeds }} + run: | + set -euo pipefail + case "$EVENT" in + workflow_dispatch) + { echo "cases<> "$GITHUB_OUTPUT" + echo "seeds=$SEEDS" >> "$GITHUB_OUTPUT"; echo "jitter=$JITTER" >> "$GITHUB_OUTPUT" + echo "run=true" >> "$GITHUB_OUTPUT" ;; + schedule) + { echo "cases<> "$GITHUB_OUTPUT" + echo "seeds=[1, 2, 3]" >> "$GITHUB_OUTPUT"; echo "jitter=50" >> "$GITHUB_OUTPUT" + echo "run=true" >> "$GITHUB_OUTPUT" ;; + workflow_run) + mkdir -p stores + gh run download "$RUN" -R "$GITHUB_REPOSITORY" -p 'mtest-unit-*' -D stores 2>/dev/null || true + # `/` -> every status the run's attempts recorded. + flips=$(find stores -name cases.ndjson -print0 | xargs -0 -r cat \ + | jq -rs 'group_by(.program + "\u0000" + .id) + | map(select((map(.status) | (index("fail") != null and index("pass") != null)) + or (map(.status) | index("flaky") != null))) + | .[] | .[0].id' | sort -u) + { echo "cases<> "$GITHUB_OUTPUT" + echo "seeds=[1, 2, 3, 4, 5]" >> "$GITHUB_OUTPUT"; echo "jitter=50" >> "$GITHUB_OUTPUT" + if [ -n "$flips" ]; then echo "run=true" >> "$GITHUB_OUTPUT"; else echo "run=false" >> "$GITHUB_OUTPUT"; fi + echo "the cases whose verdict changed in run $RUN:"; echo "${flips:-(none)}" ;; + esac + + build-linux: + needs: plan + if: ${{ needs.plan.outputs.run == 'true' }} + permissions: + contents: read + actions: write + uses: ./.github/workflows/build.yml + with: + host: linux-x86_64 + runs-on: ubuntu-24.04 + gate: '' + + unit-archive: + needs: plan + if: ${{ needs.plan.outputs.run == 'true' }} + permissions: + contents: read + actions: read + uses: ./.github/workflows/ci-unit.yml + with: + stage: archive + host: linux-x86_64 + name: linux-jitter + runs-on: ubuntu-24.04 + + # One run per seed, each a shard of its own over the selected cases. + unit: + needs: [plan, build-linux, unit-archive] + strategy: + fail-fast: false + matrix: + seed: ${{ fromJSON(needs.plan.outputs.seeds) }} + permissions: + contents: read + actions: write + uses: ./.github/workflows/ci-unit.yml + with: + stage: run + host: linux-x86_64 + name: linux-jitter-${{ matrix.seed }} + runs-on: ubuntu-24.04 + archived: ${{ needs.unit-archive.outputs.archived }} + gate: '' + select: ${{ needs.plan.outputs.cases }} + jitter: ${{ needs.plan.outputs.jitter }} + jitter-seed: ${{ matrix.seed }} diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml new file mode 100644 index 000000000..08c072972 --- /dev/null +++ b/.github/workflows/release-build.yml @@ -0,0 +1,762 @@ +name: release-build + +# THE FOUR ARCHIVES OF A RELEASE, BUILT FROM ONE TREE (design 2026-11-20 §5.22, CI2). Called by +# release.yml, when no run verified the tag's tree, and by ci.yml in its `full` tier, whose +# Linux shards then test the program of the Linux archive: what a release promotes is what a run +# tested. Each job keeps its archives as the artifact `release-`; release.yml creates the +# release from them, from its own run or from the run that verified the tree. + +on: + workflow_call: + inputs: + tag: + description: the release's tag; empty for `v` of the tree's mcpp.toml + type: string + required: false + default: '' + ref: + description: the commit to build; empty for the caller's + type: string + required: false + default: '' + outputs: + tag: + value: ${{ jobs.build-release.outputs.tag }} + version: + value: ${{ jobs.build-release.outputs.version }} + +jobs: + build-release: + name: build (linux / x86_64) + runs-on: ubuntu-24.04 + permissions: + contents: read + timeout-minutes: 60 + outputs: + tag: ${{ steps.stage.outputs.tag }} + version: ${{ steps.stage.outputs.version }} + env: + # mcpp resolves MCPP_HOME from the binary's location by default, + # but here we want to share toolchains with the bootstrap sandbox, + # so we pin to a known path. + MCPP_HOME: /home/runner/.mcpp + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Resolve target tag + commit + id: resolve + # The caller's tag, else `v` of mcpp.toml: the tree is the + # commit the caller checked out (`ref`). + run: | + if [ -n "${{ inputs.tag }}" ]; then + TAG="${{ inputs.tag }}" + else + VER=$(awk -F '"' '/^version[[:space:]]*=/{print $2; exit}' mcpp.toml) + test -n "$VER" || { echo 'failed to read [package].version from mcpp.toml'; exit 1; } + TAG="v$VER" + fi + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "version=${TAG#v}" >> "$GITHUB_OUTPUT" + echo "Resolved tag: $TAG (commit $(git rev-parse --short HEAD))" + + # Cache mcpp's sandbox: musl-gcc 15.1 + binutils + glibc + linux-headers + # + patchelf + ninja is ~800 MB on disk; without this every release + # rebuilds from cold install. Key on the workspace manifest so a + # toolchain change in mcpp.toml refreshes the cache. + - name: Cache mcpp sandbox + uses: actions/cache@v4 + with: + path: ~/.mcpp + key: mcpp-sandbox-${{ runner.os }}-release-${{ hashFiles('mcpp.toml', '.xlings.json') }} + restore-keys: | + mcpp-sandbox-${{ runner.os }}-release- + + # Cache xlings + xim:mcpp install. + - name: Cache xlings + uses: actions/cache@v4 + with: + path: ~/.xlings + key: xlings-${{ runner.os }}-release-xl0462-${{ hashFiles('.xlings.json') }} + restore-keys: | + xlings-${{ runner.os }}-release-xl0462- + + - name: Bootstrap mcpp via xlings + env: + XLINGS_NON_INTERACTIVE: '1' + # Pin xlings to a known-good version. The upstream install + # script always grabs `latest` (no version override), so we + # download + self-install manually to avoid broken releases. + XLINGS_VERSION: '2026.11.17.2' + run: | + # BY VERSION, NOT BY PRESENCE. A restored ~/.xlings holds the + # xlings of an earlier release, and this binary is the one the + # tarball bundles: 2026.10.10.1 shipped xlings 2026.10.8.1 on + # linux-x86_64 because one was present. + have=$("$HOME/.xlings/subos/default/bin/xlings" --version 2>/dev/null | awk '{print $2}') + if [ "$have" != "$XLINGS_VERSION" ]; then + tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" + bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ + "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ + "/tmp/${tarball}" + tar -xzf "/tmp/${tarball}" -C /tmp + "/tmp/xlings-${XLINGS_VERSION}-linux-x86_64/subos/default/bin/xlings" self install + fi + export PATH="$HOME/.xlings/subos/default/bin:$PATH" + xlings --version + test "$(xlings --version | awk '{print $2}')" = "$XLINGS_VERSION" || { + echo "::error::the xlings this release bundles is $(xlings --version), not $XLINGS_VERSION"; exit 1; } + # Pinned to .xlings.json — a bare `xlings install mcpp` resolves + # "newest in this runner's index copy" and put 0.0.105 (below the + # index floor) into this job. See .github/tools/install_pinned_mcpp.sh. + MCPP=$(bash "$GITHUB_WORKSPACE/.github/tools/install_pinned_mcpp.sh" "$GITHUB_WORKSPACE") + echo "MCPP=$MCPP" >> "$GITHUB_ENV" + echo "XLINGS_BIN=$HOME/.xlings/subos/default/bin/xlings" >> "$GITHUB_ENV" + + - name: Build + pack release artefact (musl static) + id: stage + # Build for the musl-static target, strip the produced ELF, then + # let `mcpp pack` assemble the tarball (binary + top-level wrapper + # + README + LICENSE, contents at archive root). Inject xlings + # afterwards so install.sh consumers get a single self-contained + # bundle. + run: | + TAG="${{ steps.resolve.outputs.tag }}" + VERSION="${{ steps.resolve.outputs.version }}" + TARBALL_NAME="mcpp-${VERSION}-linux-x86_64.tar.gz" + + # Build first so we can strip the ELF before pack copies it. + export MCPP_VENDORED_XLINGS="$XLINGS_BIN" + ARTIFACT=$(bash "$GITHUB_WORKSPACE/.github/tools/built_artifact.sh" "$MCPP" mcpp --target x86_64-linux-musl) + test -n "$ARTIFACT" + file "$ARTIFACT" | grep -q 'statically linked' + # NB: stripping here is pointless — `mcpp pack` below rebuilds the + # binary and overwrites it, which is why every release up to and + # including 2026.7.28.2 shipped an UNSTRIPPED bin/mcpp despite the + # strip that used to live on this line. The payload is slimmed after + # packing instead, in the inject step below. + + # Pack with the freshly-built mcpp (not the bootstrap) so any + # fixes to the pack code path are exercised in the same release + # they ship in. MCPP_HOME is forced so the new binary uses the + # pinned sandbox instead of resolving relative to its own + # location under target/. + MCPP_HOME="$MCPP_HOME" "$ARTIFACT" pack \ + --target x86_64-linux-musl \ + --mode static \ + -o "${TARBALL_NAME}" + + # Inject xlings: extract → add registry/bin/xlings to the wrapper + # dir → re-tar preserving the wrapper. Since 0.0.4 the bundled + # xlings lives at /registry/bin/xlings (= /bin/xlings). + TARBALL="target/dist/${TARBALL_NAME}" + WRAPPER="${TARBALL_NAME%.tar.gz}" + test -f "$TARBALL" + INJECT=$(mktemp -d) + tar -xzf "$TARBALL" -C "$INJECT" + mkdir -p "$INJECT/$WRAPPER/registry/bin" + cp "$XLINGS_BIN" "$INJECT/$WRAPPER/registry/bin/xlings" + chmod +x "$INJECT/$WRAPPER/registry/bin/xlings" + # Slim AFTER pack (pack rebuilds bin/mcpp) and BEFORE tar. Asserts + # the result, so a strip that silently stops working fails the + # release instead of quietly shipping a 34.8MB tarball again. + bash .github/tools/slim_linux_payload.sh "$INJECT/$WRAPPER" + (cd "$INJECT" && tar -czf "$GITHUB_WORKSPACE/${TARBALL}" "$WRAPPER") + rm -rf "$INJECT" + + # Stage final dist/ (tarball + sidecars) for upload. + mkdir -p dist + cp "$TARBALL" "dist/${TARBALL_NAME}" + (cd dist && cp "${TARBALL_NAME}" "mcpp-linux-x86_64.tar.gz") + (cd dist && sha256sum "${TARBALL_NAME}" "mcpp-linux-x86_64.tar.gz" > SHA256SUMS) + (cd dist && sha256sum "${TARBALL_NAME}" > "${TARBALL_NAME}.sha256") + (cd dist && sha256sum "mcpp-linux-x86_64.tar.gz" > "mcpp-linux-x86_64.tar.gz.sha256") + + # Top-level install.sh — fetched by `curl | bash`. + cp install.sh dist/install.sh + chmod +x dist/install.sh + + echo "tag=$TAG" >> $GITHUB_OUTPUT + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "tarball=${TARBALL_NAME}" >> $GITHUB_OUTPUT + ls -la dist/ + + - name: Smoke-test the bundled tarball + # Extract to a scratch dir and run mcpp from there with MCPP_HOME + # unset — proves the release artefact is genuinely self-contained. + run: | + VERSION="${{ steps.stage.outputs.version }}" + TARBALL_NAME="${{ steps.stage.outputs.tarball }}" + # Wrapper dir inside the tarball matches its stem (mcpp pack + # ties the two together). + WRAPPER="${TARBALL_NAME%.tar.gz}" + SMOKE=$(mktemp -d) + tar -xzf "dist/${TARBALL_NAME}" -C "$SMOKE" + ROOT="$SMOKE/$WRAPPER" + test -x "$ROOT/bin/mcpp" + test -x "$ROOT/registry/bin/xlings" + test -x "$ROOT/mcpp" + file "$ROOT/bin/mcpp" | grep -q 'statically linked' + env -u MCPP_HOME "$ROOT/bin/mcpp" --version + env -u MCPP_HOME "$ROOT/bin/mcpp" --help | head -10 + # Top-level wrapper reports the same version we're shipping. + env -u MCPP_HOME "$ROOT/mcpp" --version | grep -q "$VERSION" + # MCPP_HOME should auto-resolve to the extracted root. + out=$(env -u MCPP_HOME "$ROOT/bin/mcpp" self env) + # R4 (mcpp#832): the binary pack rebuilt with this release's own + # engine trims the paths it carries. + bash .github/tools/check_release_paths.sh "$ROOT/bin/mcpp" + echo "$out" | grep -q "MCPP_HOME *= *$ROOT" + + - name: Generate source tarball + xpkg.lua via mcpp publish + # Use the freshly-built mcpp to produce the source tarball + xpkg + # descriptor for mcpp-index. The release tarball wraps its + # contents in a `/` directory so the extract path + # is $PUB/$WRAPPER/bin/mcpp. + run: | + VERSION="${{ steps.stage.outputs.version }}" + TARBALL_NAME="${{ steps.stage.outputs.tarball }}" + WRAPPER="${TARBALL_NAME%.tar.gz}" + PUB=$(mktemp -d) + tar -xzf "dist/${TARBALL_NAME}" -C "$PUB" + MCPP_BIN="$PUB/$WRAPPER/bin/mcpp" + env -u MCPP_HOME "$MCPP_BIN" publish --dry-run --allow-dirty + test -f "target/dist/mcpp-${VERSION}.tar.gz" + test -f "target/dist/mcpp.lua" + cp "target/dist/mcpp-${VERSION}.tar.gz" dist/ + cp "target/dist/mcpp.lua" dist/ + ls -la dist/ + + - name: Extract release notes from CHANGELOG + id: notes + run: | + TAG="${{ steps.stage.outputs.tag }}" + VERSION="${{ steps.stage.outputs.version }}" + awk -v v="$VERSION" ' + /^## \[/ { + if (in_section) exit + if ($0 ~ "\\[" v "\\]") { in_section=1; next } + } + in_section { print } + ' CHANGELOG.md > dist/RELEASE_NOTES.md || true + # A RELEASE IS OUTWARD-FACING AND PERMANENT: publishing one whose + # notes silently came out empty is not a lesser failure than any + # other defect this workflow guards against, and until now it + # produced none — v2026.9.18.1 went out with its notes body reading + # literally "(no CHANGELOG entry found for 2026.9.18.1)", because + # CHANGELOG.md had no matching `## []` heading and this + # step wrote a placeholder and moved on rather than stopping. Stop + # instead: no matching section is an authoring mistake (the + # section is missing, or its version string does not match what is + # being tagged), and the fix belongs in CHANGELOG.md before the + # release is retried, not in a release whose notes nobody reads. + if [ ! -s dist/RELEASE_NOTES.md ]; then + echo "::error::CHANGELOG.md has no '## [$VERSION]' section — refusing to publish a release with empty notes. Add that section (docs/92-release.md) and re-run, rather than let this placeholder ship: (no CHANGELOG entry found for $VERSION)" + exit 1 + fi + echo "--- RELEASE_NOTES.md ---" + cat dist/RELEASE_NOTES.md + + - name: Keep the Linux archives for the release + uses: actions/upload-artifact@v4 + with: + name: release-linux-x86_64 + if-no-files-found: error + retention-days: 7 + path: | + dist/mcpp-${{ steps.stage.outputs.version }}-linux-x86_64.tar.gz + dist/mcpp-${{ steps.stage.outputs.version }}-linux-x86_64.tar.gz.sha256 + dist/mcpp-linux-x86_64.tar.gz + dist/mcpp-linux-x86_64.tar.gz.sha256 + dist/install.sh + dist/mcpp-${{ steps.stage.outputs.version }}.tar.gz + dist/mcpp.lua + dist/RELEASE_NOTES.md + + build-linux-aarch64: + name: build (linux / aarch64, cross) + runs-on: ubuntu-24.04 + permissions: + contents: read + timeout-minutes: 70 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Install system deps + qemu + run: | + # The runner image carries third-party apt lists (Google Chrome + # among them) that this job does not use, and a transient + # `Hash Sum mismatch` on one of them fails the whole update -- which + # killed two cross-build jobs in setup, before a single byte was + # compiled. Dropping the lists this job has no use for is what makes + # the step's failure mean something about this job. + # + # BY CONTENT, NOT BY FILENAME. The first attempt removed + # `google-chrome.list` and the update failed on the same URL: on + # ubuntu-24.04 the runner writes deb822 `.sources` files, so the + # name was a guess and the guess was wrong. + sudo grep -rlE 'dl[.]google[.]com|packages[.]microsoft[.]com' \ + /etc/apt/sources.list.d/ 2>/dev/null | xargs -r sudo rm -f + sudo apt-get update -qq + sudo apt-get install -y curl git build-essential qemu-user-static + qemu-aarch64-static --version | head -1 + + - name: Resolve tag + version + id: resolve + run: | + VERSION=$(grep -E '^version' mcpp.toml | head -1 | sed 's/.*"\([^"]*\)".*/\1/') + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "tag=v$VERSION" >> "$GITHUB_OUTPUT" + + - name: Bootstrap mcpp via xlings + env: + XLINGS_NON_INTERACTIVE: '1' + XLINGS_VERSION: '2026.11.17.2' + run: | + tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" + bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ + "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ + "/tmp/${tarball}" + tar -xzf "/tmp/${tarball}" -C /tmp + "/tmp/xlings-${XLINGS_VERSION}-linux-x86_64/subos/default/bin/xlings" self install + echo "$HOME/.xlings/subos/current/bin" >> "$GITHUB_PATH" + echo "$HOME/.xlings/bin" >> "$GITHUB_PATH" + echo "XLINGS_BIN=$HOME/.xlings/subos/default/bin/xlings" >> "$GITHUB_ENV" + + - name: Bootstrap mcpp + refresh index (latest, GLOBAL) + run: | + xlings config --mirror GLOBAL 2>/dev/null || true + xlings update -y 2>/dev/null || xlings update 2>/dev/null || true + MCPP=$(bash "$GITHUB_WORKSPACE/.github/tools/install_pinned_mcpp.sh" "$GITHUB_WORKSPACE") + echo "MCPP=$MCPP" >> "$GITHUB_ENV" + "$MCPP" --version + + - name: Cross-build mcpp -> aarch64-linux-musl (this release's source) + run: | + export MCPP_VENDORED_XLINGS="$XLINGS_BIN" + # "$MCPP", not a bare `mcpp`: the bare form runs the PATH shim, which + # resolves to whatever version xvm has selected — so pinning the + # bootstrap would have looked correct and changed nothing here. + "$MCPP" self config --mirror GLOBAL 2>/dev/null || true + # The published bootstrap mcpp predates aarch64 cross-build support + # (the feature landed after the last release), so it resolves the + # x86_64 host musl-gcc for an aarch64 target. Two-stage instead: build + # THIS release's x86_64 mcpp first, then cross-build aarch64 with it. + FRESH=$(bash "$GITHUB_WORKSPACE/.github/tools/built_artifact.sh" "$MCPP" mcpp --target x86_64-linux-musl) + test -n "$FRESH" + BIN=$(bash "$GITHUB_WORKSPACE/.github/tools/built_artifact.sh" "$FRESH" mcpp --target aarch64-linux-musl) + test -n "$BIN" + file "$BIN" | grep -q 'ARM aarch64' + file "$BIN" | grep -q 'statically linked' + echo "MCPP_AARCH64=$BIN" >> "$GITHUB_ENV" + + - name: Package aarch64 release (+ bundle aarch64 xlings) + id: stage + run: | + VERSION="${{ steps.resolve.outputs.version }}" + TARBALL_NAME="mcpp-${VERSION}-linux-aarch64.tar.gz" + WRAPPER="mcpp-${VERSION}-linux-aarch64" + STAGING=$(mktemp -d) + mkdir -p "$STAGING/$WRAPPER/bin" + cp "$MCPP_AARCH64" "$STAGING/$WRAPPER/bin/mcpp" + # The binary is aarch64, so the host x86_64 `strip` cannot touch it — + # resolve a cross-capable one. Required, not best-effort: the old + # `|| true` here meant a missing tool silently shipped a fat tarball. + # (slim_linux_payload.sh runs after xlings is staged, below.) + STRIP=$(find "$HOME/.mcpp" -name 'aarch64-linux-musl-strip' -type f 2>/dev/null | head -1) + [ -n "$STRIP" ] || STRIP=$(command -v llvm-strip 2>/dev/null || true) + [ -n "$STRIP" ] || STRIP=$(command -v aarch64-linux-gnu-strip 2>/dev/null || true) + [ -n "$STRIP" ] || { echo "no aarch64-capable strip found"; exit 1; } + echo "aarch64 strip: $STRIP" + cp LICENSE "$STAGING/$WRAPPER/" 2>/dev/null || true + cp README.md "$STAGING/$WRAPPER/" 2>/dev/null || true + cat > "$STAGING/$WRAPPER/mcpp" << 'LAUNCHER' + #!/bin/sh + exec "$(dirname "$0")/bin/mcpp" "$@" + LAUNCHER + chmod +x "$STAGING/$WRAPPER/mcpp" + # Bundle the aarch64 xlings so install.sh consumers on aarch64 get an + # aarch64 xlings, not the x86_64 bootstrap one. The three literals + # below are pinned to the same version as XLINGS_VERSION; they are + # NOT interpolated from it, so check_version_pins.sh scans for them + # explicitly (they were absent from the old lock-step comment). + XLA="xlings-2026.11.17.2-linux-aarch64.tar.gz" + # NOT fetch_release.sh: this asset is OPTIONAL and the `if` is the + # point — an arch with no prebuilt xlings must fall through quietly, + # while the helper retries a 404 five times before giving up. The one + # flag that matters here is --retry-all-errors: `curl: (52) Empty + # reply from server` is a transport error, so plain --retry does not + # cover it. + if curl -fsSL --retry 3 --retry-delay 2 --retry-all-errors \ + --connect-timeout 20 --max-time 600 -o "/tmp/$XLA" \ + "https://github.com/openxlings/xlings/releases/download/v2026.11.17.2/$XLA"; then + tar -xzf "/tmp/$XLA" -C /tmp + XLBIN=$(find /tmp/xlings-2026.11.17.2-linux-aarch64 -path '*/bin/xlings' -type f | head -1) + if [ -n "$XLBIN" ]; then + mkdir -p "$STAGING/$WRAPPER/registry/bin" + cp "$XLBIN" "$STAGING/$WRAPPER/registry/bin/xlings" + chmod +x "$STAGING/$WRAPPER/registry/bin/xlings" + fi + fi + # Slim both shipped ELFs with the cross strip resolved above, and + # assert the result (the vendored xlings was 86.9MB unstripped here). + bash .github/tools/slim_linux_payload.sh "$STAGING/$WRAPPER" "$STRIP" + mkdir -p dist + (cd "$STAGING" && tar -czf "$GITHUB_WORKSPACE/dist/${TARBALL_NAME}" "$WRAPPER") + cp "dist/${TARBALL_NAME}" "dist/mcpp-linux-aarch64.tar.gz" + (cd dist && sha256sum "${TARBALL_NAME}" > "${TARBALL_NAME}.sha256") + (cd dist && sha256sum "mcpp-linux-aarch64.tar.gz" > "mcpp-linux-aarch64.tar.gz.sha256") + echo "tarball=${TARBALL_NAME}" >> "$GITHUB_OUTPUT" + ls -la dist/ + + - name: Smoke-test the aarch64 tarball (qemu) + run: | + TARBALL_NAME="${{ steps.stage.outputs.tarball }}" + WRAPPER="${TARBALL_NAME%.tar.gz}" + SMOKE=$(mktemp -d) + tar -xzf "dist/${TARBALL_NAME}" -C "$SMOKE" + ver=$(qemu-aarch64-static "$SMOKE/$WRAPPER/bin/mcpp" --version) + echo "$ver"; echo "$ver" | grep -q 'mcpp' + + - name: Keep the aarch64 archives for the release + uses: actions/upload-artifact@v4 + with: + name: release-linux-aarch64 + if-no-files-found: error + retention-days: 7 + path: | + dist/mcpp-${{ steps.resolve.outputs.version }}-linux-aarch64.tar.gz + dist/mcpp-${{ steps.resolve.outputs.version }}-linux-aarch64.tar.gz.sha256 + dist/mcpp-linux-aarch64.tar.gz + dist/mcpp-linux-aarch64.tar.gz.sha256 + + build-macos: + name: build (macOS / ARM64) + runs-on: macos-15 + permissions: + contents: read + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Resolve tag + id: resolve + run: | + if [ -n "${{ inputs.tag }}" ]; then + TAG="${{ inputs.tag }}" + else + VER=$(awk -F '"' '/^version[[:space:]]*=/{print $2; exit}' mcpp.toml) + TAG="v$VER" + fi + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "version=${TAG#v}" >> "$GITHUB_OUTPUT" + + - name: Cache xlings + uses: actions/cache@v4 + with: + path: ~/.xlings + key: xlings-macos15-release-xl0462-${{ hashFiles('.xlings.json') }} + restore-keys: | + xlings-macos15-release-xl0462- + + - name: Bootstrap mcpp via xlings + env: + XLINGS_NON_INTERACTIVE: '1' + XLINGS_VERSION: '2026.11.17.2' + run: | + # By version, not by presence (see the Linux job). + have=$("$HOME/.xlings/subos/default/bin/xlings" --version 2>/dev/null | awk '{print $2}') + if [ "$have" != "$XLINGS_VERSION" ]; then + WORK=$(mktemp -d) + tarball="xlings-${XLINGS_VERSION}-macosx-arm64.tar.gz" + bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ + "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ + "${WORK}/${tarball}" + tar -xzf "${WORK}/${tarball}" -C "${WORK}" + "${WORK}/xlings-${XLINGS_VERSION}-macosx-arm64/subos/default/bin/xlings" self install + fi + export PATH="$HOME/.xlings/subos/default/bin:$PATH" + xlings --version + test "$(xlings --version | awk '{print $2}')" = "$XLINGS_VERSION" || { + echo "::error::the xlings this release bundles is $(xlings --version), not $XLINGS_VERSION"; exit 1; } + # Pinned to .xlings.json — a bare `xlings install mcpp` resolves + # "newest in this runner's index copy" and put 0.0.105 (below the + # index floor) into this job. See .github/tools/install_pinned_mcpp.sh. + MCPP=$(bash "$GITHUB_WORKSPACE/.github/tools/install_pinned_mcpp.sh" "$GITHUB_WORKSPACE") + echo "MCPP=$MCPP" >> "$GITHUB_ENV" + echo "XLINGS_BIN=$HOME/.xlings/subos/default/bin/xlings" >> "$GITHUB_ENV" + + - name: Build mcpp from source (two-stage self-host) + env: + # macOS min-version support: target macOS 14 so the release runs + # on 14.0+ instead of only the runner's OS (the official LLVM + # static libc++ archives are built for macOS 14 — going lower + # needs a custom libc++ build, tracked as follow-up). Needs + # static LLVM libc++ — the system libc++ on older macOS lacks + # LLVM-20-era C++23 symbols (std::print's __is_posix_terminal + # etc.; minos-14 + dynamic libc++ dies at launch on macos-14 CI). + # See xlings .agents/docs/2026-06-05-macos-min-version-support.md. + MACOSX_DEPLOYMENT_TARGET: '14.0' + run: | + export PATH="$HOME/.xlings/subos/default/bin:$PATH" + export MCPP_VENDORED_XLINGS="$XLINGS_BIN" + + # Stage 1: the bootstrap mcpp builds this release's source. The + # bootstrap's macOS link path predates the staticStdlib + # implementation (hardcoded -lc++), so stage 1 links the system + # libc++ — fine, it only needs to RUN on this runner. + STAGE1=$(bash "$GITHUB_WORKSPACE/.github/tools/built_artifact.sh" "$MCPP" mcpp) + # --no-cache removes target/, including the stage-1 compiler. Ninja + # invokes that compiler again for dependency scanning, so keep the + # bootstrap outside the tree the second stage is about to clear. + STAGE1_COPY=$(mktemp -d "$RUNNER_TEMP/mcpp-stage1.XXXXXX")/mcpp + cp "$STAGE1" "$STAGE1_COPY" + STAGE1="$STAGE1_COPY" + "$STAGE1" --version + + # Stage 2: this release's mcpp rebuilds itself — flags.cppm's + # native staticStdlib link produces the static minos-14 binary. + # Stage 2 lands in a NEW fingerprint directory (its fingerprint + # includes the deployment target; the bootstrap's did not): the + # build names its own product. + MCPP_BIN=$(bash "$GITHUB_WORKSPACE/.github/tools/built_artifact.sh" "$STAGE1" mcpp --no-cache) + test -x "$MCPP_BIN" + file "$MCPP_BIN" + otool -L "$MCPP_BIN" + echo "=== LC_BUILD_VERSION (must be minos 14.0) ===" + otool -l "$MCPP_BIN" | grep -A4 LC_BUILD_VERSION | head -6 + otool -l "$MCPP_BIN" | grep -A4 LC_BUILD_VERSION | grep -q "minos 14.0" \ + || { echo "FAIL: expected minos 14.0"; exit 1; } + if otool -L "$MCPP_BIN" | grep -q "libc++"; then + echo "FAIL: still linked against system libc++"; exit 1 + fi + "$MCPP_BIN" --version + echo "MCPP_BIN=$MCPP_BIN" >> "$GITHUB_ENV" + + - name: Package macOS release + id: stage + run: | + VERSION="${{ steps.resolve.outputs.version }}" + TARBALL_NAME="mcpp-${VERSION}-macosx-arm64.tar.gz" + WRAPPER="mcpp-${VERSION}-macosx-arm64" + + # Create release layout + STAGING=$(mktemp -d) + mkdir -p "$STAGING/$WRAPPER/bin" + cp "$MCPP_BIN" "$STAGING/$WRAPPER/bin/mcpp" + # Strip (Mach-O) + strip "$STAGING/$WRAPPER/bin/mcpp" 2>/dev/null || true + # Copy metadata + cp LICENSE "$STAGING/$WRAPPER/" 2>/dev/null || true + cp README.md "$STAGING/$WRAPPER/" 2>/dev/null || true + + # Shell launcher (same as Linux) + cat > "$STAGING/$WRAPPER/mcpp" << 'LAUNCHER' + #!/bin/sh + exec "$(dirname "$0")/bin/mcpp" "$@" + LAUNCHER + chmod +x "$STAGING/$WRAPPER/mcpp" + + # Bundle xlings for install.sh consumers + XLINGS_BIN="$HOME/.xlings/subos/default/bin/xlings" + if [ -x "$XLINGS_BIN" ]; then + mkdir -p "$STAGING/$WRAPPER/registry/bin" + cp "$XLINGS_BIN" "$STAGING/$WRAPPER/registry/bin/xlings" + chmod +x "$STAGING/$WRAPPER/registry/bin/xlings" + fi + + # Create tarball + mkdir -p dist + (cd "$STAGING" && tar -czf "$GITHUB_WORKSPACE/dist/${TARBALL_NAME}" "$WRAPPER") + # Versionless alias + cp "dist/${TARBALL_NAME}" "dist/mcpp-macosx-arm64.tar.gz" + # SHA256 + (cd dist && shasum -a 256 "${TARBALL_NAME}" > "${TARBALL_NAME}.sha256") + (cd dist && shasum -a 256 "mcpp-macosx-arm64.tar.gz" > "mcpp-macosx-arm64.tar.gz.sha256") + + echo "tarball=${TARBALL_NAME}" >> "$GITHUB_OUTPUT" + ls -la dist/ + + - name: Smoke-test the tarball + run: | + VERSION="${{ steps.resolve.outputs.version }}" + TARBALL_NAME="${{ steps.stage.outputs.tarball }}" + WRAPPER="${TARBALL_NAME%.tar.gz}" + SMOKE=$(mktemp -d) + tar -xzf "dist/${TARBALL_NAME}" -C "$SMOKE" + "$SMOKE/$WRAPPER/bin/mcpp" --version + "$SMOKE/$WRAPPER/mcpp" --version | grep -q "$VERSION" + # R4 (mcpp#832): stage 2 is this release's own engine, and it trims + # the paths the binary carries. + bash .github/tools/check_release_paths.sh "$SMOKE/$WRAPPER/bin/mcpp" + + - name: Keep the macOS archives for the release + uses: actions/upload-artifact@v4 + with: + name: release-macosx-arm64 + if-no-files-found: error + retention-days: 7 + path: | + dist/mcpp-${{ steps.resolve.outputs.version }}-macosx-arm64.tar.gz + dist/mcpp-${{ steps.resolve.outputs.version }}-macosx-arm64.tar.gz.sha256 + dist/mcpp-macosx-arm64.tar.gz + dist/mcpp-macosx-arm64.tar.gz.sha256 + + build-windows: + name: build (Windows / x86_64) + runs-on: windows-latest + permissions: + contents: read + timeout-minutes: 45 + env: + MCPP_HOME: C:\Users\runneradmin\.mcpp + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Resolve tag + id: resolve + shell: bash + run: | + if [ -n "${{ inputs.tag }}" ]; then + TAG="${{ inputs.tag }}" + else + VER=$(awk -F '"' '/^version[[:space:]]*=/{print $2; exit}' mcpp.toml) + TAG="v$VER" + fi + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "version=${TAG#v}" >> "$GITHUB_OUTPUT" + + - name: Cache mcpp sandbox + uses: actions/cache@v4 + with: + path: ~\.mcpp + key: mcpp-sandbox-${{ runner.os }}-release-${{ hashFiles('mcpp.toml', '.xlings.json') }} + restore-keys: | + mcpp-sandbox-${{ runner.os }}-release- + + - name: Cache xlings + uses: actions/cache@v4 + with: + path: ~\.xlings + key: xlings-${{ runner.os }}-release-xl0462-${{ hashFiles('.xlings.json') }} + restore-keys: | + xlings-${{ runner.os }}-release-xl0462- + + - name: Bootstrap mcpp via xlings + shell: bash + env: + XLINGS_NON_INTERACTIVE: '1' + XLINGS_VERSION: '2026.11.17.2' + run: | + # Captured before the `cd` below, in POSIX form: this step never + # returns to the workspace, and GITHUB_WORKSPACE is a backslash + # Windows path that git-bash tools mangle. + REPO_DIR="$(pwd)" + WORK=$(mktemp -d) + zipfile="xlings-${XLINGS_VERSION}-windows-x86_64.zip" + bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ + "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${zipfile}" \ + "${WORK}/${zipfile}" + cd "${WORK}" + unzip -q "${zipfile}" + "$WORK/xlings-${XLINGS_VERSION}-windows-x86_64/subos/default/bin/xlings.exe" self install + export PATH="$USERPROFILE/.xlings/subos/default/bin:$PATH" + echo "$USERPROFILE/.xlings/subos/default/bin" >> "$GITHUB_PATH" + xlings.exe --version + test "$(xlings.exe --version | awk '{print $2}' | tr -d '\r')" = "$XLINGS_VERSION" || { + echo "::error::the xlings this release bundles is not $XLINGS_VERSION"; exit 1; } + # Pinned + version-scoped lookup. The old `find | head -1` returned + # whichever version the directory walk reached first. + MCPP=$(bash "$REPO_DIR/.github/tools/install_pinned_mcpp.sh" "$REPO_DIR") + echo "MCPP=$MCPP" >> "$GITHUB_ENV" + XLINGS_BIN=$(cygpath -w "$USERPROFILE/.xlings/subos/default/bin/xlings.exe") + echo "XLINGS_BIN=$XLINGS_BIN" >> "$GITHUB_ENV" + echo "XLINGS_BIN_UNIX=$USERPROFILE/.xlings/subos/default/bin/xlings.exe" >> "$GITHUB_ENV" + echo "XLINGS_XPKGS=$USERPROFILE/.xlings/data/xpkgs" >> "$GITHUB_ENV" + + - name: Build mcpp from source (self-host) + shell: bash + run: | + export MCPP_VENDORED_XLINGS="$XLINGS_BIN" + + "$MCPP" build + # Pick the NEWEST mcpp.exe, not an arbitrary one: `target/` is + # restored from cache and keeps a directory per build fingerprint, + # so after a version bump the freshly built binary sits alongside + # the previous release's. `find | head -1` returned whichever the + # directory walk hit first — which is how a 0.0.106 build ran the + # 0.0.105 binary and failed 01_help_and_version. + MCPP_BIN=$(find target -name "mcpp.exe" -path "*/bin/*" -printf "%T@ %p\n" \ + | sort -rn | head -1 | cut -d" " -f2-) + test -n "$MCPP_BIN" || { echo "FAIL: no mcpp.exe in target/"; exit 1; } + MCPP_BIN=$(cd "$(dirname "$MCPP_BIN")" && pwd)/$(basename "$MCPP_BIN") + echo "Self-hosted binary: $MCPP_BIN" + "$MCPP_BIN" --version + echo "MCPP_BIN=$MCPP_BIN" >> "$GITHUB_ENV" + + - name: Package Windows release zip + id: stage + shell: bash + run: | + VERSION="${{ steps.resolve.outputs.version }}" + WRAPPER="mcpp-${VERSION}-windows-x86_64" + ZIPNAME="${WRAPPER}.zip" + + STAGING=$(mktemp -d) + mkdir -p "$STAGING/$WRAPPER/bin" "$STAGING/$WRAPPER/registry/bin" + cp "$MCPP_BIN" "$STAGING/$WRAPPER/bin/mcpp.exe" + + # Windows batch launcher + printf '@echo off\r\n"%%~dp0bin\\mcpp.exe" %%*\r\n' > "$STAGING/$WRAPPER/mcpp.bat" + cp README.md "$STAGING/$WRAPPER/" 2>/dev/null || true + cp LICENSE "$STAGING/$WRAPPER/" 2>/dev/null || true + + # Bundle xlings.exe for install consumers + if [ -f "$XLINGS_BIN_UNIX" ]; then + cp "$XLINGS_BIN_UNIX" "$STAGING/$WRAPPER/registry/bin/xlings.exe" + fi + + # Pack with 7z (available on windows-latest) + mkdir -p dist + (cd "$STAGING" && 7z a -tzip "$ZIPNAME" "$WRAPPER") + cp "$STAGING/$ZIPNAME" "dist/$ZIPNAME" + # Versionless alias + cp "dist/$ZIPNAME" "dist/mcpp-windows-x86_64.zip" + # SHA256 + (cd dist && sha256sum "$ZIPNAME" > "$ZIPNAME.sha256") + (cd dist && sha256sum "mcpp-windows-x86_64.zip" > "mcpp-windows-x86_64.zip.sha256") + + echo "zipname=$ZIPNAME" >> "$GITHUB_OUTPUT" + ls -la dist/ + + - name: Smoke-test the packaged zip + shell: bash + run: | + ZIPNAME="${{ steps.stage.outputs.zipname }}" + WRAPPER="${ZIPNAME%.zip}" + SMOKE=$(mktemp -d) + (cd "$SMOKE" && unzip -q "$GITHUB_WORKSPACE/dist/$ZIPNAME") + "$SMOKE/$WRAPPER/bin/mcpp.exe" --version + "$SMOKE/$WRAPPER/bin/mcpp.exe" --help | head -5 + test -f "$SMOKE/$WRAPPER/registry/bin/xlings.exe" + test -f "$SMOKE/$WRAPPER/mcpp.bat" + echo "Smoke-test passed" + + - name: Keep the Windows archives for the release + uses: actions/upload-artifact@v4 + with: + name: release-windows-x86_64 + if-no-files-found: error + retention-days: 7 + path: | + dist/mcpp-${{ steps.resolve.outputs.version }}-windows-x86_64.zip + dist/mcpp-${{ steps.resolve.outputs.version }}-windows-x86_64.zip.sha256 + dist/mcpp-windows-x86_64.zip + dist/mcpp-windows-x86_64.zip.sha256 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9f46dc603..19c17d588 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,9 +1,10 @@ name: release -# Self-host release: bootstrap mcpp from xlings (xim:mcpp), build the -# musl-static artefact via `mcpp pack --target x86_64-linux-musl -o ...`, -# inject xlings into the produced tarball for install.sh consumers, -# smoke-test, upload. +# The release of a tag: its four archives, promoted from the run that verified +# the tag's tree or built by release-build.yml (bootstrap mcpp from xlings, +# `mcpp pack --target x86_64-linux-musl`, xlings injected for install.sh +# consumers, smoke tests), then the release, its sealed manifest and the +# ecosystem's publication. on: push: @@ -28,7 +29,9 @@ on: # when all four have passed, so no release exists with a platform missing. The # release v2026.11.12.1 took 109 minutes, its jobs one after another: the # canaries 48, the Linux build 27 (it created the release the others waited -# for), the aarch64 build 29, the seal and the publication 5. +# for), the aarch64 build 29, the seal and the publication 5. The builds are +# release-build.yml's, which ci.yml's `full` tier runs too; a tree such a run +# verified is not built again (design 2026-11-20 §5.22): `find`, `promote`. jobs: # THE ECOSYSTEM'S OWN PROJECTS, BUILT WITH THE CANDIDATE (the 2026-09-28 # design, WS10; .github/release-canaries.toml), when the release is started @@ -39,772 +42,142 @@ jobs: if: github.event_name == 'workflow_dispatch' && inputs.canaries uses: ./.github/workflows/release-canaries.yml - build-release: - name: build (linux / x86_64) + # THE TAG'S TREE, AND THE RUN THAT VERIFIED IT (design 2026-11-20 §5.22, CI3). A `full` run of + # ci.yml that passed left `verified-`, and its `release-form` job kept the four archives + # this workflow would build (release-build.yml). When the tag's tree has that mark and the run's + # archives are still kept, the release promotes them (`promote`): the archives a run tested are + # the ones released. Otherwise it builds them (`build`), and the summary says which it did. + find: + name: the tag's tree and the run that verified it runs-on: ubuntu-24.04 permissions: contents: read - timeout-minutes: 60 + actions: read + timeout-minutes: 10 outputs: - tag: ${{ steps.stage.outputs.tag }} - version: ${{ steps.stage.outputs.version }} - env: - # mcpp resolves MCPP_HOME from the binary's location by default, - # but here we want to share toolchains with the bootstrap sandbox, - # so we pin to a known path. - MCPP_HOME: /home/runner/.mcpp + tag: ${{ steps.find.outputs.tag }} + version: ${{ steps.find.outputs.version }} + ref: ${{ steps.find.outputs.ref }} + run: ${{ steps.find.outputs.run }} steps: # fetch-depth: 0 instead of fetch-tags: true — actions/checkout@v4 # fails on push-tag triggers when both the ref'd tag and # `fetch-tags: true` are set: # "Cannot fetch both and refs/tags/vX.Y.Z to refs/tags/vX.Y.Z" - # Full-history fetch covers the resolve-tag step's needs without - # that contention. - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Resolve target tag + commit - id: resolve - # Three trigger shapes converge here: - # 1. push: refs/tags/vX.Y.Z → use that tag, build at its commit - # 2. workflow_dispatch with `tag` input set: - # - tag exists on remote → check it out (rebuild scenario) - # - tag doesn't exist → use current HEAD; gh-release - # creates the tag at that commit on upload - # 3. workflow_dispatch with no input → derive `v` from - # mcpp.toml's [package].version, build at current HEAD; - # gh-release creates the tag. + # Three trigger shapes converge here: + # 1. push: refs/tags/vX.Y.Z → that tag, at its commit + # 2. workflow_dispatch with `tag`: + # - the tag exists → its commit (a rebuild) + # - it does not → this run's commit; the release creates the tag there + # 3. workflow_dispatch without `tag` → `v` of mcpp.toml, as in 2. + - name: The tag, its commit and tree, and the run that verified the tree + id: find + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + EVENT: ${{ github.event_name }} + INPUT_TAG: ${{ inputs.tag }} run: | - if [ "${{ github.event_name }}" = "push" ]; then - TAG="${{ github.ref_name }}" - elif [ -n "${{ github.event.inputs.tag }}" ]; then - TAG="${{ github.event.inputs.tag }}" + set -euo pipefail + if [ "$EVENT" = push ]; then + TAG="$GITHUB_REF_NAME" + elif [ -n "$INPUT_TAG" ]; then + TAG="$INPUT_TAG" else VER=$(awk -F '"' '/^version[[:space:]]*=/{print $2; exit}' mcpp.toml) test -n "$VER" || { echo 'failed to read [package].version from mcpp.toml'; exit 1; } TAG="v$VER" fi - echo "tag=$TAG" >> "$GITHUB_OUTPUT" - echo "version=${TAG#v}" >> "$GITHUB_OUTPUT" - # If the tag exists on remote AND we're on workflow_dispatch, - # check it out so we rebuild that exact commit. push-tag runs - # already start at the tag commit. - if [ "${{ github.event_name }}" = "workflow_dispatch" ] \ - && git rev-parse --verify "refs/tags/$TAG" >/dev/null 2>&1; then - git checkout --detach "refs/tags/$TAG" - fi - echo "Resolved tag: $TAG (commit $(git rev-parse --short HEAD))" - - # Cache mcpp's sandbox: musl-gcc 15.1 + binutils + glibc + linux-headers - # + patchelf + ninja is ~800 MB on disk; without this every release - # rebuilds from cold install. Key on the workspace manifest so a - # toolchain change in mcpp.toml refreshes the cache. - - name: Cache mcpp sandbox - uses: actions/cache@v4 - with: - path: ~/.mcpp - key: mcpp-sandbox-${{ runner.os }}-release-${{ hashFiles('mcpp.toml', '.xlings.json') }} - restore-keys: | - mcpp-sandbox-${{ runner.os }}-release- - - # Cache xlings + xim:mcpp install. - - name: Cache xlings - uses: actions/cache@v4 - with: - path: ~/.xlings - key: xlings-${{ runner.os }}-release-xl0462-${{ hashFiles('.xlings.json') }} - restore-keys: | - xlings-${{ runner.os }}-release-xl0462- - - - name: Bootstrap mcpp via xlings - env: - XLINGS_NON_INTERACTIVE: '1' - # Pin xlings to a known-good version. The upstream install - # script always grabs `latest` (no version override), so we - # download + self-install manually to avoid broken releases. - XLINGS_VERSION: '2026.11.17.2' - run: | - # BY VERSION, NOT BY PRESENCE. A restored ~/.xlings holds the - # xlings of an earlier release, and this binary is the one the - # tarball bundles: 2026.10.10.1 shipped xlings 2026.10.8.1 on - # linux-x86_64 because one was present. - have=$("$HOME/.xlings/subos/default/bin/xlings" --version 2>/dev/null | awk '{print $2}') - if [ "$have" != "$XLINGS_VERSION" ]; then - tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" - bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ - "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ - "/tmp/${tarball}" - tar -xzf "/tmp/${tarball}" -C /tmp - "/tmp/xlings-${XLINGS_VERSION}-linux-x86_64/subos/default/bin/xlings" self install - fi - export PATH="$HOME/.xlings/subos/default/bin:$PATH" - xlings --version - test "$(xlings --version | awk '{print $2}')" = "$XLINGS_VERSION" || { - echo "::error::the xlings this release bundles is $(xlings --version), not $XLINGS_VERSION"; exit 1; } - # Pinned to .xlings.json — a bare `xlings install mcpp` resolves - # "newest in this runner's index copy" and put 0.0.105 (below the - # index floor) into this job. See .github/tools/install_pinned_mcpp.sh. - MCPP=$(bash "$GITHUB_WORKSPACE/.github/tools/install_pinned_mcpp.sh" "$GITHUB_WORKSPACE") - echo "MCPP=$MCPP" >> "$GITHUB_ENV" - echo "XLINGS_BIN=$HOME/.xlings/subos/default/bin/xlings" >> "$GITHUB_ENV" - - - name: Build + pack release artefact (musl static) - id: stage - # Build for the musl-static target, strip the produced ELF, then - # let `mcpp pack` assemble the tarball (binary + top-level wrapper - # + README + LICENSE, contents at archive root). Inject xlings - # afterwards so install.sh consumers get a single self-contained - # bundle. - run: | - TAG="${{ steps.resolve.outputs.tag }}" - VERSION="${{ steps.resolve.outputs.version }}" - TARBALL_NAME="mcpp-${VERSION}-linux-x86_64.tar.gz" - - # Build first so we can strip the ELF before pack copies it. - export MCPP_VENDORED_XLINGS="$XLINGS_BIN" - ARTIFACT=$(bash "$GITHUB_WORKSPACE/.github/tools/built_artifact.sh" "$MCPP" mcpp --target x86_64-linux-musl) - test -n "$ARTIFACT" - file "$ARTIFACT" | grep -q 'statically linked' - # NB: stripping here is pointless — `mcpp pack` below rebuilds the - # binary and overwrites it, which is why every release up to and - # including 2026.7.28.2 shipped an UNSTRIPPED bin/mcpp despite the - # strip that used to live on this line. The payload is slimmed after - # packing instead, in the inject step below. - - # Pack with the freshly-built mcpp (not the bootstrap) so any - # fixes to the pack code path are exercised in the same release - # they ship in. MCPP_HOME is forced so the new binary uses the - # pinned sandbox instead of resolving relative to its own - # location under target/. - MCPP_HOME="$MCPP_HOME" "$ARTIFACT" pack \ - --target x86_64-linux-musl \ - --mode static \ - -o "${TARBALL_NAME}" - - # Inject xlings: extract → add registry/bin/xlings to the wrapper - # dir → re-tar preserving the wrapper. Since 0.0.4 the bundled - # xlings lives at /registry/bin/xlings (= /bin/xlings). - TARBALL="target/dist/${TARBALL_NAME}" - WRAPPER="${TARBALL_NAME%.tar.gz}" - test -f "$TARBALL" - INJECT=$(mktemp -d) - tar -xzf "$TARBALL" -C "$INJECT" - mkdir -p "$INJECT/$WRAPPER/registry/bin" - cp "$XLINGS_BIN" "$INJECT/$WRAPPER/registry/bin/xlings" - chmod +x "$INJECT/$WRAPPER/registry/bin/xlings" - # Slim AFTER pack (pack rebuilds bin/mcpp) and BEFORE tar. Asserts - # the result, so a strip that silently stops working fails the - # release instead of quietly shipping a 34.8MB tarball again. - bash .github/tools/slim_linux_payload.sh "$INJECT/$WRAPPER" - (cd "$INJECT" && tar -czf "$GITHUB_WORKSPACE/${TARBALL}" "$WRAPPER") - rm -rf "$INJECT" - - # Stage final dist/ (tarball + sidecars) for upload. - mkdir -p dist - cp "$TARBALL" "dist/${TARBALL_NAME}" - (cd dist && cp "${TARBALL_NAME}" "mcpp-linux-x86_64.tar.gz") - (cd dist && sha256sum "${TARBALL_NAME}" "mcpp-linux-x86_64.tar.gz" > SHA256SUMS) - (cd dist && sha256sum "${TARBALL_NAME}" > "${TARBALL_NAME}.sha256") - (cd dist && sha256sum "mcpp-linux-x86_64.tar.gz" > "mcpp-linux-x86_64.tar.gz.sha256") - - # Top-level install.sh — fetched by `curl | bash`. - cp install.sh dist/install.sh - chmod +x dist/install.sh - - echo "tag=$TAG" >> $GITHUB_OUTPUT - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "tarball=${TARBALL_NAME}" >> $GITHUB_OUTPUT - ls -la dist/ - - - name: Smoke-test the bundled tarball - # Extract to a scratch dir and run mcpp from there with MCPP_HOME - # unset — proves the release artefact is genuinely self-contained. - run: | - VERSION="${{ steps.stage.outputs.version }}" - TARBALL_NAME="${{ steps.stage.outputs.tarball }}" - # Wrapper dir inside the tarball matches its stem (mcpp pack - # ties the two together). - WRAPPER="${TARBALL_NAME%.tar.gz}" - SMOKE=$(mktemp -d) - tar -xzf "dist/${TARBALL_NAME}" -C "$SMOKE" - ROOT="$SMOKE/$WRAPPER" - test -x "$ROOT/bin/mcpp" - test -x "$ROOT/registry/bin/xlings" - test -x "$ROOT/mcpp" - file "$ROOT/bin/mcpp" | grep -q 'statically linked' - env -u MCPP_HOME "$ROOT/bin/mcpp" --version - env -u MCPP_HOME "$ROOT/bin/mcpp" --help | head -10 - # Top-level wrapper reports the same version we're shipping. - env -u MCPP_HOME "$ROOT/mcpp" --version | grep -q "$VERSION" - # MCPP_HOME should auto-resolve to the extracted root. - out=$(env -u MCPP_HOME "$ROOT/bin/mcpp" self env) - # R4 (mcpp#832): the binary pack rebuilt with this release's own - # engine trims the paths it carries. - bash .github/tools/check_release_paths.sh "$ROOT/bin/mcpp" - echo "$out" | grep -q "MCPP_HOME *= *$ROOT" - - - name: Generate source tarball + xpkg.lua via mcpp publish - # Use the freshly-built mcpp to produce the source tarball + xpkg - # descriptor for mcpp-index. The release tarball wraps its - # contents in a `/` directory so the extract path - # is $PUB/$WRAPPER/bin/mcpp. - run: | - VERSION="${{ steps.stage.outputs.version }}" - TARBALL_NAME="${{ steps.stage.outputs.tarball }}" - WRAPPER="${TARBALL_NAME%.tar.gz}" - PUB=$(mktemp -d) - tar -xzf "dist/${TARBALL_NAME}" -C "$PUB" - MCPP_BIN="$PUB/$WRAPPER/bin/mcpp" - env -u MCPP_HOME "$MCPP_BIN" publish --dry-run --allow-dirty - test -f "target/dist/mcpp-${VERSION}.tar.gz" - test -f "target/dist/mcpp.lua" - cp "target/dist/mcpp-${VERSION}.tar.gz" dist/ - cp "target/dist/mcpp.lua" dist/ - ls -la dist/ - - - name: Extract release notes from CHANGELOG - id: notes - run: | - TAG="${{ steps.stage.outputs.tag }}" - VERSION="${{ steps.stage.outputs.version }}" - awk -v v="$VERSION" ' - /^## \[/ { - if (in_section) exit - if ($0 ~ "\\[" v "\\]") { in_section=1; next } - } - in_section { print } - ' CHANGELOG.md > dist/RELEASE_NOTES.md || true - # A RELEASE IS OUTWARD-FACING AND PERMANENT: publishing one whose - # notes silently came out empty is not a lesser failure than any - # other defect this workflow guards against, and until now it - # produced none — v2026.9.18.1 went out with its notes body reading - # literally "(no CHANGELOG entry found for 2026.9.18.1)", because - # CHANGELOG.md had no matching `## []` heading and this - # step wrote a placeholder and moved on rather than stopping. Stop - # instead: no matching section is an authoring mistake (the - # section is missing, or its version string does not match what is - # being tagged), and the fix belongs in CHANGELOG.md before the - # release is retried, not in a release whose notes nobody reads. - if [ ! -s dist/RELEASE_NOTES.md ]; then - echo "::error::CHANGELOG.md has no '## [$VERSION]' section — refusing to publish a release with empty notes. Add that section (docs/92-release.md) and re-run, rather than let this placeholder ship: (no CHANGELOG entry found for $VERSION)" - exit 1 + if git rev-parse --verify "refs/tags/$TAG" >/dev/null 2>&1; then + REF=$(git rev-list -n 1 "refs/tags/$TAG") + else + REF="$GITHUB_SHA" fi - echo "--- RELEASE_NOTES.md ---" - cat dist/RELEASE_NOTES.md - - - name: Keep the Linux archives for the release - uses: actions/upload-artifact@v4 - with: - name: release-linux-x86_64 - if-no-files-found: error - retention-days: 7 - path: | - dist/mcpp-${{ steps.stage.outputs.version }}-linux-x86_64.tar.gz - dist/mcpp-${{ steps.stage.outputs.version }}-linux-x86_64.tar.gz.sha256 - dist/mcpp-linux-x86_64.tar.gz - dist/mcpp-linux-x86_64.tar.gz.sha256 - dist/install.sh - dist/mcpp-${{ steps.stage.outputs.version }}.tar.gz - dist/mcpp.lua - dist/RELEASE_NOTES.md - - build-linux-aarch64: - name: build (linux / aarch64, cross) - runs-on: ubuntu-24.04 - permissions: - contents: read - timeout-minutes: 70 - steps: - - uses: actions/checkout@v4 - - - name: Install system deps + qemu - run: | - # The runner image carries third-party apt lists (Google Chrome - # among them) that this job does not use, and a transient - # `Hash Sum mismatch` on one of them fails the whole update -- which - # killed two cross-build jobs in setup, before a single byte was - # compiled. Dropping the lists this job has no use for is what makes - # the step's failure mean something about this job. - # - # BY CONTENT, NOT BY FILENAME. The first attempt removed - # `google-chrome.list` and the update failed on the same URL: on - # ubuntu-24.04 the runner writes deb822 `.sources` files, so the - # name was a guess and the guess was wrong. - sudo grep -rlE 'dl[.]google[.]com|packages[.]microsoft[.]com' \ - /etc/apt/sources.list.d/ 2>/dev/null | xargs -r sudo rm -f - sudo apt-get update -qq - sudo apt-get install -y curl git build-essential qemu-user-static - qemu-aarch64-static --version | head -1 - - - name: Resolve tag + version - id: resolve - run: | - VERSION=$(grep -E '^version' mcpp.toml | head -1 | sed 's/.*"\([^"]*\)".*/\1/') - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "tag=v$VERSION" >> "$GITHUB_OUTPUT" - - - name: Bootstrap mcpp via xlings - env: - XLINGS_NON_INTERACTIVE: '1' - XLINGS_VERSION: '2026.11.17.2' - run: | - tarball="xlings-${XLINGS_VERSION}-linux-x86_64.tar.gz" - bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ - "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ - "/tmp/${tarball}" - tar -xzf "/tmp/${tarball}" -C /tmp - "/tmp/xlings-${XLINGS_VERSION}-linux-x86_64/subos/default/bin/xlings" self install - echo "$HOME/.xlings/subos/current/bin" >> "$GITHUB_PATH" - echo "$HOME/.xlings/bin" >> "$GITHUB_PATH" - echo "XLINGS_BIN=$HOME/.xlings/subos/default/bin/xlings" >> "$GITHUB_ENV" - - - name: Bootstrap mcpp + refresh index (latest, GLOBAL) - run: | - xlings config --mirror GLOBAL 2>/dev/null || true - xlings update -y 2>/dev/null || xlings update 2>/dev/null || true - MCPP=$(bash "$GITHUB_WORKSPACE/.github/tools/install_pinned_mcpp.sh" "$GITHUB_WORKSPACE") - echo "MCPP=$MCPP" >> "$GITHUB_ENV" - "$MCPP" --version - - - name: Cross-build mcpp -> aarch64-linux-musl (this release's source) - run: | - export MCPP_VENDORED_XLINGS="$XLINGS_BIN" - # "$MCPP", not a bare `mcpp`: the bare form runs the PATH shim, which - # resolves to whatever version xvm has selected — so pinning the - # bootstrap would have looked correct and changed nothing here. - "$MCPP" self config --mirror GLOBAL 2>/dev/null || true - # The published bootstrap mcpp predates aarch64 cross-build support - # (the feature landed after the last release), so it resolves the - # x86_64 host musl-gcc for an aarch64 target. Two-stage instead: build - # THIS release's x86_64 mcpp first, then cross-build aarch64 with it. - FRESH=$(bash "$GITHUB_WORKSPACE/.github/tools/built_artifact.sh" "$MCPP" mcpp --target x86_64-linux-musl) - test -n "$FRESH" - BIN=$(bash "$GITHUB_WORKSPACE/.github/tools/built_artifact.sh" "$FRESH" mcpp --target aarch64-linux-musl) - test -n "$BIN" - file "$BIN" | grep -q 'ARM aarch64' - file "$BIN" | grep -q 'statically linked' - echo "MCPP_AARCH64=$BIN" >> "$GITHUB_ENV" - - - name: Package aarch64 release (+ bundle aarch64 xlings) - id: stage - run: | - VERSION="${{ steps.resolve.outputs.version }}" - TARBALL_NAME="mcpp-${VERSION}-linux-aarch64.tar.gz" - WRAPPER="mcpp-${VERSION}-linux-aarch64" - STAGING=$(mktemp -d) - mkdir -p "$STAGING/$WRAPPER/bin" - cp "$MCPP_AARCH64" "$STAGING/$WRAPPER/bin/mcpp" - # The binary is aarch64, so the host x86_64 `strip` cannot touch it — - # resolve a cross-capable one. Required, not best-effort: the old - # `|| true` here meant a missing tool silently shipped a fat tarball. - # (slim_linux_payload.sh runs after xlings is staged, below.) - STRIP=$(find "$HOME/.mcpp" -name 'aarch64-linux-musl-strip' -type f 2>/dev/null | head -1) - [ -n "$STRIP" ] || STRIP=$(command -v llvm-strip 2>/dev/null || true) - [ -n "$STRIP" ] || STRIP=$(command -v aarch64-linux-gnu-strip 2>/dev/null || true) - [ -n "$STRIP" ] || { echo "no aarch64-capable strip found"; exit 1; } - echo "aarch64 strip: $STRIP" - cp LICENSE "$STAGING/$WRAPPER/" 2>/dev/null || true - cp README.md "$STAGING/$WRAPPER/" 2>/dev/null || true - cat > "$STAGING/$WRAPPER/mcpp" << 'LAUNCHER' - #!/bin/sh - exec "$(dirname "$0")/bin/mcpp" "$@" - LAUNCHER - chmod +x "$STAGING/$WRAPPER/mcpp" - # Bundle the aarch64 xlings so install.sh consumers on aarch64 get an - # aarch64 xlings, not the x86_64 bootstrap one. The three literals - # below are pinned to the same version as XLINGS_VERSION; they are - # NOT interpolated from it, so check_version_pins.sh scans for them - # explicitly (they were absent from the old lock-step comment). - XLA="xlings-2026.11.17.2-linux-aarch64.tar.gz" - # NOT fetch_release.sh: this asset is OPTIONAL and the `if` is the - # point — an arch with no prebuilt xlings must fall through quietly, - # while the helper retries a 404 five times before giving up. The one - # flag that matters here is --retry-all-errors: `curl: (52) Empty - # reply from server` is a transport error, so plain --retry does not - # cover it. - if curl -fsSL --retry 3 --retry-delay 2 --retry-all-errors \ - --connect-timeout 20 --max-time 600 -o "/tmp/$XLA" \ - "https://github.com/openxlings/xlings/releases/download/v2026.11.17.2/$XLA"; then - tar -xzf "/tmp/$XLA" -C /tmp - XLBIN=$(find /tmp/xlings-2026.11.17.2-linux-aarch64 -path '*/bin/xlings' -type f | head -1) - if [ -n "$XLBIN" ]; then - mkdir -p "$STAGING/$WRAPPER/registry/bin" - cp "$XLBIN" "$STAGING/$WRAPPER/registry/bin/xlings" - chmod +x "$STAGING/$WRAPPER/registry/bin/xlings" + TREE=$(git rev-parse "$REF^{tree}") + RUN=$(gh api "repos/$REPO/actions/artifacts?name=verified-$TREE&per_page=5" \ + --jq '[.artifacts[] | select(.expired | not)][0].workflow_run.id // empty' 2>/dev/null || true) + WHY="" + if [ -n "$RUN" ]; then + kept=$(gh api "repos/$REPO/actions/runs/$RUN/artifacts?per_page=100" \ + --jq '[.artifacts[] | select((.expired | not) and (.name | startswith("release-")))] | length') + if [ "$kept" != 4 ]; then + WHY="run $RUN verified the tree, and keeps $kept of its four archives" + RUN="" fi + else + WHY="no run verified the tree" fi - # Slim both shipped ELFs with the cross strip resolved above, and - # assert the result (the vendored xlings was 86.9MB unstripped here). - bash .github/tools/slim_linux_payload.sh "$STAGING/$WRAPPER" "$STRIP" - mkdir -p dist - (cd "$STAGING" && tar -czf "$GITHUB_WORKSPACE/dist/${TARBALL_NAME}" "$WRAPPER") - cp "dist/${TARBALL_NAME}" "dist/mcpp-linux-aarch64.tar.gz" - (cd dist && sha256sum "${TARBALL_NAME}" > "${TARBALL_NAME}.sha256") - (cd dist && sha256sum "mcpp-linux-aarch64.tar.gz" > "mcpp-linux-aarch64.tar.gz.sha256") - echo "tarball=${TARBALL_NAME}" >> "$GITHUB_OUTPUT" - ls -la dist/ - - - name: Smoke-test the aarch64 tarball (qemu) - run: | - TARBALL_NAME="${{ steps.stage.outputs.tarball }}" - WRAPPER="${TARBALL_NAME%.tar.gz}" - SMOKE=$(mktemp -d) - tar -xzf "dist/${TARBALL_NAME}" -C "$SMOKE" - ver=$(qemu-aarch64-static "$SMOKE/$WRAPPER/bin/mcpp" --version) - echo "$ver"; echo "$ver" | grep -q 'mcpp' - - - name: Keep the aarch64 archives for the release - uses: actions/upload-artifact@v4 - with: - name: release-linux-aarch64 - if-no-files-found: error - retention-days: 7 - path: | - dist/mcpp-${{ steps.resolve.outputs.version }}-linux-aarch64.tar.gz - dist/mcpp-${{ steps.resolve.outputs.version }}-linux-aarch64.tar.gz.sha256 - dist/mcpp-linux-aarch64.tar.gz - dist/mcpp-linux-aarch64.tar.gz.sha256 + { + echo "tag=$TAG"; echo "version=${TAG#v}"; echo "ref=$REF"; echo "run=$RUN" + } >> "$GITHUB_OUTPUT" + { + echo "## $TAG" + echo "Commit \`$REF\`, tree \`$TREE\`." + if [ -n "$RUN" ]; then + echo "Promoted: the archives of run [$RUN](https://github.com/$REPO/actions/runs/$RUN), which verified the tree." + else + echo "Built: $WHY." + fi + } >> "$GITHUB_STEP_SUMMARY" + echo "$TAG at $REF (tree $TREE): ${RUN:+promoted from run $RUN}${WHY:+built, $WHY}" - build-macos: - name: build (macOS / ARM64) - runs-on: macos-15 + build: + needs: find + if: ${{ needs.find.outputs.run == '' }} permissions: contents: read - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Resolve tag - id: resolve - run: | - if [ "${{ github.event_name }}" = "push" ]; then - TAG="${{ github.ref_name }}" - elif [ -n "${{ github.event.inputs.tag }}" ]; then - TAG="${{ github.event.inputs.tag }}" - else - VER=$(awk -F '"' '/^version[[:space:]]*=/{print $2; exit}' mcpp.toml) - TAG="v$VER" - fi - echo "tag=$TAG" >> "$GITHUB_OUTPUT" - echo "version=${TAG#v}" >> "$GITHUB_OUTPUT" - if [ "${{ github.event_name }}" = "workflow_dispatch" ] \ - && git rev-parse --verify "refs/tags/$TAG" >/dev/null 2>&1; then - git checkout --detach "refs/tags/$TAG" - fi - - - name: Cache xlings - uses: actions/cache@v4 - with: - path: ~/.xlings - key: xlings-macos15-release-xl0462-${{ hashFiles('.xlings.json') }} - restore-keys: | - xlings-macos15-release-xl0462- - - - name: Bootstrap mcpp via xlings - env: - XLINGS_NON_INTERACTIVE: '1' - XLINGS_VERSION: '2026.11.17.2' - run: | - # By version, not by presence (see the Linux job). - have=$("$HOME/.xlings/subos/default/bin/xlings" --version 2>/dev/null | awk '{print $2}') - if [ "$have" != "$XLINGS_VERSION" ]; then - WORK=$(mktemp -d) - tarball="xlings-${XLINGS_VERSION}-macosx-arm64.tar.gz" - bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ - "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${tarball}" \ - "${WORK}/${tarball}" - tar -xzf "${WORK}/${tarball}" -C "${WORK}" - "${WORK}/xlings-${XLINGS_VERSION}-macosx-arm64/subos/default/bin/xlings" self install - fi - export PATH="$HOME/.xlings/subos/default/bin:$PATH" - xlings --version - test "$(xlings --version | awk '{print $2}')" = "$XLINGS_VERSION" || { - echo "::error::the xlings this release bundles is $(xlings --version), not $XLINGS_VERSION"; exit 1; } - # Pinned to .xlings.json — a bare `xlings install mcpp` resolves - # "newest in this runner's index copy" and put 0.0.105 (below the - # index floor) into this job. See .github/tools/install_pinned_mcpp.sh. - MCPP=$(bash "$GITHUB_WORKSPACE/.github/tools/install_pinned_mcpp.sh" "$GITHUB_WORKSPACE") - echo "MCPP=$MCPP" >> "$GITHUB_ENV" - echo "XLINGS_BIN=$HOME/.xlings/subos/default/bin/xlings" >> "$GITHUB_ENV" - - - name: Build mcpp from source (two-stage self-host) - env: - # macOS min-version support: target macOS 14 so the release runs - # on 14.0+ instead of only the runner's OS (the official LLVM - # static libc++ archives are built for macOS 14 — going lower - # needs a custom libc++ build, tracked as follow-up). Needs - # static LLVM libc++ — the system libc++ on older macOS lacks - # LLVM-20-era C++23 symbols (std::print's __is_posix_terminal - # etc.; minos-14 + dynamic libc++ dies at launch on macos-14 CI). - # See xlings .agents/docs/2026-06-05-macos-min-version-support.md. - MACOSX_DEPLOYMENT_TARGET: '14.0' - run: | - export PATH="$HOME/.xlings/subos/default/bin:$PATH" - export MCPP_VENDORED_XLINGS="$XLINGS_BIN" - - # Stage 1: the bootstrap mcpp builds this release's source. The - # bootstrap's macOS link path predates the staticStdlib - # implementation (hardcoded -lc++), so stage 1 links the system - # libc++ — fine, it only needs to RUN on this runner. - STAGE1=$(bash "$GITHUB_WORKSPACE/.github/tools/built_artifact.sh" "$MCPP" mcpp) - # --no-cache removes target/, including the stage-1 compiler. Ninja - # invokes that compiler again for dependency scanning, so keep the - # bootstrap outside the tree the second stage is about to clear. - STAGE1_COPY=$(mktemp -d "$RUNNER_TEMP/mcpp-stage1.XXXXXX")/mcpp - cp "$STAGE1" "$STAGE1_COPY" - STAGE1="$STAGE1_COPY" - "$STAGE1" --version - - # Stage 2: this release's mcpp rebuilds itself — flags.cppm's - # native staticStdlib link produces the static minos-14 binary. - # Stage 2 lands in a NEW fingerprint directory (its fingerprint - # includes the deployment target; the bootstrap's did not): the - # build names its own product. - MCPP_BIN=$(bash "$GITHUB_WORKSPACE/.github/tools/built_artifact.sh" "$STAGE1" mcpp --no-cache) - test -x "$MCPP_BIN" - file "$MCPP_BIN" - otool -L "$MCPP_BIN" - echo "=== LC_BUILD_VERSION (must be minos 14.0) ===" - otool -l "$MCPP_BIN" | grep -A4 LC_BUILD_VERSION | head -6 - otool -l "$MCPP_BIN" | grep -A4 LC_BUILD_VERSION | grep -q "minos 14.0" \ - || { echo "FAIL: expected minos 14.0"; exit 1; } - if otool -L "$MCPP_BIN" | grep -q "libc++"; then - echo "FAIL: still linked against system libc++"; exit 1 - fi - "$MCPP_BIN" --version - echo "MCPP_BIN=$MCPP_BIN" >> "$GITHUB_ENV" - - - name: Package macOS release - id: stage - run: | - VERSION="${{ steps.resolve.outputs.version }}" - TARBALL_NAME="mcpp-${VERSION}-macosx-arm64.tar.gz" - WRAPPER="mcpp-${VERSION}-macosx-arm64" - - # Create release layout - STAGING=$(mktemp -d) - mkdir -p "$STAGING/$WRAPPER/bin" - cp "$MCPP_BIN" "$STAGING/$WRAPPER/bin/mcpp" - # Strip (Mach-O) - strip "$STAGING/$WRAPPER/bin/mcpp" 2>/dev/null || true - # Copy metadata - cp LICENSE "$STAGING/$WRAPPER/" 2>/dev/null || true - cp README.md "$STAGING/$WRAPPER/" 2>/dev/null || true - - # Shell launcher (same as Linux) - cat > "$STAGING/$WRAPPER/mcpp" << 'LAUNCHER' - #!/bin/sh - exec "$(dirname "$0")/bin/mcpp" "$@" - LAUNCHER - chmod +x "$STAGING/$WRAPPER/mcpp" - - # Bundle xlings for install.sh consumers - XLINGS_BIN="$HOME/.xlings/subos/default/bin/xlings" - if [ -x "$XLINGS_BIN" ]; then - mkdir -p "$STAGING/$WRAPPER/registry/bin" - cp "$XLINGS_BIN" "$STAGING/$WRAPPER/registry/bin/xlings" - chmod +x "$STAGING/$WRAPPER/registry/bin/xlings" - fi - - # Create tarball - mkdir -p dist - (cd "$STAGING" && tar -czf "$GITHUB_WORKSPACE/dist/${TARBALL_NAME}" "$WRAPPER") - # Versionless alias - cp "dist/${TARBALL_NAME}" "dist/mcpp-macosx-arm64.tar.gz" - # SHA256 - (cd dist && shasum -a 256 "${TARBALL_NAME}" > "${TARBALL_NAME}.sha256") - (cd dist && shasum -a 256 "mcpp-macosx-arm64.tar.gz" > "mcpp-macosx-arm64.tar.gz.sha256") - - echo "tarball=${TARBALL_NAME}" >> "$GITHUB_OUTPUT" - ls -la dist/ - - - name: Smoke-test the tarball - run: | - VERSION="${{ steps.resolve.outputs.version }}" - TARBALL_NAME="${{ steps.stage.outputs.tarball }}" - WRAPPER="${TARBALL_NAME%.tar.gz}" - SMOKE=$(mktemp -d) - tar -xzf "dist/${TARBALL_NAME}" -C "$SMOKE" - "$SMOKE/$WRAPPER/bin/mcpp" --version - "$SMOKE/$WRAPPER/mcpp" --version | grep -q "$VERSION" - # R4 (mcpp#832): stage 2 is this release's own engine, and it trims - # the paths the binary carries. - bash .github/tools/check_release_paths.sh "$SMOKE/$WRAPPER/bin/mcpp" - - - name: Keep the macOS archives for the release - uses: actions/upload-artifact@v4 - with: - name: release-macosx-arm64 - if-no-files-found: error - retention-days: 7 - path: | - dist/mcpp-${{ steps.resolve.outputs.version }}-macosx-arm64.tar.gz - dist/mcpp-${{ steps.resolve.outputs.version }}-macosx-arm64.tar.gz.sha256 - dist/mcpp-macosx-arm64.tar.gz - dist/mcpp-macosx-arm64.tar.gz.sha256 - - build-windows: - name: build (Windows / x86_64) - runs-on: windows-latest + uses: ./.github/workflows/release-build.yml + with: + tag: ${{ needs.find.outputs.tag }} + ref: ${{ needs.find.outputs.ref }} + + # The verified run's four archives, checked against their checksums and their version, kept as + # this run's for `create-release`. + promote: + name: promote the verified run's archives + needs: find + if: ${{ needs.find.outputs.run != '' }} + runs-on: ubuntu-24.04 permissions: contents: read - timeout-minutes: 45 - env: - MCPP_HOME: C:\Users\runneradmin\.mcpp + actions: read + timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Resolve tag - id: resolve - shell: bash - run: | - if [ "${{ github.event_name }}" = "push" ]; then - TAG="${{ github.ref_name }}" - elif [ -n "${{ github.event.inputs.tag }}" ]; then - TAG="${{ github.event.inputs.tag }}" - else - VER=$(awk -F '"' '/^version[[:space:]]*=/{print $2; exit}' mcpp.toml) - TAG="v$VER" - fi - echo "tag=$TAG" >> "$GITHUB_OUTPUT" - echo "version=${TAG#v}" >> "$GITHUB_OUTPUT" - if [ "${{ github.event_name }}" = "workflow_dispatch" ] \ - && git rev-parse --verify "refs/tags/$TAG" >/dev/null 2>&1; then - git checkout --detach "refs/tags/$TAG" - fi - - - name: Cache mcpp sandbox - uses: actions/cache@v4 - with: - path: ~\.mcpp - key: mcpp-sandbox-${{ runner.os }}-release-${{ hashFiles('mcpp.toml', '.xlings.json') }} - restore-keys: | - mcpp-sandbox-${{ runner.os }}-release- - - - name: Cache xlings - uses: actions/cache@v4 - with: - path: ~\.xlings - key: xlings-${{ runner.os }}-release-xl0462-${{ hashFiles('.xlings.json') }} - restore-keys: | - xlings-${{ runner.os }}-release-xl0462- - - - name: Bootstrap mcpp via xlings - shell: bash + - name: The archives, their checksums and their version env: - XLINGS_NON_INTERACTIVE: '1' - XLINGS_VERSION: '2026.11.17.2' - run: | - # Captured before the `cd` below, in POSIX form: this step never - # returns to the workspace, and GITHUB_WORKSPACE is a backslash - # Windows path that git-bash tools mangle. - REPO_DIR="$(pwd)" - WORK=$(mktemp -d) - zipfile="xlings-${XLINGS_VERSION}-windows-x86_64.zip" - bash "$GITHUB_WORKSPACE/.github/tools/fetch_release.sh" \ - "https://github.com/openxlings/xlings/releases/download/v${XLINGS_VERSION}/${zipfile}" \ - "${WORK}/${zipfile}" - cd "${WORK}" - unzip -q "${zipfile}" - "$WORK/xlings-${XLINGS_VERSION}-windows-x86_64/subos/default/bin/xlings.exe" self install - export PATH="$USERPROFILE/.xlings/subos/default/bin:$PATH" - echo "$USERPROFILE/.xlings/subos/default/bin" >> "$GITHUB_PATH" - xlings.exe --version - test "$(xlings.exe --version | awk '{print $2}' | tr -d '\r')" = "$XLINGS_VERSION" || { - echo "::error::the xlings this release bundles is not $XLINGS_VERSION"; exit 1; } - # Pinned + version-scoped lookup. The old `find | head -1` returned - # whichever version the directory walk reached first. - MCPP=$(bash "$REPO_DIR/.github/tools/install_pinned_mcpp.sh" "$REPO_DIR") - echo "MCPP=$MCPP" >> "$GITHUB_ENV" - XLINGS_BIN=$(cygpath -w "$USERPROFILE/.xlings/subos/default/bin/xlings.exe") - echo "XLINGS_BIN=$XLINGS_BIN" >> "$GITHUB_ENV" - echo "XLINGS_BIN_UNIX=$USERPROFILE/.xlings/subos/default/bin/xlings.exe" >> "$GITHUB_ENV" - echo "XLINGS_XPKGS=$USERPROFILE/.xlings/data/xpkgs" >> "$GITHUB_ENV" - - - name: Build mcpp from source (self-host) - shell: bash - run: | - export MCPP_VENDORED_XLINGS="$XLINGS_BIN" - - "$MCPP" build - # Pick the NEWEST mcpp.exe, not an arbitrary one: `target/` is - # restored from cache and keeps a directory per build fingerprint, - # so after a version bump the freshly built binary sits alongside - # the previous release's. `find | head -1` returned whichever the - # directory walk hit first — which is how a 0.0.106 build ran the - # 0.0.105 binary and failed 01_help_and_version. - MCPP_BIN=$(find target -name "mcpp.exe" -path "*/bin/*" -printf "%T@ %p\n" \ - | sort -rn | head -1 | cut -d" " -f2-) - test -n "$MCPP_BIN" || { echo "FAIL: no mcpp.exe in target/"; exit 1; } - MCPP_BIN=$(cd "$(dirname "$MCPP_BIN")" && pwd)/$(basename "$MCPP_BIN") - echo "Self-hosted binary: $MCPP_BIN" - "$MCPP_BIN" --version - echo "MCPP_BIN=$MCPP_BIN" >> "$GITHUB_ENV" - - - name: Package Windows release zip - id: stage - shell: bash - run: | - VERSION="${{ steps.resolve.outputs.version }}" - WRAPPER="mcpp-${VERSION}-windows-x86_64" - ZIPNAME="${WRAPPER}.zip" - - STAGING=$(mktemp -d) - mkdir -p "$STAGING/$WRAPPER/bin" "$STAGING/$WRAPPER/registry/bin" - cp "$MCPP_BIN" "$STAGING/$WRAPPER/bin/mcpp.exe" - - # Windows batch launcher - printf '@echo off\r\n"%%~dp0bin\\mcpp.exe" %%*\r\n' > "$STAGING/$WRAPPER/mcpp.bat" - cp README.md "$STAGING/$WRAPPER/" 2>/dev/null || true - cp LICENSE "$STAGING/$WRAPPER/" 2>/dev/null || true - - # Bundle xlings.exe for install consumers - if [ -f "$XLINGS_BIN_UNIX" ]; then - cp "$XLINGS_BIN_UNIX" "$STAGING/$WRAPPER/registry/bin/xlings.exe" - fi - - # Pack with 7z (available on windows-latest) - mkdir -p dist - (cd "$STAGING" && 7z a -tzip "$ZIPNAME" "$WRAPPER") - cp "$STAGING/$ZIPNAME" "dist/$ZIPNAME" - # Versionless alias - cp "dist/$ZIPNAME" "dist/mcpp-windows-x86_64.zip" - # SHA256 - (cd dist && sha256sum "$ZIPNAME" > "$ZIPNAME.sha256") - (cd dist && sha256sum "mcpp-windows-x86_64.zip" > "mcpp-windows-x86_64.zip.sha256") - - echo "zipname=$ZIPNAME" >> "$GITHUB_OUTPUT" - ls -la dist/ - - - name: Smoke-test the packaged zip - shell: bash + GH_TOKEN: ${{ github.token }} + RUN: ${{ needs.find.outputs.run }} + VERSION: ${{ needs.find.outputs.version }} run: | - ZIPNAME="${{ steps.stage.outputs.zipname }}" - WRAPPER="${ZIPNAME%.zip}" - SMOKE=$(mktemp -d) - (cd "$SMOKE" && unzip -q "$GITHUB_WORKSPACE/dist/$ZIPNAME") - "$SMOKE/$WRAPPER/bin/mcpp.exe" --version - "$SMOKE/$WRAPPER/bin/mcpp.exe" --help | head -5 - test -f "$SMOKE/$WRAPPER/registry/bin/xlings.exe" - test -f "$SMOKE/$WRAPPER/mcpp.bat" - echo "Smoke-test passed" + set -euo pipefail + gh run download "$RUN" -R "$GITHUB_REPOSITORY" -p 'release-*' -D promoted + mkdir dist + for dir in promoted/release-*/; do cp "$dir"* dist/; done + cd dist + for platform in linux-x86_64 linux-aarch64 macosx-arm64 windows-x86_64; do + ls "mcpp-$VERSION-$platform".tar.gz "mcpp-$VERSION-$platform".zip 2>/dev/null | grep -q . || { + echo "::error::run $RUN holds no archive of $VERSION for $platform"; ls -la; exit 1; } + done + for sum in *.sha256; do sha256sum -c "$sum"; done + test -s RELEASE_NOTES.md && test -f install.sh && test -f mcpp.lua && test -f "mcpp-$VERSION.tar.gz" + root=$(mktemp -d) + tar -xzf "mcpp-$VERSION-linux-x86_64.tar.gz" -C "$root" + ver=$(env -u MCPP_HOME "$root/mcpp-$VERSION-linux-x86_64/bin/mcpp" --version) + echo "$ver" + grep -qF "$VERSION" <<< "$ver" || { echo "::error::the promoted program reports $ver, not $VERSION"; exit 1; } + ls -la - - name: Keep the Windows archives for the release + - name: Keep the archives for the release uses: actions/upload-artifact@v4 with: - name: release-windows-x86_64 + name: release-promoted if-no-files-found: error retention-days: 7 - path: | - dist/mcpp-${{ steps.resolve.outputs.version }}-windows-x86_64.zip - dist/mcpp-${{ steps.resolve.outputs.version }}-windows-x86_64.zip.sha256 - dist/mcpp-windows-x86_64.zip - dist/mcpp-windows-x86_64.zip.sha256 + path: dist/ # Seal the complete, non-draft release inventory only after every platform # uploader has finished. The manifest is desired state for downstream @@ -815,14 +188,14 @@ jobs: # release, the four builds and the source. create-release: name: create the release - needs: [canaries, build-release, build-linux-aarch64, build-macos, build-windows] + needs: [canaries, find, build, promote] if: ${{ !cancelled() && !failure() }} runs-on: ubuntu-24.04 permissions: contents: write # required to create releases + push tags timeout-minutes: 15 steps: - - name: The archives of the four builds + - name: The archives of the four builds, built or promoted uses: actions/download-artifact@v4 with: pattern: release-* @@ -840,9 +213,9 @@ jobs: - name: Create GitHub Release uses: softprops/action-gh-release@v2 with: - tag_name: ${{ needs.build-release.outputs.tag }} - name: ${{ needs.build-release.outputs.tag }} - target_commitish: ${{ github.sha }} + tag_name: ${{ needs.find.outputs.tag }} + name: ${{ needs.find.outputs.tag }} + target_commitish: ${{ needs.find.outputs.ref }} body_path: dist/RELEASE_NOTES.md draft: false prerelease: false diff --git a/docs/92-release.md b/docs/92-release.md index 911424042..14e60dd6b 100644 --- a/docs/92-release.md +++ b/docs/92-release.md @@ -48,7 +48,9 @@ install a version that did not exist yet. tag from `mcpp.toml`) does all of this: ``` -build ×4 at once (linux x86_64 / linux aarch64 / macOS ARM64 / Windows x64) +find: the tag's commit and tree, and the ci.yml run that verified the tree + → promote that run's four archives (checksums and version checked) + or, without such a run, build ×4 at once (linux x86_64 / linux aarch64 / macOS ARM64 / Windows x64) [canaries beside them, when started by hand with `canaries`] → create-release: GitHub Release v with every archive + .sha256 sidecars → recompute every payload hash and publish immutable mcpp-release.json @@ -69,6 +71,20 @@ merges the index PR after that (§3): GITCODE_TOKEN=... MIRROR_HOSTS=gtc bash .github/tools/mirror_res.sh mcpp # gtc on PATH ``` +**Promoted or built (@NEXT@+).** The four builds are +`release-build.yml`, which `ci.yml` also runs in its `full` tier; there its +Linux unit and C++ e2e shards test the program of the Linux archive. A `full` +run that passes leaves the artifact `verified-`. `find` reads the tag's +tree: when a run verified it and still keeps its four `release-*` archives +(seven days), `promote` takes them, checks each `.sha256` sidecar and the +version the Linux program reports, and the release ships the archives that run +tested. Otherwise the four builds run. The run's summary states which it did. + +A push to main whose tree a run verified runs no test: its `register` job names +that run as main's (`.github/tools/main_run.sh`). The release pull request +therefore passes a `ci:full` run before it is merged, and nothing moves main +between that run and the merge; main's run is not cancelled. + The four builds start together and keep their archives as artifacts; `create-release` creates the release with all of them once every build has passed, so no release exists with a platform missing (2026.11.13.1+). The @@ -319,9 +335,11 @@ would again let the index guard and the installed version drift apart. [ ] version bumped in mcpp.toml + fingerprint.cppm (one commit) [ ] CHANGELOG entry, in English (it is the release notes; commits and pull requests are English too) [ ] `bash .github/tools/check_version_pins.sh` passes (verifies `mcpp.toml` = `MCPP_VERSION`, and `.xlings.json` is not newer) -[ ] merge to main, CI green +[ ] the release pull request passes a `ci:full` run, its last commit included (it leaves `verified-`) +[ ] merge to main with no later change; main's run registers the verified run [ ] gh workflow run release.yml --ref main -[ ] release.yml green (4 builds + immutable manifest + publish-ecosystem) +[ ] release.yml green; its summary says "Promoted" (else it built, and says why) +[ ] release.yml green (4 archives + immutable manifest + publish-ecosystem) [ ] downloaded mcpp-release.json regenerates byte-identically from public assets [ ] GitCode mirrored (release started with `gitcode`, or MIRROR_HOSTS=gtc locally) [ ] mirrors serve all four platforms on BOTH hosts, sha256 recomputed diff --git a/docs/zh/92-release.md b/docs/zh/92-release.md index 8457a1ec7..d99f6153a 100644 --- a/docs/zh/92-release.md +++ b/docs/zh/92-release.md @@ -45,7 +45,9 @@ POSIX `sh`/dash 解析不了——`sh check_version_pins.sh` 会在第 95 行附 推导 tag)会做完这些: ``` -四平台同时构建(linux x86_64 / linux aarch64 / macOS ARM64 / Windows x64) +find:tag 的提交与树,以及验证了这棵树的 ci.yml 运行 + → 提升那次运行的四个归档(核对校验和与版本) + 或者,没有这样的运行时,四平台同时构建(linux x86_64 / linux aarch64 / macOS ARM64 / Windows x64) [手动发布且带 `canaries` 时,canaries 与之并行] → create-release:GitHub Release v,含全部归档与 .sha256 边车文件 → 重新计算每个载荷的哈希并发布不可变 mcpp-release.json @@ -64,6 +66,16 @@ PR(§3): GITCODE_TOKEN=... MIRROR_HOSTS=gtc bash .github/tools/mirror_res.sh mcpp # PATH 上需有 gtc ``` +**提升或构建(2026.11.21.1+)。** 四个构建是 `release-build.yml`,`ci.yml` 的 `full` 档也运行它;在那里 +Linux 的单元与 C++ e2e 分片测试 Linux 归档中的程序。通过的 `full` 运行留下 artifact `verified-`。 +`find` 读取 tag 的树:有运行验证过它、且那次运行仍保留四个 `release-*` 归档(七天)时,`promote` 取用它们, +核对每个 `.sha256` 边车文件与 Linux 程序报告的版本,release 发出的就是那次运行测试过的归档。否则运行四个 +构建。运行的摘要写明是哪一种。 + +树已被验证的 main 推送不运行测试:它的 `register` 作业把那次运行登记为 main 的 +(`.github/tools/main_run.sh`)。因此发布的拉取请求在合入前通过一次 `ci:full` 运行,并且从那次运行到合入之间 +main 不移动;main 的运行不取消。 + 四个构建同时开始,各自把归档留作 artifact;`create-release` 在每个构建都通过后一次创建带全部归档的 release,因此不存在缺少某个平台的 release(2026.11.13.1+)。生态 canaries(xlings、mcppls; `.github/release-canaries.toml`)只在手动发布且带 `canaries` 时用候选版本构建 @@ -290,9 +302,11 @@ curl -fsSL https://github.com/xlings-res/xim-index/releases/download/latest/xim- [ ] version bumped in mcpp.toml + fingerprint.cppm (one commit) [ ] CHANGELOG 条目,用英文书写(它就是 release notes;commit 与 PR 同样用英文) [ ] `bash .github/tools/check_version_pins.sh` passes (verifies `mcpp.toml` = `MCPP_VERSION`, and `.xlings.json` is not newer) -[ ] merge to main, CI green +[ ] 发布的拉取请求通过一次 `ci:full` 运行,含其最后一个提交(留下 `verified-`) +[ ] 合入 main,其后没有改动;main 的运行登记被验证的运行 [ ] gh workflow run release.yml --ref main -[ ] release.yml green (4 builds + immutable manifest + publish-ecosystem) +[ ] release.yml green;摘要写 "Promoted"(否则是构建,并写明原因) +[ ] release.yml green (4 archives + immutable manifest + publish-ecosystem) [ ] downloaded mcpp-release.json regenerates byte-identically from public assets [ ] GitCode mirrored (release started with `gitcode`, or MIRROR_HOSTS=gtc locally) [ ] mirrors serve all four platforms on BOTH hosts, sha256 recomputed diff --git a/tests/e2e/mtest.toml b/tests/e2e/mtest.toml index dbbae4b7f..f3d2e3ad5 100644 --- a/tests/e2e/mtest.toml +++ b/tests/e2e/mtest.toml @@ -29,3 +29,17 @@ hosts = ["linux", "macos", "windows"] shard = "12m" max_shards = 8 unknown = "30s" + +# THE PROCESSOR TIME A CASE MAY TAKE (design 2026-11-20 §5.17, mtest 0.11.0): `mtest gate` fails a +# case whose `cpu_ms` passes it, unless a waiver names it with a reason; a waiver that excuses +# nothing fails the gate too, so the list only shrinks. Compared within one class of machine. +[budget] +case_cpu = "10m" + +# THE SURFACE'S LEDGER (design 2026-11-20 §5.16): every item `mcpp emit surface` lists is used by a +# passing case (its `exercise` events, which the kit writes from MCPP_SURFACE_RECORD), or a waiver +# names it with a reason. A waiver of an item a case uses fails the gate. The items are the golden +# surface, which the case `golden/1124` holds equal to what the binary under test emits. +[ledger.surface] +file = "golden/surface.txt" +evidence = "exercised" From 16cde79304be5fd8f4bad2b09c115dfa0ddf728b Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 06:17:19 +0900 Subject: [PATCH 24/38] mtest 0.11.1 throughout, and CI's engine with it (a command an e2e case runs holds to what its case has left) --- .github/versions.env | 4 +- CHANGELOG.md | 61 +++++++++++++++++++++++++++++++ bench/mcpp.toml | 2 +- docs/05-dependencies.md | 4 +- docs/zh/05-dependencies.md | 4 +- mcpp.toml | 2 +- modules/build-record/mcpp.toml | 2 +- modules/buildmcpp/mcpp.toml | 2 +- modules/buildopt/mcpp.toml | 2 +- modules/graph/mcpp.toml | 2 +- modules/manifest/mcpp.toml | 2 +- modules/surface/mcpp.toml | 2 +- modules/toolchain-model/mcpp.toml | 2 +- modules/unit-cache/mcpp.toml | 2 +- modules/versioning/mcpp.toml | 2 +- src/build/test_protocol.cppm | 2 +- tests/canary/cxx/mcpp.toml | 2 +- tests/support/mcpp.toml | 2 +- 18 files changed, 81 insertions(+), 20 deletions(-) diff --git a/.github/versions.env b/.github/versions.env index c571ac8c9..ce084f55f 100644 --- a/.github/versions.env +++ b/.github/versions.env @@ -1,5 +1,5 @@ # The engines CI pins, read by the composite actions (.github/actions/mtest). # mtest: an xlings package (xim:mtest) whose sha256 the index states; the release # bundle's sha256 checks the fallback download. -MTEST_VERSION=0.10.0 -MTEST_SHA256=c7187e6211ea15cff23f622c449247508f6f661f0d3b45dcd031f91104b8f499 +MTEST_VERSION=0.11.1 +MTEST_SHA256=daa17456a8bade983395277d084160ffc7ad0c680a7a71a41b82ee59ba617670 diff --git a/CHANGELOG.md b/CHANGELOG.md index bccb78669..cebbfe8dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,67 @@ ## [Unreleased] +### Added + +- **The home's unit cache** (mcpp#927, docs/26). Before it starts a compiler, the compiler + executor looks up the compile by its command, its compiler and its source's content; when the + files an earlier compile of that key read still hold their contents, its object and BMI are + placed and no compiler starts. A BMI is named by the key of the compile that made it. A fresh + checkout of mcpp on a filled cache built in 4.1 s instead of 164 s, its 288 compiles placed. + The split schedule places its units from the cache too. cl.exe's compiles are not cached. +- **A cacheable package's build program runs in the home** (mcpp#986, mcpp#927, SPEC-004 §9 + item 18). A package whose source is named -- an index's version, a git commit, a path + dependency's content digest -- runs its build program into + `build-cache/v1/out/@//out`, keyed by its configuration, and a second + project takes the outputs. A path enters a cache key by its role (``, ``, + ``), never as written. +- **`[profile.] dependency_debug`** (mcpp#927, SPEC-004 §9 item 20): the debug information + of the dependencies from the package store and git, `true`, `false` or `"lines"`. Omitted, + `dev` and `debug` take `"lines"` while their `debug` is on. A test program whose only + dependency is mtest fell from 43.7 MB to 8.3 MB and its build from 15.9 s to 7.2 s. +- **`[build] memory_threshold`** (mcpp#802, SPEC-004 §9 item 19): the available memory below + which the compiler executor waits for a running compiler to end before it starts another; the + engine's value is 1 GiB, `MCPP_MEMORY_THRESHOLD` overrides it, and after a minute below it the + compiler starts and standard error says so. +- **One budget of slots for the machine** (mcpp#976, SPEC-017 O11). `mcpp test` states a + directory of one slot file per job (`MTEST_BUDGET`); a compiler a test starts through mcpp + holds a slot of it, so a test that builds a project takes from the run's jobs. A run inside + another runner's budget uses that one. +- **The record of a build and of a run** (mcpp#927, SPEC-017 B4, P5.6, P12.4). + `MCPP_BUILD_RECORD=` appends a line per edge a build ran: its package, its time, the + compiler's processor time and largest resident set, and whether the unit cache placed it. + `mcpp test` keeps a run's builds in `builds.ndjson`; a run names its machine's class, its + processors and memory, and a case its processor time, its steps, the surface items it used and + the limit it ran within. Estimates prefer the runs of the same class of machine. An edge that + runs a minute is named on a `Running` line; `--verbose` names the five slowest. +- **Run fixtures** (mcpp#293, SPEC-017 O6, P11). A program declares a fixture the run shares and + its cases name it in `uses`; `mcpp test` starts it once, with `MTEST_SERVE`, before the first + program that uses it, gives its endpoint as `MTEST_FIXTURE_`, and ends it after the + last. A fixture that does not become ready is the machine's failure: its users record `infra`. + `fixtures.ndjson` states each fixture's start, readiness and end. +- **A header selects the tests that read it.** `mcpp test --changed`/`--since` takes the units + that read a changed header from the build directory's records (ninja's deps log, the scans' + depfiles) and runs the tests built from them; while a unit of the project has no record, a + header affects every test. +- **`mcpp emit surface`** writes mcpp's surface, one ` ` per line: the commands and + flags of the command line's definition, the manifest's keys and the refusals' reason tokens. + `MCPP_SURFACE_RECORD=` makes each process of an invocation append the items it used. +- `wire` features `test-steps`, `test-budget`, `test-jitter` and `test-fixtures` (docs/50). + +### Changed + +- **A package's instance is a function of its configuration, not of its units** (mcpp#986): a + test build after `mcpp build` no longer compiles the package again, and each refusal of the + cache names its reason. +- **A case's time is the one its scopes state**: a case selected with `--case` runs within the + `timeout` its listed case states, not the run's. +- mcpplibs.mtest 0.11.1 and mcpplibs.os 0.4.7 throughout, the new project's skeleton included. +- **CI verifies a tree once** (design 2026-11-20 §5.22). A pull request's push runs the `pr` tier; + a `ci:full` run that passes leaves `verified-`, a push to main of that tree registers + the run instead of testing again, and `release.yml` promotes the four archives that run built + and tested (`release-build.yml`) instead of building them. The release pull request passes + `ci:full` before it is merged, and main's run is not cancelled (docs/92). + ### Fixed - A program's resource script is compiled with its package's `include_dirs` expanded, as the diff --git a/bench/mcpp.toml b/bench/mcpp.toml index 10dbaa374..2933612f5 100644 --- a/bench/mcpp.toml +++ b/bench/mcpp.toml @@ -21,4 +21,4 @@ windows = "llvm@23.1.3" # small fixture. `mcpp build` ignores [dev-dependencies], so the harness itself # still takes nothing beyond `import std;`. [dev-dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" diff --git a/docs/05-dependencies.md b/docs/05-dependencies.md index dca3fe935..6523b159e 100644 --- a/docs/05-dependencies.md +++ b/docs/05-dependencies.md @@ -67,8 +67,8 @@ not deprecated: ```toml [dev-dependencies] -mcpplibs.mtest = "0.11.0" -"mcpplibs.mtest.gtest" = "0.11.0" +mcpplibs.mtest = "0.11.1" +"mcpplibs.mtest.gtest" = "0.11.1" ``` ```toml diff --git a/docs/zh/05-dependencies.md b/docs/zh/05-dependencies.md index 694b90cfb..7ac73de21 100644 --- a/docs/zh/05-dependencies.md +++ b/docs/zh/05-dependencies.md @@ -62,8 +62,8 @@ mcpp 把它读作这些点所拼写的选择器,警告该拼法已弃用(自 ```toml [dev-dependencies] -mcpplibs.mtest = "0.11.0" -"mcpplibs.mtest.gtest" = "0.11.0" +mcpplibs.mtest = "0.11.1" +"mcpplibs.mtest.gtest" = "0.11.1" ``` ```toml diff --git a/mcpp.toml b/mcpp.toml index f00310198..4dcd7c049 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -186,7 +186,7 @@ members = [ # `mcpp build` ignores [dev-dependencies]; only `mcpp test` resolves them. [dev-dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" # The support of the end-to-end cases (tests/support, design 2026-11-12 §5.9). mcpp.test-support = { path = "tests/support" } diff --git a/modules/build-record/mcpp.toml b/modules/build-record/mcpp.toml index 27754ce01..e4ad55c13 100644 --- a/modules/build-record/mcpp.toml +++ b/modules/build-record/mcpp.toml @@ -17,4 +17,4 @@ license = "Apache-2.0" nlohmann.json = "3.12.0" [dev-dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" diff --git a/modules/buildmcpp/mcpp.toml b/modules/buildmcpp/mcpp.toml index ac7fd6b02..a83027dcc 100644 --- a/modules/buildmcpp/mcpp.toml +++ b/modules/buildmcpp/mcpp.toml @@ -41,4 +41,4 @@ mcpp.source-kind = { path = "../source-kind" } mcpp.toolchain-model = { path = "../toolchain-model" } [dev-dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" diff --git a/modules/buildopt/mcpp.toml b/modules/buildopt/mcpp.toml index ab3a03ccb..07879c19e 100644 --- a/modules/buildopt/mcpp.toml +++ b/modules/buildopt/mcpp.toml @@ -21,4 +21,4 @@ mcpp.manifest = { path = "../manifest" } mcpp.toolchain-model = { path = "../toolchain-model" } [dev-dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" diff --git a/modules/graph/mcpp.toml b/modules/graph/mcpp.toml index fa53d0444..a05b6ebbb 100644 --- a/modules/graph/mcpp.toml +++ b/modules/graph/mcpp.toml @@ -17,4 +17,4 @@ description = "Algorithms over index-addressed directed graphs: topological orde license = "Apache-2.0" [dev-dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" diff --git a/modules/manifest/mcpp.toml b/modules/manifest/mcpp.toml index 6608ff1ec..56a2c75cc 100644 --- a/modules/manifest/mcpp.toml +++ b/modules/manifest/mcpp.toml @@ -31,4 +31,4 @@ mcpplibs.os = "0.4.7" # state its contract in isolation; the root `tests/unit/` states the facts two # layers must agree on. [dev-dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" diff --git a/modules/surface/mcpp.toml b/modules/surface/mcpp.toml index 3b6f7fb30..ef27c8d55 100644 --- a/modules/surface/mcpp.toml +++ b/modules/surface/mcpp.toml @@ -14,4 +14,4 @@ description = "The items of mcpp's surface and the record of those an invocation license = "Apache-2.0" [dev-dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" diff --git a/modules/toolchain-model/mcpp.toml b/modules/toolchain-model/mcpp.toml index 7b07c45a2..13d7e3542 100644 --- a/modules/toolchain-model/mcpp.toml +++ b/modules/toolchain-model/mcpp.toml @@ -29,7 +29,7 @@ mcpplibs.os = "0.4.7" # state its contract in isolation; the root `tests/unit/` states the facts two # layers must agree on. [dev-dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" # build.mcpp compiles data/builtin.toml (the built-in target rows and providers) # into a module of static tables, with mcpp's own TOML parser. diff --git a/modules/unit-cache/mcpp.toml b/modules/unit-cache/mcpp.toml index cb15d905c..adadf39cc 100644 --- a/modules/unit-cache/mcpp.toml +++ b/modules/unit-cache/mcpp.toml @@ -18,4 +18,4 @@ license = "Apache-2.0" mcpp.toolchain-model = { path = "../toolchain-model" } [dev-dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" diff --git a/modules/versioning/mcpp.toml b/modules/versioning/mcpp.toml index 0e4ff3542..b1302dd8f 100644 --- a/modules/versioning/mcpp.toml +++ b/modules/versioning/mcpp.toml @@ -25,4 +25,4 @@ license = "Apache-2.0" # state its contract in isolation; the root `tests/unit/` states the same facts # as the engine consumes them, and the pair is the cross-check. [dev-dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" diff --git a/src/build/test_protocol.cppm b/src/build/test_protocol.cppm index 23a370f0b..f7d2d588e 100644 --- a/src/build/test_protocol.cppm +++ b/src/build/test_protocol.cppm @@ -19,7 +19,7 @@ inline constexpr int kVersion = 1; // The framework mcpp scaffolds tests with and suggests where a package has // none (docs/01, docs/08): mtest, by this release, from its repository. -inline constexpr std::string_view kMtestRelease = "0.11.0"; +inline constexpr std::string_view kMtestRelease = "0.11.1"; inline constexpr std::string_view kMtestRepo = "https://github.com/mcpp-community/mtest"; // The protocol's variables, removed from the environment a program inherits diff --git a/tests/canary/cxx/mcpp.toml b/tests/canary/cxx/mcpp.toml index 269c4e9fe..5a87ed892 100644 --- a/tests/canary/cxx/mcpp.toml +++ b/tests/canary/cxx/mcpp.toml @@ -9,4 +9,4 @@ kind = "bin" main = "src/main.cpp" [dev-dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" diff --git a/tests/support/mcpp.toml b/tests/support/mcpp.toml index 778ed1c4b..c21b87383 100644 --- a/tests/support/mcpp.toml +++ b/tests/support/mcpp.toml @@ -13,5 +13,5 @@ license = "Apache-2.0" kind = "lib" [dependencies] -mcpplibs.mtest = "0.11.0" +mcpplibs.mtest = "0.11.1" nlohmann.json = "3.12.0" From 4d4c2fc34f6b862de1f6c2c9b3b982f4eb9e0c3d Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 06:17:19 +0900 Subject: [PATCH 25/38] =?UTF-8?q?design=202026-11-20=20=C2=A714:=20the=20r?= =?UTF-8?q?ecord=20of=20the=20implementation=20--=20repositories,=20measur?= =?UTF-8?q?ements,=20and=20where=20it=20differs=20from=20the=20design?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...-11-20-test-architecture-upgrade-design.md | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/.agents/docs/2026-11-20-test-architecture-upgrade-design.md b/.agents/docs/2026-11-20-test-architecture-upgrade-design.md index ec24f3075..81f1e340c 100644 --- a/.agents/docs/2026-11-20-test-architecture-upgrade-design.md +++ b/.agents/docs/2026-11-20-test-architecture-upgrade-design.md @@ -5,7 +5,7 @@ status: active # 生态的测试与 CI 方案:mtest 的框架与运行控制、mcpp 自身的测试系统、CI 的形态(mcpp#986、#976、#927、#802) -**日期:** 2026-11-20 · **版本:** v4(未实施。负责人 2026-11-20 确认 N1 至 N6,N7 不发出,确定本文为生态级的测试与 CI 方案(1.1),确认评审中补充的 5.16 至 5.22;2026-11-21 确认 N8 至 N10,5.19 不每夜运行,5.22 按树认定而不以合并队列为前提,mcpp 的拉取请求复用 #984) · **基线:** mcpp 2026.11.20.1(`722ec1b6`)、mtest 0.10.0、 +**日期:** 2026-11-20 · **版本:** v5(实施于 mcpp #984 等六个仓库,记录在第 14 节。负责人 2026-11-20 确认 N1 至 N6,N7 不发出,确定本文为生态级的测试与 CI 方案(1.1),确认评审中补充的 5.16 至 5.22;2026-11-21 确认 N8 至 N10,5.19 不每夜运行,5.22 按树认定而不以合并队列为前提,mcpp 的拉取请求复用 #984) · **基线:** mcpp 2026.11.20.1(`722ec1b6`)、mtest 0.10.0、 compat.doctest 2.4.12、mcpplibs/boost-m 拉取请求 5(`e7cc105c`) · **前文:** [CI 与发布的时间成本](2026-11-14-ci-and-release-time-cost-design.md)、 [生态规则收敛方案](2026-11-19-ecosystem-rule-convergence-design.md)(第 11 节,包的实例)、 @@ -810,3 +810,52 @@ doctest 适配器(5.14)、依赖的调试信息(5.15)。boost-m 自己 仍未核对、列为实施中的前提的各项:5.5 的槽位协议与现有 generation 的衔接(只读了注释);5.3 的 BMI 在 Clang 与 MSVC 上 是否逐字节可复现;5.22 的两项前提;5.15 中工具链运行时库的调试信息的份额;mtest 库不由缓存提供的成因。 + +## 14. 实施记录 + +### 14.1 各仓库 + +| 仓库 | 内容 | 状态 | +|---|---|---| +| mcpplibs/os | O1:`machine::processor_model()`;0.4.7 | 已发布 | +| mcpp-community/mtest | T1 至 T11;0.11.0。0.11.1 增加 `time_left()`:`e2e::run` 让一条命令在用例余下的时间之内结束,扣除结束它与报告所需的时间(K1 的「命令的时限取用例的余量」) | 已发布 | +| mcpplibs/cmdline | 0.0.5 导出 `snapshot(app)`:M12 由命令行的定义取得命令与选项 | 已发布 | +| mcpplibs/mcpp-index | #865(cmdline 0.0.5)、#866(mtest 0.11.1) | 已合入 | +| openxlings/xim-pkgindex | #1210(xim:mtest 0.11.0);#1211:链接条目(14.3 第 5 项)与 xim:mtest 0.11.1 | #1211 待合入 | +| mcpp-community/mcpp | #984:S1、S2、M0 至 M14、K1 至 K9、CI1 至 CI9 | 本拉取请求 | + +### 14.2 测量 + +| 项 | 之前 | 之后 | 条件 | +|---|---|---|---| +| 新检出的 mcpp 在已填充的单元缓存上构建(5.3) | 164 s | 4.1 s,288 个编译全部由缓存放置 | 同一 home,第二个检出 | +| 依赖的调试信息(5.15):只依赖 mtest 的工程的测试程序 | 43.7 MB,构建 15.9 s | 8.3 MB,构建 7.2 s | `dev`,`dependency_debug` 缺省(`"lines"`) | +| mtest 库:一个文件的首个用例与期望(T4) | 1.1 s | 265 ms | GCC;目标 150 ms,未达到 | + +### 14.3 与第 5 至 12 节的差别 + +1. **M6、M7、M9 的位置。** 预算的客户在 `src/build/schedule/detach_codegen.cppm` 与 `src/home/files.*`,运行器的记录与 + 夹具在 `src/build/execute.cppm` 与新的 `src/build/test_fixture.cppm`,没有成为 `modules/budget`、`modules/test-run`: + 它们接入运行器的工作循环及其类型(`TestRunnable`、分片、`TestResult`),这些类型位于 `execute.cppm`;运行器移出 + `src/` 是一次单独的移动。新包为 `modules/build-record`、`modules/unit-cache`、`modules/surface`。 +2. **K1 的用到的项来自 mcpp 自身。** kit 不解析命令行:解析器知道一个选项属于哪个命令,kit 重新实现它会与之分歧。 + `MCPP_SURFACE_RECORD` 命名一个文件,调用的每个进程追加它用到的命令、选项、键与原因(每项每进程一次);kit 读取后 + 写出 `exercise`。 +3. **M13 的记录来自构建目录。** 头文件的读者取自 ninja 的依赖日志(格式 4)与扫描的 `.ddi.dep`,而不是单元缓存的清单: + 清单只存在于缓存保留的编译(cl.exe 的编译与没有缓存的 mcpp 没有),构建目录记录它构建的每个单元。工程的某个单元 + 没有记录时,头文件影响全部测试,并说明原因。 +4. **K5 以报告的策略声明。** 诊断行按 `warning:`、`error:` 两个模式以 `report` 策略声明,运行给出各用例的数目,不逐个用例 + 声明。 +5. **K2 与上游的链接条目。** 继承的 home 以 `[xlings] upstream` 指向共享 home 与用户的 `~/.xlings`,不再链接载荷、不复制 + subos。实施中发现:上游的版本目录在下游是指向所有者载荷的链接,配方的 `config()` 钩子经由它写入了所有者的文件—— + 用户 home 中 GCC 的 specs 与 binutils 的 `ld` 包装被写入了临时 home 的路径(#293 的机制)。已就地修复,并在配方中 + 修正:链接条目走借用的分支,`ld` 包装跳过链接条目(xim-pkgindex #1211)。此后 914 检查共享 home 在一次构建后没有 + 文件被写入。 +6. **CI6。** home 夹具的恢复即 `bootstrap-mcpp` 的沙箱缓存,其键(`mcpp.toml`、`.xlings.json`)就是夹具的身份;不另设 + 缓存。 +7. **CI2 的档位。** 发布形态只在 `full` 档构建(`release-build.yml`,与 `release.yml` 同一组作业),Linux 的单元与 C++ e2e + 分片测试 Linux 归档中的程序;`pr` 档测试宿主构建。发布形态是又一次自举构建,`pr` 档的目的是时间。macOS、Windows + 与 aarch64 的归档由各自的冒烟步骤核对,与发布相同。 +8. **P10.4。** 不实现:工作线程在同一进程中运行测试程序,系统对子进程的计量按进程汇总,一个程序的用量无法分离 + (SPEC-017 写明)。 +9. **CI10。** 按 7.3,在固定版本移动到含 5.3 的版本之后。 From 0e107356ff6721290ffaf5ab90e5bce482b4f64f Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 06:17:34 +0900 Subject: [PATCH 26/38] =?UTF-8?q?design=202026-11-20=20=C2=A714:=20the=20c?= =?UTF-8?q?overage=20tables=20stay=20with=20check=5Fe2e=5Fcoverage.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .agents/docs/2026-11-20-test-architecture-upgrade-design.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.agents/docs/2026-11-20-test-architecture-upgrade-design.md b/.agents/docs/2026-11-20-test-architecture-upgrade-design.md index 81f1e340c..6068aa005 100644 --- a/.agents/docs/2026-11-20-test-architecture-upgrade-design.md +++ b/.agents/docs/2026-11-20-test-architecture-upgrade-design.md @@ -859,3 +859,7 @@ doctest 适配器(5.14)、依赖的调试信息(5.15)。boost-m 自己 8. **P10.4。** 不实现:工作线程在同一进程中运行测试程序,系统对子进程的计量按进程汇总,一个程序的用量无法分离 (SPEC-017 写明)。 9. **CI10。** 按 7.3,在固定版本移动到含 5.3 的版本之后。 +10. **K7 的两份表。** `coverage-exceptions.tsv`、`coverage-debt.tsv` 与 `host-requirements.tsv` 仍由 + `check_e2e_coverage.py` 判定,没有迁入 `[ci.coverage]`:它的 `job`(工作流按名字运行一个测试并自行断言其结果)与 + `unaffected`(`pr` 集合按变更略去的测试)两种判定在 `mtest ci coverage` 中没有对应,迁入会使这两类测试成为未覆盖。 + 新的账本是面的账本(`[ledger.surface]`)。 From 0aef4729f9cf8eb0fe5e0603be62518cab8155ac Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 06:24:06 +0900 Subject: [PATCH 27/38] docs/08: the downstream action project-test --- CHANGELOG.md | 3 +++ docs/08-testing.md | 17 +++++++++++++++++ docs/zh/08-testing.md | 13 +++++++++++++ 3 files changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cebbfe8dc..f300e6247 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,9 @@ flags of the command line's definition, the manifest's keys and the refusals' reason tokens. `MCPP_SURFACE_RECORD=` makes each process of an invocation append the items it used. - `wire` features `test-steps`, `test-budget`, `test-jitter` and `test-fixtures` (docs/50). +- `mcpp-community/mcpp/.github/actions/project-test`, a downstream repository's test job: the mcpp + the project pins, its home restored by the project's identity, `mcpp test`, the run's store as + an artifact and its slowest cases and edges in the summary (docs/08). ### Changed diff --git a/docs/08-testing.md b/docs/08-testing.md index 2e915b5c5..f91676243 100644 --- a/docs/08-testing.md +++ b/docs/08-testing.md @@ -693,6 +693,23 @@ Builds the one test the pattern names and prints how `mcpp test` would start it (the `mcpp.launch` envelope, mcpp 2026.10.31.2+), for a debugger to start: the runner, the environment the run adds and the arguments after `--` are in it. +### In a GitHub Actions job (@NEXT@+) + +```yaml +- uses: actions/checkout@v4 +- uses: mcpp-community/mcpp/.github/actions/project-test@main + with: + args: --workspace # the words of `mcpp test` +``` + +The action installs the mcpp the project's `.xlings.json` names (else the input +`mcpp`, else the newest release), restores the home by the system, that version +and the project's manifests and lock (`home-key` replaces the last), runs +`mcpp test`, keeps the run's store (`target/mtest`) as an artifact and writes the +run's result, its ten slowest cases and the ten slowest edges of its builds to +the job's summary. A pull request reads the home; a push to the default branch +saves it. + ## Current limitations - A test is one `.cpp` producing one program. The cases inside it are the diff --git a/docs/zh/08-testing.md b/docs/zh/08-testing.md index 3c478c49c..00cd430e3 100644 --- a/docs/zh/08-testing.md +++ b/docs/zh/08-testing.md @@ -516,6 +516,19 @@ mcpp test unit/math --print-launch -- --gtest_filter=Math.Adds 构建该模式指定的一个测试,并打印 `mcpp test` 启动它的方式(`mcpp.launch` 信封,mcpp 2026.10.31.2+), 供调试器启动:其中包括运行器、运行所加的环境与 `--` 之后的参数。 +### 在 GitHub Actions 的作业中(@NEXT@+) + +```yaml +- uses: actions/checkout@v4 +- uses: mcpp-community/mcpp/.github/actions/project-test@main + with: + args: --workspace # `mcpp test` 的参数 +``` + +该动作安装工程的 `.xlings.json` 所写的 mcpp(否则为输入 `mcpp`,再否则为最新的发布),按系统、该版本与工程的清单和锁文件 +(`home-key` 取代后两者)恢复 home,运行 `mcpp test`,把运行的存储(`target/mtest`)保留为 artifact,并把运行的结果、最慢的 +十个用例与其构建中最慢的十条边写入作业的摘要。拉取请求只读取 home;向缺省分支的推送保存它。 + ## 当前边界 - 一个测试是一个 `.cpp` 产出一个程序。其中的用例属于程序:mcpp 只对实现测试协议的程序列出并选择用例; From bb94dab71968e08463953beb95f935e6c3f2b401 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 06:49:38 +0900 Subject: [PATCH 28/38] The build record states an edge that copied a unit of a package the cache served (staged); 1122 bounds the dependency's compiles at its entry unit --- docs/08-testing.md | 3 ++- docs/50-machine-output.md | 2 +- docs/specs/test.md | 2 +- docs/zh/08-testing.md | 2 +- docs/zh/50-machine-output.md | 2 +- modules/build-record/src/build_record.cppm | 4 +++- .../build-record/tests/test_build_record.cpp | 16 ++++++++++++++++ src/build/progress.cppm | 15 +++++++++++++-- tests/e2e/cache/invariants.cpp | 19 +++++++++++++------ tests/unit/test_build_progress.cpp | 16 ++++++++++++++++ 10 files changed, 67 insertions(+), 14 deletions(-) diff --git a/docs/08-testing.md b/docs/08-testing.md index f91676243..06ab569fb 100644 --- a/docs/08-testing.md +++ b/docs/08-testing.md @@ -169,7 +169,8 @@ with its time and status), the items of the product's surface it used (`exercised`) and the limit it ran within (`limit_ms`). `builds.ndjson` holds the builds of the run: a line per edge that ran, with its package, its time, the processor time and the largest resident set of the compiler, and whether the unit -cache placed it (`served`), and a line per package the cache served. The +cache placed it (`served`) or it copied a unit of a package the cache served +(`staged`), and a line per package the cache served. The estimates of the order prefer the runs of the same class of machine. **One budget for the machine** (@NEXT@+, SPEC-017 O11). `mcpp test` makes a diff --git a/docs/50-machine-output.md b/docs/50-machine-output.md index 6a0ea4fc8..14e81f38f 100644 --- a/docs/50-machine-output.md +++ b/docs/50-machine-output.md @@ -1000,7 +1000,7 @@ so a build's own sub-builds and the compilers it starts through mcpp add to the | variable | each line | |---|---| | `MCPP_SURFACE_RECORD` | an item of the surface the process used, once per process: the command and flags it was given, the registry keys its manifests state, the reasons it recorded | -| `MCPP_BUILD_RECORD` | an edge a build ran (`build`, `output`, `package`, `program`, `started_ms`, `ms`, and for a compile `cpu_ms`, `peak_bytes` and `served`), or a package the cache served (`build`, `package`, `cached`, `units`): SPEC-017 B4's `builds.ndjson` | +| `MCPP_BUILD_RECORD` | an edge a build ran (`build`, `output`, `package`, `program`, `started_ms`, `ms`, and for a compile `cpu_ms`, `peak_bytes` and `served`; `staged` for a copy of a unit of a package the cache served), or a package the cache served (`build`, `package`, `cached`, `units`): SPEC-017 B4's `builds.ndjson` | A test kit turns the first into a case's `exercise` events (SPEC-017 P5.6), and the surface's ledger compares the two. diff --git a/docs/specs/test.md b/docs/specs/test.md index a672ef462..001573890 100644 --- a/docs/specs/test.md +++ b/docs/specs/test.md @@ -175,7 +175,7 @@ | `runs//run.json` | 运行记录:`schema`(`mtest.run/1`)、`id`、`started`、`finished`、`status`、`system`、`arch`、`machine`、`commit`、`set`、`selection`、`group`、`monitor`、`ms`、`jobs`、`counts`;`processors`、`memory`、`budget`、`jitter`(@NEXT@) | | `runs//programs.ndjson` | 每个程序进程一行:`name`、`argv`、`status`、`exit`、`ms`、`started_ms`、`worker`、`select`、`resources`、`fixtures`,运行器构建它时的 `build_started_ms` 与 `build_ms`,`usage`(带 `source`:`program` 或 `engine`) | | `runs//cases.ndjson` | 每个用例一行:`program`、`id`、`kind`、`status`、`ms`、`started_ms`、`reason`、`failures`、`metrics`、`bounds`、`tags`、`clauses`;`cpu_ms`、`steps`(`step` 事件的字段,`id` 除外)、`exercised`、`limit_ms`(用例所受的时限,O5;未知时省略)(@NEXT@) | -| `runs//builds.ndjson` | 运行器为这次运行所做的构建(@NEXT@):每条运行过的边一行,`build`(构建记录的标识)、`output`、`package`、`program`(该边所服务的测试程序,可为空)、`started_ms`、`ms`、`cpu_ms`、`peak_bytes`,单元缓存放置的边另有 `served`(`true`,没有编译器运行);由缓存提供的包一行,`build`、`package`、`cached`(`true`)、`units` | +| `runs//builds.ndjson` | 运行器为这次运行所做的构建(@NEXT@):每条运行过的边一行,`build`(构建记录的标识)、`output`、`package`、`program`(该边所服务的测试程序,可为空)、`started_ms`、`ms`、`cpu_ms`、`peak_bytes`,单元缓存放置的边另有 `served`(`true`,没有编译器运行),为缓存提供的包复制单元的边另有 `staged`(`true`,同样没有编译器运行);由缓存提供的包一行,`build`、`package`、`cached`(`true`)、`units` | | `runs//fixtures.ndjson` | 每个运行夹具一行:`name`、`program`、`started_ms`、`ready_ms`、`ended_ms`、`status` | - **P12.4 机器(@NEXT@)。** `machine` 是机器的类别:处理器的型号,系统不给出型号时为 `/<逻辑处理器数>`;`processors` 为逻辑处理器数,`memory` 为物理内存的字节数。比较不同运行的用时(O10 的估计、界限、趋势)时,读者**应当**按 `machine` 分组;没有同类的历史时取全部。处理器时间(`cpu_ms`)在同类机器之间可比,墙钟只在同一次运行之内可比。 diff --git a/docs/zh/08-testing.md b/docs/zh/08-testing.md index 00cd430e3..54252d63e 100644 --- a/docs/zh/08-testing.md +++ b/docs/zh/08-testing.md @@ -119,7 +119,7 @@ mtest 自身的运行一样读取它们(Markdown 报告、最慢的用例、 环境未陈述时为 `basic`:每个用例的处理器时间)与陈述了的抖动。用例的一行另有其处理器时间(`cpu_ms`)、步骤(端到端 用例运行的每条命令及其用时与状态)、它用到的产品面的项(`exercised`)与它所受的时限(`limit_ms`)。`builds.ndjson` 保存运行中的构建:每条运行的边一行,带包、用时、编译器的处理器时间与最大驻留集,以及是否由单元缓存放置 -(`served`);缓存提供的每个包一行。调度的估计优先取同类机器的运行。 +(`served`)或为缓存提供的包复制(`staged`);缓存提供的每个包一行。调度的估计优先取同类机器的运行。 **机器的一份预算**(@NEXT@+,SPEC-017 O11)。`mcpp test` 建立每个作业一个槽位文件的目录,并告知其程序 (`MTEST_BUDGET`,以及工作者持有的槽位 `MTEST_BUDGET_HELD`);测试经由 mcpp 启动的编译器在启动前持有其中一个槽位, diff --git a/docs/zh/50-machine-output.md b/docs/zh/50-machine-output.md index d14243250..dfbd153df 100644 --- a/docs/zh/50-machine-output.md +++ b/docs/zh/50-machine-output.md @@ -888,7 +888,7 @@ reason unknown-target | 变量 | 每行 | |---|---| | `MCPP_SURFACE_RECORD` | 进程用到的面的一项,每进程一次:所给的命令与选项、其清单陈述的登记表中的键、它记录的原因 | -| `MCPP_BUILD_RECORD` | 构建运行的一条边(`build`、`output`、`package`、`program`、`started_ms`、`ms`,编译另有 `cpu_ms`、`peak_bytes` 与 `served`),或缓存提供的一个包(`build`、`package`、`cached`、`units`):SPEC-017 B4 的 `builds.ndjson` | +| `MCPP_BUILD_RECORD` | 构建运行的一条边(`build`、`output`、`package`、`program`、`started_ms`、`ms`,编译另有 `cpu_ms`、`peak_bytes` 与 `served`;为缓存提供的包复制单元的边另有 `staged`),或缓存提供的一个包(`build`、`package`、`cached`、`units`):SPEC-017 B4 的 `builds.ndjson` | 测试 kit 把前者转为用例的 `exercise` 事件(SPEC-017 P5.6),面的账本比较二者。 diff --git a/modules/build-record/src/build_record.cppm b/modules/build-record/src/build_record.cppm index 9f262acd7..175e8d407 100644 --- a/modules/build-record/src/build_record.cppm +++ b/modules/build-record/src/build_record.cppm @@ -44,6 +44,7 @@ struct Edge { std::optional cpuMs; std::optional peakBytes; bool served = false; // placed by the unit cache (design 2026-11-20 §5.3) + bool staged = false; // placed a unit of a package the cache served: a copy }; // A package the cache served, with the units it placed. struct Cached { @@ -162,6 +163,7 @@ std::string to_ndjson(const Record& r) { if (e.cpuMs) j["cpu_ms"] = *e.cpuMs; if (e.peakBytes) j["peak_bytes"] = *e.peakBytes; if (e.served) j["served"] = true; + if (e.staged) j["staged"] = true; out += j.dump(-1, ' ', false, json::error_handler_t::replace) + "\n"; } for (const auto& c : r.cached) @@ -184,7 +186,7 @@ std::vector from_ndjson(std::string_view text) { at->edges.push_back({text_of(j, "output"), text_of(j, "package"), text_of(j, "program"), number_of(j, "started_ms").value_or(0), number_of(j, "ms").value_or(0), number_of(j, "cpu_ms"), bytes_of(j, "peak_bytes"), - j.value("served", false)}); + j.value("served", false), j.value("staged", false)}); } return out; } diff --git a/modules/build-record/tests/test_build_record.cpp b/modules/build-record/tests/test_build_record.cpp index ad61747e7..2d73ff6e1 100644 --- a/modules/build-record/tests/test_build_record.cpp +++ b/modules/build-record/tests/test_build_record.cpp @@ -52,6 +52,22 @@ UTest a_record_reads_back_from_its_lines{"BuildRecord/ARecordReadsBackFromItsLin expect(eq(top[0].output, std::string("obj/c.o"))); }}; +UTest placement_and_staging_read_back{"BuildRecord/PlacementAndStagingReadBack", [] { + // A unit the unit cache placed, one copied for a package the cache served, and a compile: a + // reader that counts compiles tells the three apart. + rec::Edge placed{"obj/p.o", "app"}; + placed.served = true; + rec::Edge copied{"obj/dep/s.o", "dep"}; + copied.staged = true; + const rec::Record r{"b", {placed, copied, {"obj/c.o", "app"}}, {{"dep", 1}}}; + const auto back = rec::from_ndjson(rec::to_ndjson(r)); + expect(fatal(eq(back.size(), std::size_t{1}))); + expect(fatal(eq(back[0].edges.size(), std::size_t{3}))); + expect(back[0].edges[0].served && !back[0].edges[0].staged); + expect(back[0].edges[1].staged && !back[0].edges[1].served); + expect(!back[0].edges[2].served && !back[0].edges[2].staged); +}}; + UTest identifiers_differ{"BuildRecord/IdentifiersDiffer", [] { std::set ids; for (int i = 0; i < 64; ++i) ids.insert(rec::new_id()); diff --git a/src/build/progress.cppm b/src/build/progress.cppm index 48e3ac9e6..1bd801dc5 100644 --- a/src/build/progress.cppm +++ b/src/build/progress.cppm @@ -68,6 +68,9 @@ struct Record { // first input, relative to the package root that holds it. `Finished` // names the longest step by it (build output design revision 3, §7.3). std::unordered_map sources; + // The outputs of the steps that place a unit of a package the cache served + // (`stage_file`): a copy, not a compile (the build record's `staged`). + std::unordered_set staged; std::size_t steps = 0; // every step of the graph // The profile as `Finished` describes it. The fast path, which has no // plan, reads it from the header line alone (`read_descriptor`). @@ -366,8 +369,9 @@ long long to_ll(std::string_view s) { } // namespace // Format: a header, then `P` lines (packages, in order), `A` lines (actions), -// `S` lines (a step's identity and its source) and `O` lines (an output, its -// package's index and its step's identity). Version 2 adds the profile's +// `S` lines (a step's identity and its source), `O` lines (an output, its +// package's index and its step's identity) and `T` lines (an output a staging +// step writes; a reader that predates them skips them). Version 2 adds the profile's // descriptor to the header, a package's detail and source to its `P` line, // and the `S` lines; a version 1 record (mcpp 2026.9.29.5) is read with all // of them empty, and its subject then carries what version 1 wrote there. @@ -392,6 +396,9 @@ std::string format_record(const Record& r) { out += std::format("O\t{}\t{}\t{}\n", index, st == r.step.end() ? std::size_t{0} : st->second, field(path)); } + std::vector staged(r.staged.begin(), r.staged.end()); + std::ranges::sort(staged); + for (auto const& path : staged) out += std::format("T\t{}\n", field(path)); return out; } @@ -418,6 +425,8 @@ Record parse_record(std::string_view text) { r.actions.emplace(std::string(f[1]), std::string(f[2])); } else if (f[0] == "S" && f.size() >= 3) { r.sources.emplace(to_size(f[1]), std::string(f[2])); + } else if (f[0] == "T" && f.size() >= 2) { + r.staged.emplace(std::string(f[1])); } else if (f[0] == "O" && f.size() >= 4) { const auto index = to_size(f[1]); if (index < r.packages.size()) { @@ -571,6 +580,7 @@ Record Attribution::record(const std::vector& declared, for (auto const& o : s.outputs) { r.owner.emplace(o, it->second); r.step.emplace(o, id); + if (s.rule == "stage_file") r.staged.emplace(o); } if (auto src = source_in(s.input, roots); !src.empty()) r.sources.emplace(id, std::move(src)); } @@ -1830,6 +1840,7 @@ mcpp::build::record::Record ran_since(const std::filesystem::path& buildDir, con const auto& first = st.outputs.front(); if (!kept && !measured.contains(first)) continue; rec::Edge e{first}; + e.staged = stepRecord && stepRecord->staged.contains(first); if (stepRecord) for (const auto& o : st.outputs) if (auto it = stepRecord->owner.find(o); it != stepRecord->owner.end()) { diff --git a/tests/e2e/cache/invariants.cpp b/tests/e2e/cache/invariants.cpp index a95afff37..400dc02e3 100644 --- a/tests/e2e/cache/invariants.cpp +++ b/tests/e2e/cache/invariants.cpp @@ -21,6 +21,7 @@ struct Built { std::size_t compiled = 0; std::map cached; std::set compiledPackages; + std::vector compiledOutputs; // ` `, for the failure's message }; bool compiles(std::string_view output) { @@ -36,9 +37,12 @@ Built read_record(const fs::path& file) { b.cached[line.value("package", "")] += line.value("units", std::size_t{0}); continue; } - if (compiles(line.value("output", "")) && !line.value("served", false)) { + // A unit the unit cache placed (`served`) or copied for a package the cache served + // (`staged`) ran no compiler. + if (compiles(line.value("output", "")) && !line.value("served", false) && !line.value("staged", false)) { ++b.compiled; b.compiledPackages.insert(line.value("package", "")); + b.compiledOutputs.push_back(line.value("package", "") + " " + line.value("output", "")); } } return b; @@ -65,9 +69,10 @@ E2E invariants{"invariants", s::home_fixture, [] { metric("edges.compiled", static_cast(rebuilt.compiled), "edges"); }; - "1122 a dev-dependency is served from the cache to a second project"_test[s::gcc][at_most{"edges.compiled.dependency", 0}] = [] { + "1122 a dev-dependency is served from the cache to a second project"_test[s::gcc][at_most{"edges.compiled.dependency", 1}] = [] { // 5.4: mcpplibs.mtest, a dev-dependency of every test, is compiled once per home, not once - // per project. + // per project. Its entry unit (`main`) is the one compile left: a package's entry unit is + // compiled with the image that links it, never taken from the package's entry in the cache. s::own_home(); auto one = s::project::scaffold("one"); auto [first, built] = recorded(one, {"test"}, "one"); @@ -76,9 +81,11 @@ E2E invariants{"invariants", s::home_fixture, [] { auto [second, rebuilt] = recorded(two, {"test"}, "two"); expect(fatal(e2e::succeeded(second))) << s::text(second); std::size_t dependency = 0; - for (const auto& pkg : rebuilt.compiledPackages) - if (pkg.contains("mtest")) ++dependency; - expect(eq(dependency, std::size_t{0})) << "the second project compiled the dev-dependency"; + std::string compiled; + for (const auto& edge : rebuilt.compiledOutputs) + if (edge.contains("mtest")) { ++dependency; compiled += "\n " + edge; } + expect(le(dependency, std::size_t{1})) << "the second project compiled the dev-dependency:" << compiled << "\n" + << s::text(second); bool served = false; for (const auto& [pkg, units] : rebuilt.cached) served = served || (pkg.contains("mtest") && units > 0); expect(served) << "the record names no unit of mcpplibs.mtest served from the cache"; diff --git a/tests/unit/test_build_progress.cpp b/tests/unit/test_build_progress.cpp index 471576d29..2da3325a8 100644 --- a/tests/unit/test_build_progress.cpp +++ b/tests/unit/test_build_progress.cpp @@ -190,6 +190,22 @@ UTest every_statement_is_recorded_with_its_owner{"ProgressRecord/EveryStatementI expect(eq(back.actions, r.actions)); }}; +UTest a_staging_step_is_recorded_as_staged{"ProgressRecord/AStagingStepIsRecordedAsStaged", [] { + // A unit of a package the cache served is copied by a `stage_file` step: the build record + // states it as staged, not as a compile. + Attribution a; + a.owner("dep"); + a.statement("build obj/dep/s.o : stage_file /home/cache/s.o\n verify = --verify size\n"); + a.statement("build gcm.cache/dep.gcm : stage_file /home/cache/dep.gcm\n"); + a.owner("app"); + a.statement("build obj/main.o : cxx_obj src/main.cpp\n"); + const auto r = a.record({{"app", true, "app", 0, 0}}); + expect(r.staged.contains("obj/dep/s.o")); + expect(r.staged.contains("gcm.cache/dep.gcm")); + expect(!r.staged.contains("obj/main.o")); + expect(eq(parse_record(format_record(r)).staged, r.staged)); +}}; + UTest a_version_one_record_is_read{"ProgressRecord/AVersionOneRecordIsRead", [] { // 2026.9.29.5 wrote version 1: six fields to a package line, the subject // carrying the origin. The fast path of a newer mcpp reads it as it is. From c14b434b1371996fa04d85fcba7c1132658bb154 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 06:49:38 +0900 Subject: [PATCH 29/38] CI: a consumer of this commit's mcpp reads the current index's recipes (a restored sandbox keeps an index older than xim-pkgindex c0023dd) --- .github/actions/use-built-mcpp/use.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/actions/use-built-mcpp/use.sh b/.github/actions/use-built-mcpp/use.sh index 3d8d275b7..d2c0ec0ec 100644 --- a/.github/actions/use-built-mcpp/use.sh +++ b/.github/actions/use-built-mcpp/use.sh @@ -94,6 +94,14 @@ if [ -n "${XLINGS_BIN:-}" ]; then fi MCPP_VENDORED_XLINGS="${XLINGS_BIN:-}" "$bin" self config --mirror "$mirror" +# THE RECIPES OF THE CURRENT INDEX. A restored sandbox keeps the index it was +# saved with, and the tests' homes read this home's index and name this home an +# upstream (tests/support, tests/e2e/_inherit_toolchain.sh). A recipe older than +# xim-pkgindex c0023dd wrote its `config()` hook through an upstream's link +# entry: measured, gcc's specs and binutils' ld wrapper of this home named a +# case's deleted home after one case, and every later link through them failed. +MCPP_VENDORED_XLINGS="${XLINGS_BIN:-}" "$bin" index update + # THE TOOLCHAIN THE BUILD USED IS INSTALLED, AS IT WAS WHEN EVERY JOB BUILT. # A job that built mcpp itself installed this toolchain as a side effect, and # the steps after the build relied on it without saying so: measured on the From 29f7ddff1e23701a7d36dce086950fca92612f80 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 06:50:27 +0900 Subject: [PATCH 30/38] =?UTF-8?q?design=202026-11-20=20=C2=A714:=20the=20i?= =?UTF-8?q?ndex=20the=20fix=20needs,=20and=20the=20record's=20staged=20edg?= =?UTF-8?q?es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../docs/2026-11-20-test-architecture-upgrade-design.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.agents/docs/2026-11-20-test-architecture-upgrade-design.md b/.agents/docs/2026-11-20-test-architecture-upgrade-design.md index 6068aa005..dd88230a0 100644 --- a/.agents/docs/2026-11-20-test-architecture-upgrade-design.md +++ b/.agents/docs/2026-11-20-test-architecture-upgrade-design.md @@ -850,7 +850,8 @@ doctest 适配器(5.14)、依赖的调试信息(5.15)。boost-m 自己 subos。实施中发现:上游的版本目录在下游是指向所有者载荷的链接,配方的 `config()` 钩子经由它写入了所有者的文件—— 用户 home 中 GCC 的 specs 与 binutils 的 `ld` 包装被写入了临时 home 的路径(#293 的机制)。已就地修复,并在配方中 修正:链接条目走借用的分支,`ld` 包装跳过链接条目(xim-pkgindex #1211)。此后 914 检查共享 home 在一次构建后没有 - 文件被写入。 + 文件被写入。修正只在索引含有它时生效:用例的 home 链接用户 home 的索引,本地的索引停在 #1211 之前时再次写穿; + CI 恢复的沙箱同样带着保存时的索引,因此使用本提交 mcpp 的每个作业先运行 `mcpp index update`(`use-built-mcpp`)。 6. **CI6。** home 夹具的恢复即 `bootstrap-mcpp` 的沙箱缓存,其键(`mcpp.toml`、`.xlings.json`)就是夹具的身份;不另设 缓存。 7. **CI2 的档位。** 发布形态只在 `full` 档构建(`release-build.yml`,与 `release.yml` 同一组作业),Linux 的单元与 C++ e2e @@ -863,3 +864,7 @@ doctest 适配器(5.14)、依赖的调试信息(5.15)。boost-m 自己 `check_e2e_coverage.py` 判定,没有迁入 `[ci.coverage]`:它的 `job`(工作流按名字运行一个测试并自行断言其结果)与 `unaffected`(`pr` 集合按变更略去的测试)两种判定在 `mtest ci coverage` 中没有对应,迁入会使这两类测试成为未覆盖。 新的账本是面的账本(`[ledger.surface]`)。 +11. **构建记录的 `staged`。** 1122 在实施中失败:缓存提供的包的单元由 `stage_file` 边复制,构建记录把这些边与编译 + 一同列出,第二个工程看起来编译了 mtest 的 13 个输出。记录为这类边增加 `staged`(SPEC-017 B4),它与单元缓存的 + `served` 一样不是编译。余下的一个编译是 mtest 的入口单元:包的入口单元随链接它的映像编译,不取自缓存中包的条目, + 1122 的界限为 1。 From f81f15312a961ef9e0ed51f7e8bbb0bc5bf8f5d6 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 06:51:03 +0900 Subject: [PATCH 31/38] 2026.11.21.1 --- CHANGELOG.md | 2 +- docs/04-mcpp-toml.md | 4 ++-- docs/08-testing.md | 12 +++++------ docs/26-build-optimizations.md | 4 ++-- docs/50-machine-output.md | 10 ++++----- docs/92-release.md | 2 +- docs/specs/manifest-semantics.md | 4 ++-- docs/specs/test.md | 32 ++++++++++++++--------------- docs/zh/04-mcpp-toml.md | 4 ++-- docs/zh/08-testing.md | 12 +++++------ docs/zh/26-build-optimizations.md | 4 ++-- docs/zh/50-machine-output.md | 10 ++++----- mcpp.toml | 2 +- modules/versioning/src/version.cppm | 2 +- 14 files changed, 52 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f300e6247..d66d996e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ > Each `## []` section is that release's notes. Entries are written in English > from 2026.9.28.3 on; earlier entries remain as written. -## [Unreleased] +## [2026.11.21.1] - 2026-11-21 ### Added diff --git a/docs/04-mcpp-toml.md b/docs/04-mcpp-toml.md index a5b01b7ea..09d7b1c33 100644 --- a/docs/04-mcpp-toml.md +++ b/docs/04-mcpp-toml.md @@ -1225,7 +1225,7 @@ its final path with `O_TRUNC` and a reader could observe a half-written file. MSVC is left alone — neither `/ifcOnly`'s cost nor `.ifc` atomicity has been measured, and guessing either wrong is silent. -### Available memory (`memory_threshold`) *(@NEXT@+)* +### Available memory (`memory_threshold`) *(2026.11.21.1+)* ```toml [build] @@ -1848,7 +1848,7 @@ anything compiles. into the program (`__builtin_LINE()`, `std::source_location::current()` in a template's body) and a graph compiled with `-fcontracts` are compared exactly under either value. Clang's BMI has no such part and the key has no effect on it. -- **`dependency_debug` (@NEXT@+)** states the debug information of the dependencies +- **`dependency_debug` (2026.11.21.1+)** states the debug information of the dependencies that come from the package store and from git: `true` (the profile's `debug`), `false` (none), `"lines"` (the line tables alone: `-g1` on GCC, `-gline-tables-only` on Clang, from the toolchain's row). Omitted, `dev` and `debug` diff --git a/docs/08-testing.md b/docs/08-testing.md index 06ab569fb..2999b693c 100644 --- a/docs/08-testing.md +++ b/docs/08-testing.md @@ -159,7 +159,7 @@ the slowest cases, the critical path and the schedule). In CI, `commit` and `group` are `GITHUB_SHA` and `GITHUB_RUN_ID`. `[test] store = ""` names another directory, relative to the project; `"none"` keeps nothing. -**What a run records** (@NEXT@+, [SPEC-017](specs/test.md) B4, P5.6, P12.4). A run's +**What a run records** (2026.11.21.1+, [SPEC-017](specs/test.md) B4, P5.6, P12.4). A run's `run.json` and its line in `index.ndjson` name the machine's class (`machine`: the processor's model), its `processors` and `memory`, the monitoring level the run stated (`MTEST_MONITOR`, `basic` unless the environment states another: each @@ -173,14 +173,14 @@ cache placed it (`served`) or it copied a unit of a package the cache served (`staged`), and a line per package the cache served. The estimates of the order prefer the runs of the same class of machine. -**One budget for the machine** (@NEXT@+, SPEC-017 O11). `mcpp test` makes a +**One budget for the machine** (2026.11.21.1+, SPEC-017 O11). `mcpp test` makes a directory of one slot file per job and states it to its programs (`MTEST_BUDGET`, and `MTEST_BUDGET_HELD`, the slot a worker holds); a compiler a test starts through mcpp holds a slot of it before it starts, so a test that builds a project takes from the run's jobs rather than adding its own. A run inside another runner's budget uses that one. -**A case's time is the one its scopes state** (@NEXT@+). A case selected with +**A case's time is the one its scopes state** (2026.11.21.1+). A case selected with `--case` runs within the `timeout` its listed case states, not the run's. A suite compiled from several sources of its own is also a package: a workspace @@ -361,7 +361,7 @@ the tests a change affects, read from the plan before anything is built: of its link that belongs to no module; - a change of a manifest, a lock, a build program, or a source the plan does not hold affects every test; -- a header (@NEXT@+) affects the tests built from a unit that read it when +- a header (2026.11.21.1+) affects the tests built from a unit that read it when the build directory last built it, as ninja's deps log and the scans' depfiles record it, and the tests that run a program of the package when such a unit is not a test's own; while a unit of the project has no such @@ -641,7 +641,7 @@ The records of `--message-format json` carry the protocol, the cases and their failures ([50](50-machine-output.md)); `--print-launch` states the protocol for a debugger that starts the program. -**Run fixtures** (@NEXT@+, SPEC-017 O6). A program may declare a run fixture -- +**Run fixtures** (2026.11.21.1+, SPEC-017 O6). A program may declare a run fixture -- one object the whole run shares: a sandboxed home, a server, an index -- and its cases may name it in `uses` (mtest's `run_fixture` and `uses{}`). `mcpp test` reads both from the programs' listings. Before the first program whose selected @@ -694,7 +694,7 @@ Builds the one test the pattern names and prints how `mcpp test` would start it (the `mcpp.launch` envelope, mcpp 2026.10.31.2+), for a debugger to start: the runner, the environment the run adds and the arguments after `--` are in it. -### In a GitHub Actions job (@NEXT@+) +### In a GitHub Actions job (2026.11.21.1+) ```yaml - uses: actions/checkout@v4 diff --git a/docs/26-build-optimizations.md b/docs/26-build-optimizations.md index fee726ba2..f55d340f0 100644 --- a/docs/26-build-optimizations.md +++ b/docs/26-build-optimizations.md @@ -86,7 +86,7 @@ The note is given when the rebuild started at a module of one file with at least importing units, once a build directory. Beside it, a build that replaced or kept BMIs states which ([04](04-mcpp-toml.md), `bmi_cutoff`). -## What the home's cache places (@NEXT@+) +## What the home's cache places (2026.11.21.1+) Beside the entries, the home's build cache (`build-cache/v1/`, [25](25-home-indices-and-store.md)) keeps what one build made for the next, in any project of the home, whenever `[build] cache` @@ -109,7 +109,7 @@ is the home's (the default): Entries no build used within `[cache] unused_days` are removed as other entries are. -## The record of a build (@NEXT@+) +## The record of a build (2026.11.21.1+) A build records each edge it ran: its package, its time, and, for a compile, the compiler's processor time and largest resident set and whether the unit cache placed it; and the units diff --git a/docs/50-machine-output.md b/docs/50-machine-output.md index 14e81f38f..415737789 100644 --- a/docs/50-machine-output.md +++ b/docs/50-machine-output.md @@ -254,10 +254,10 @@ envelope says when either has changed since. | `test-frameworks` | 2026.11.10.1 | `mcpp test --guide` names each framework's `capabilities` and `session` (SPEC-017 B6) | an IDE choosing between a session and processes | | `test-checks` | 2026.11.10.1 | with `test-protocol`: a failure's `source` and `fatal`, and a case's `checks` (`held`, `failed`), as the program reported them (SPEC-017 O3) | an IDE's list of checks | | `test-output` | 2026.11.10.1 | with `test-case-stream`: a case's own output, `output`: the program's `output` events, or all the output of a program that ran that case alone, in its program's `cases` and, from events, in its `end` record (SPEC-017 O8) | an IDE's output of one case | -| `test-steps` | @NEXT@ | with `test-protocol`: a case's `steps` and the surface items it used (`exercised`), its `cpu_ms` and `limit_ms`, and the run's `builds.ndjson` (SPEC-017 P5.6, B4) | a run's report | -| `test-budget` | @NEXT@ | `mcpp test` states one budget of slots (`MTEST_BUDGET`) its programs' compilers take, and uses a budget its environment states (SPEC-017 O11) | `mtest run`, a runner that runs `mcpp test` | -| `test-jitter` | @NEXT@ | the jitter a run states (`MTEST_JITTER`, `MTEST_JITTER_SEED`) reaches its programs and its record (SPEC-017 O12) | the jitter workflow | -| `test-fixtures` | @NEXT@ | run fixtures are served for the programs whose cases use them (`MTEST_SERVE`, `MTEST_FIXTURE_`), and a run's `fixtures.ndjson` states them (SPEC-017 O6) | a framework's shared fixtures | +| `test-steps` | 2026.11.21.1 | with `test-protocol`: a case's `steps` and the surface items it used (`exercised`), its `cpu_ms` and `limit_ms`, and the run's `builds.ndjson` (SPEC-017 P5.6, B4) | a run's report | +| `test-budget` | 2026.11.21.1 | `mcpp test` states one budget of slots (`MTEST_BUDGET`) its programs' compilers take, and uses a budget its environment states (SPEC-017 O11) | `mtest run`, a runner that runs `mcpp test` | +| `test-jitter` | 2026.11.21.1 | the jitter a run states (`MTEST_JITTER`, `MTEST_JITTER_SEED`) reaches its programs and its record (SPEC-017 O12) | the jitter workflow | +| `test-fixtures` | 2026.11.21.1 | run fixtures are served for the programs whose cases use them (`MTEST_SERVE`, `MTEST_FIXTURE_`), and a run's `fixtures.ndjson` states them (SPEC-017 O6) | a framework's shared fixtures | ### How a tool runs mcpp for a project @@ -975,7 +975,7 @@ or `null`, `guide` or `null`, `repo`), `commands` (`list`, `one`, `records`, `debug`), `docs`, and `suggest` (the dev-dependency line and repository of mtest) when the package has no framework. -### `mcpp emit surface` and the records an invocation appends *(mcpp @NEXT@+)* +### `mcpp emit surface` and the records an invocation appends *(mcpp 2026.11.21.1+)* `mcpp emit surface [-o FILE]` writes mcpp's surface, one item per line, ` `, in the order `command`, `flag`, `key`, `reason`, each sorted by name: diff --git a/docs/92-release.md b/docs/92-release.md index 14e60dd6b..319cff013 100644 --- a/docs/92-release.md +++ b/docs/92-release.md @@ -71,7 +71,7 @@ merges the index PR after that (§3): GITCODE_TOKEN=... MIRROR_HOSTS=gtc bash .github/tools/mirror_res.sh mcpp # gtc on PATH ``` -**Promoted or built (@NEXT@+).** The four builds are +**Promoted or built (2026.11.21.1+).** The four builds are `release-build.yml`, which `ci.yml` also runs in its `full` tier; there its Linux unit and C++ e2e shards test the program of the Linux archive. A `full` run that passes leaves the artifact `verified-`. `find` reads the tag's diff --git a/docs/specs/manifest-semantics.md b/docs/specs/manifest-semantics.md index a6403516c..c3f2e2fef 100644 --- a/docs/specs/manifest-semantics.md +++ b/docs/specs/manifest-semantics.md @@ -680,7 +680,7 @@ mcpp 2026.9.26.2,#703)。** `obj/<包>/<摘要>/`,BMI 在 BMI 目录的 `<包>/<摘要>/`。摘要**必须**是包的配置的函数:声明的 标志与宏、展开后的包含目录、按 glob 的规则(以规则本身,而不以它匹配到的单元)、生效的特性、 语言标准,以及它所导入模块的各包的摘要;单元的集合与单元的输出路径**禁止**进入,版本与编译器 - 的构建号**禁止**进入。实例目录的摘要与缓存键的配置部分由同一个函数产生(@NEXT@,mcpp#986)。 + 的构建号**禁止**进入。实例目录的摘要与缓存键的配置部分由同一个函数产生(2026.11.21.1,mcpp#986)。 因此不同命令或选择对同一个包请求的不同 feature 各自占有一个实例,交替执行时不重新编译;特性 相同时 `mcpp build` 与 `mcpp test` 共用一个实例。实现以 `instances.tsv` 记录每个实例最近被 构建使用的时间;`mcpp clean --stale` 删除窗口内未被使用的实例,写出该配置构建图的规划所用的 @@ -707,7 +707,7 @@ mcpp 2026.9.26.2,#703)。** #714、#710;第 10 条的 `dialect_cxxflags` 为 mcpp 2026.9.28.1,#717;第 1 条的根包、第 10 条的 `[profile.]` 与第 11 条为 mcpp 2026.10.5.2,#771;第 7、8、10 条的工作空间层与第 12、13 条为 mcpp 2026.10.10.1,#785、#786;第 14 至 16 条与第 13 条的依赖位置为 mcpp 2026.10.11.1,#797; -第 17 条为 mcpp 2026.11.19.1,#953,其摘要为配置的函数与第 18 至 20 条为 @NEXT@,mcpp#986、 +第 17 条为 mcpp 2026.11.19.1,#953,其摘要为配置的函数与第 18 至 20 条为 2026.11.21.1,mcpp#986、 #802、#927)。** ## 10. 依赖的程序 diff --git a/docs/specs/test.md b/docs/specs/test.md index 001573890..d9303d238 100644 --- a/docs/specs/test.md +++ b/docs/specs/test.md @@ -7,7 +7,7 @@ | 状态 | 评审中 v2.5 | | 版本 | 2.5 | | 最后修改 | 2026-11-21 | -| 对应实现 | 逐条标注。B1–B4 与 O1、O2、O4–O7 mcpp >= 2026.11.8.1 至 2026.11.10.1(逐条);B5、B6、O3 的运行器一侧、O8 mcpp >= 2026.11.10.1;O9 由框架的工具实现(mtest >= 0.6.0);O10 mcpp >= 2026.11.13.1;P5 的 `step` 与 `exercise`、B4 的 `builds.ndjson` 与机器的字段、O5 在选择之下的时限、O6 的运行器一侧、O11、O12 mcpp >= @NEXT@,mtest >= 0.11.0 | +| 对应实现 | 逐条标注。B1–B4 与 O1、O2、O4–O7 mcpp >= 2026.11.8.1 至 2026.11.10.1(逐条);B5、B6、O3 的运行器一侧、O8 mcpp >= 2026.11.10.1;O9 由框架的工具实现(mtest >= 0.6.0);O10 mcpp >= 2026.11.13.1;P5 的 `step` 与 `exercise`、B4 的 `builds.ndjson` 与机器的字段、O5 在选择之下的时限、O6 的运行器一侧、O11、O12 mcpp >= 2026.11.21.1,mtest >= 0.11.0 | | 取代 | SPEC-017 v1.0(`test-runner.md`);mtest 测试协议版本 1(mtest `spec/protocol.md`)的规范正文;SPEC-004 中 `[test]` 的键 | | 相关规范 | [SPEC-004](manifest-semantics.md)(清单的平面与命名)、[SPEC-006](build-database.md)(构建数据库) | | 相关 issue | mcpp#914、mcpp#915、mcpp#917 | @@ -138,8 +138,8 @@ | `missing` | `id` | P4.1 | | `done` | `status`、`counts`(状态到数目的对象) | 末个 | -- **P5.6 步骤与用到的面(mtest >= 0.11.0;mcpp >= @NEXT@)。** 一个用例由若干步骤组成时(端到端用例运行的每条命令),程序**可以**为每个步骤写一个 `step` 事件;运行器把它们按到达顺序归入用例。`exercise` 列出用例用到的产品面的项;运行器把一个用例的各次 `exercise` 合并为一个集合。二者都不影响用例的结论。 -- **mcpp 的面(mcpp >= @NEXT@)。** `mcpp emit surface` 每行写出一项 `<种类> <名字>`:命令(`command emit surface`)、选项(`flag build --target`,全局选项为 `flag --quiet`)取自命令行的定义,键(`key build.cflags`,整张表为 `key toolchain`)取自清单的键登记表,原因(`reason unknown-target`)取自拒绝的原因标记;解析器自己加入的 `--help` 与 `--version` 不是项。环境变量 `MCPP_SURFACE_RECORD` 指明一个文件时,mcpp 的每个进程把它用到的项追加到该文件,每项每进程一次:所给的命令与选项、它读取的清单所陈述的登记表中的键、它记录的原因。测试 kit 把该文件转为用例的 `exercise` 事件。 +- **P5.6 步骤与用到的面(mtest >= 0.11.0;mcpp >= 2026.11.21.1)。** 一个用例由若干步骤组成时(端到端用例运行的每条命令),程序**可以**为每个步骤写一个 `step` 事件;运行器把它们按到达顺序归入用例。`exercise` 列出用例用到的产品面的项;运行器把一个用例的各次 `exercise` 合并为一个集合。二者都不影响用例的结论。 +- **mcpp 的面(mcpp >= 2026.11.21.1)。** `mcpp emit surface` 每行写出一项 `<种类> <名字>`:命令(`command emit surface`)、选项(`flag build --target`,全局选项为 `flag --quiet`)取自命令行的定义,键(`key build.cflags`,整张表为 `key toolchain`)取自清单的键登记表,原因(`reason unknown-target`)取自拒绝的原因标记;解析器自己加入的 `--help` 与 `--version` 不是项。环境变量 `MCPP_SURFACE_RECORD` 指明一个文件时,mcpp 的每个进程把它用到的项追加到该文件,每项每进程一次:所给的命令与选项、它读取的清单所陈述的登记表中的键、它记录的原因。测试 kit 把该文件转为用例的 `exercise` 事件。 - **P5.3** 用例的 `status` 为以下之一:`pass`、`fail`、`skip`(带 `reason`)、`xfail`(如预期地失败)、`xpass`(预期失败却通过;运行失败)、`invalid`(无法判定的测量;P6.3)。 - **P5.4** 运行器**必须**忽略未知字段与未知事件种类。运行器**必须**接受没有 `done` 的报告(程序崩溃):有 `begin` 而无 `end` 的用例失败,以程序的退出状态为原因。 - **P5.5** 程序的退出状态为零,当且仅当没有用例失败或意外通过,且没有被选用例缺失。 @@ -172,13 +172,13 @@ | 路径 | 内容 | |---|---| | `index.ndjson` | 每次运行一行:去掉程序与用例的运行记录 | -| `runs//run.json` | 运行记录:`schema`(`mtest.run/1`)、`id`、`started`、`finished`、`status`、`system`、`arch`、`machine`、`commit`、`set`、`selection`、`group`、`monitor`、`ms`、`jobs`、`counts`;`processors`、`memory`、`budget`、`jitter`(@NEXT@) | +| `runs//run.json` | 运行记录:`schema`(`mtest.run/1`)、`id`、`started`、`finished`、`status`、`system`、`arch`、`machine`、`commit`、`set`、`selection`、`group`、`monitor`、`ms`、`jobs`、`counts`;`processors`、`memory`、`budget`、`jitter`(2026.11.21.1) | | `runs//programs.ndjson` | 每个程序进程一行:`name`、`argv`、`status`、`exit`、`ms`、`started_ms`、`worker`、`select`、`resources`、`fixtures`,运行器构建它时的 `build_started_ms` 与 `build_ms`,`usage`(带 `source`:`program` 或 `engine`) | -| `runs//cases.ndjson` | 每个用例一行:`program`、`id`、`kind`、`status`、`ms`、`started_ms`、`reason`、`failures`、`metrics`、`bounds`、`tags`、`clauses`;`cpu_ms`、`steps`(`step` 事件的字段,`id` 除外)、`exercised`、`limit_ms`(用例所受的时限,O5;未知时省略)(@NEXT@) | -| `runs//builds.ndjson` | 运行器为这次运行所做的构建(@NEXT@):每条运行过的边一行,`build`(构建记录的标识)、`output`、`package`、`program`(该边所服务的测试程序,可为空)、`started_ms`、`ms`、`cpu_ms`、`peak_bytes`,单元缓存放置的边另有 `served`(`true`,没有编译器运行),为缓存提供的包复制单元的边另有 `staged`(`true`,同样没有编译器运行);由缓存提供的包一行,`build`、`package`、`cached`(`true`)、`units` | +| `runs//cases.ndjson` | 每个用例一行:`program`、`id`、`kind`、`status`、`ms`、`started_ms`、`reason`、`failures`、`metrics`、`bounds`、`tags`、`clauses`;`cpu_ms`、`steps`(`step` 事件的字段,`id` 除外)、`exercised`、`limit_ms`(用例所受的时限,O5;未知时省略)(2026.11.21.1) | +| `runs//builds.ndjson` | 运行器为这次运行所做的构建(2026.11.21.1):每条运行过的边一行,`build`(构建记录的标识)、`output`、`package`、`program`(该边所服务的测试程序,可为空)、`started_ms`、`ms`、`cpu_ms`、`peak_bytes`,单元缓存放置的边另有 `served`(`true`,没有编译器运行),为缓存提供的包复制单元的边另有 `staged`(`true`,同样没有编译器运行);由缓存提供的包一行,`build`、`package`、`cached`(`true`)、`units` | | `runs//fixtures.ndjson` | 每个运行夹具一行:`name`、`program`、`started_ms`、`ready_ms`、`ended_ms`、`status` | -- **P12.4 机器(@NEXT@)。** `machine` 是机器的类别:处理器的型号,系统不给出型号时为 `/<逻辑处理器数>`;`processors` 为逻辑处理器数,`memory` 为物理内存的字节数。比较不同运行的用时(O10 的估计、界限、趋势)时,读者**应当**按 `machine` 分组;没有同类的历史时取全部。处理器时间(`cpu_ms`)在同类机器之间可比,墙钟只在同一次运行之内可比。 +- **P12.4 机器(2026.11.21.1)。** `machine` 是机器的类别:处理器的型号,系统不给出型号时为 `/<逻辑处理器数>`;`processors` 为逻辑处理器数,`memory` 为物理内存的字节数。比较不同运行的用时(O10 的估计、界限、趋势)时,读者**应当**按 `machine` 分组;没有同类的历史时取全部。处理器时间(`cpu_ms`)在同类机器之间可比,墙钟只在同一次运行之内可比。 - **P12.2** 运行标识按时间排序。同一 `group` 的运行(一次 CI 运行的各作业)是一个整体的各部分,读者合并报告。 - **P12.3** 一次运行中程序、用例、构建与夹具的时间构成其工作的有向无环图;读者的关键路径是按实测时间加权的最长路径。 - **mcpp 的实现。** 存储位于 `[test] store`(缺省 `target/mtest`;`"none"` 不保存)。`run.json` 的 `commit` 与 `group` 在 CI 中取 `GITHUB_SHA` 与 `GITHUB_RUN_ID`;分片运行(O10)另写 `shard`(`I/N`)。程序进程一行在其为一部分时写 `part_of`(mcpp >= 2026.11.13.1)。mcpp 在一次构建中构建全部测试程序,不写逐个程序的 `build_started_ms`。 @@ -264,7 +264,7 @@ - **P7.2** `artifacts` 写出用例所读之物:`target:<名>`、`file:<路径>`、`program:<名>`;运行器据此按变更选择用例。 - **P7.3** 资源**可以**有容量:组件带来的池陈述同时持有它的用例数;运行器**禁止**超过容量。 - 为每个用例启动进程的运行器在用例的 `timeout` 时结束该进程;整体运行程序的运行器(`mcpp test`)运行其中每个用例,按报告判定。 -- **时限只陈述一次(@NEXT@)。** 用例的时限是它与其所属各级作用域声明的 `timeout` 中最小者。运行器选择一个用例或其下声明的用例(P3.4、P4)时,**必须**取这些声明,而不是运行的缺省;没有任何声明时取运行的缺省。 +- **时限只陈述一次(2026.11.21.1)。** 用例的时限是它与其所属各级作用域声明的 `timeout` 中最小者。运行器选择一个用例或其下声明的用例(P3.4、P4)时,**必须**取这些声明,而不是运行的缺省;没有任何声明时取运行的缺省。 ### O6. 组件与运行夹具(P11) @@ -272,7 +272,7 @@ - **P11.2** 运行夹具是整次运行的一个对象。用例在 `uses` 中写出它使用的夹具;运行器在第一个使用者开始前服务它,在最后一个结束后结束它,并以 `MTEST_FIXTURE_` 把端点交给使用者的程序。 - **P11.3** 运行器以 `MTEST_SERVE` 启动列出该夹具的程序来服务它,输入为管道,不带 `MTEST_PROTOCOL`。程序不运行用例:准备夹具,向输出写一行 `mtest-ready <端点>`,在输入结束时结束夹具。 - **P11.4** 未就绪的运行夹具是机器的失败:使用它的用例以状态 `infra` 结束,不运行。 -- **mcpp 的实现(已实现,mcpp >= @NEXT@)。** `mcpp test` 从程序的列出(`list` 模式)取得夹具与用例的 `uses`;所选用例使用夹具的程序在启动前取得端点,第一个取得者启动列出该夹具的程序(缺省等待就绪的时间为 `--timeout`),最后一个使用者结束后关闭其输入,等待同样长的时间后结束其进程单元。运行存储的 `fixtures.ndjson` 每个夹具一行,`programs.ndjson` 的 `fixtures` 写出程序用到的夹具;夹具未就绪时,使用它的程序不运行,记录的状态为 `infra`,`mcpp test` 以不运行的测试结束(退出码 2)。不经列出而运行的程序(运行器之后的程序)不取得夹具。 +- **mcpp 的实现(已实现,mcpp >= 2026.11.21.1)。** `mcpp test` 从程序的列出(`list` 模式)取得夹具与用例的 `uses`;所选用例使用夹具的程序在启动前取得端点,第一个取得者启动列出该夹具的程序(缺省等待就绪的时间为 `--timeout`),最后一个使用者结束后关闭其输入,等待同样长的时间后结束其进程单元。运行存储的 `fixtures.ndjson` 每个夹具一行,`programs.ndjson` 的 `fixtures` 写出程序用到的夹具;夹具未就绪时,使用它的程序不运行,记录的状态为 `infra`,`mcpp test` 以不运行的测试结束(退出码 2)。不经列出而运行的程序(运行器之后的程序)不取得夹具。 ### O7. 资源用量(P10) @@ -280,7 +280,7 @@ - **P10.2** 程度不为 `off` 的程序在 `done` 之前写一个 `usage` 事件。 - **P10.3** 在 `trace` 下,程序每 `MTEST_MONITOR_SAMPLE` 毫秒(缺省 50)采样一次,每个序列报告为一个指标,`samples` 为序列,`value` 为其最大样本,精度为 `sampled`。 - **P10.4** 运行器可以从其自己的进程测量不写 `usage` 的程序,并把结果记作引擎的。 -- **mcpp 的实现(mcpp >= 2026.11.10.1)。** `usage` 属于程序,进入程序记录,并以 `source = "program"` 进入 B4。mcpp >= @NEXT@ 以 `basic` 陈述程度,调用者的环境陈述 `MTEST_MONITOR` 时取其值;B4 的 `monitor` 为所陈述的程度。mcpp 不实现 P10.4:它的各个工作线程在同一进程中运行测试程序,系统对子进程的计量只按进程汇总,一个程序的用量无法从中分离。 +- **mcpp 的实现(mcpp >= 2026.11.10.1)。** `usage` 属于程序,进入程序记录,并以 `source = "program"` 进入 B4。mcpp >= 2026.11.21.1 以 `basic` 陈述程度,调用者的环境陈述 `MTEST_MONITOR` 时取其值;B4 的 `monitor` 为所陈述的程度。mcpp 不实现 P10.4:它的各个工作线程在同一进程中运行测试程序,系统对子进程的计量只按进程汇总,一个程序的用量无法从中分离。 ### O8. 输出归属(mcpp >= 2026.11.10.1) @@ -328,7 +328,7 @@ 4. **分片。** `--shard I/N` 把每个测试按估计从长到短(相同时按名字)分配给已分配估计之和最小的分片(相同时取编号最小者),只运行第 I 个分片的测试。分配**必须**只取决于存储与本次运行的测试列表。分片**禁止**把本组(`GITHUB_RUN_ID`,无则为空)的分片运行读作历史,使同一存储上先后启动的各分片读到相同的历史。 5. **失败重跑。** `--failed` 读取存储中最近的一次运行:状态为 `fail` 或 `build-fail` 的程序记录与状态为 `fail` 或 `xpass` 的用例。所选程序为这些记录的 `part_of`(无则为其名字);每个失败的程序都有失败用例时,只运行这些用例(与 `--case` 相同);否则整体运行这些程序。最近一次运行没有失败时,运行器不运行任何测试,退出码为 0。 -### O11. 预算(@NEXT@;mtest >= 0.11.0) +### O11. 预算(2026.11.21.1;mtest >= 0.11.0) 1. **形式。** 预算是一个目录,其中有与槽位数相同的文件 `slot-`(`i` 自 0 起)。持有一个文件的独占锁即持有一个槽位;进程结束时锁随之释放,因此崩溃不会遗失槽位。 2. **陈述。** 建立预算的运行器在它启动的程序的环境中陈述 `MTEST_BUDGET`(目录)与 `MTEST_BUDGET_HELD`(运行器为该程序持有的槽位文件)。程序以持有的槽位运行它的第一份并行工作;其余每一份并行工作(一个编译器、一个子进程)在开始前锁住一个空闲的槽位文件,结束后释放。没有空闲的槽位时等待。 @@ -336,7 +336,7 @@ 4. **不认识它的程序不受影响。** 预算只约束遵守它的程序;运行器以槽位数作为它自己的并行度。 5. **mcpp 的实现。** `mcpp test` 与 `mtest run` 建立预算,槽位数为其并行度;mcpp 的编译执行器在环境陈述了预算时,启动编译器之前先取预算的槽位,再取本次构建的槽位,并在取得之后、启动之前读取可用内存:低于 `[build] memory_threshold`(SPEC-004)时等待已运行的编译器结束,不结束它们。 -### O12. 抖动(@NEXT@;mtest >= 0.11.0) +### O12. 抖动(2026.11.21.1;mtest >= 0.11.0) 1. 运行器**可以**陈述 `MTEST_JITTER`(毫秒,上限)与 `MTEST_JITTER_SEED`(十进制)。程序在其声明的竞争点等待一段 [0, 上限] 内的随机时间,随机数由种子与竞争点的序号决定,同一种子给出同一序列。 2. 框架的竞争点:用例开始之前、共享对象构造之前;被测程序的竞争点由其自身声明(mcpp:取目录锁与发布产物之前)。 @@ -357,10 +357,10 @@ mcpp 以 feature(docs/50 §4)宣告它实现的条款;客户端按需门 | `test-frameworks` | B6 的 `capabilities` 与 `session` | 2026.11.10.1 | | `test-checks` | O3 的运行器一侧 | 2026.11.10.1 | | `test-output` | O8 | 2026.11.10.1 | -| `test-steps` | P5.6 的运行器一侧;B4 的 `steps`、`exercised` 与 `builds.ndjson` | @NEXT@ | -| `test-budget` | O11 | @NEXT@ | -| `test-jitter` | O12 的运行器一侧 | @NEXT@ | -| `test-fixtures` | O6 的运行器一侧;B4 的 `fixtures.ndjson` | @NEXT@ | +| `test-steps` | P5.6 的运行器一侧;B4 的 `steps`、`exercised` 与 `builds.ndjson` | 2026.11.21.1 | +| `test-budget` | O11 | 2026.11.21.1 | +| `test-jitter` | O12 的运行器一侧 | 2026.11.21.1 | +| `test-fixtures` | O6 的运行器一侧;B4 的 `fixtures.ndjson` | 2026.11.21.1 | 框架实现的可选能力由 B6 宣告,不由 mcpp 的 feature 宣告。 diff --git a/docs/zh/04-mcpp-toml.md b/docs/zh/04-mcpp-toml.md index e6f402d28..eb369713d 100644 --- a/docs/zh/04-mcpp-toml.md +++ b/docs/zh/04-mcpp-toml.md @@ -1101,7 +1101,7 @@ CI 上都跑通过。 一半的文件。MSVC 被留在一边不动——`/ifcOnly` 的开销与 `.ifc` 的 原子性都未经测量,两者任何一个猜错都是无声的。 -### 可用内存(`memory_threshold`)*(@NEXT@+)* +### 可用内存(`memory_threshold`)*(2026.11.21.1+)* ```toml [build] @@ -1654,7 +1654,7 @@ ldflags = [] - 内置 profile:`release`(-O2)/ `dev`、`debug`(-O0 -g)/ `dist` (-O3 + strip;**默认不启用 LTO**)。`[profile.<内置名>]` 可以整体 覆盖一个内置定义。 -- **`dependency_debug`(@NEXT@+)** 陈述来自包存储与 git 的依赖的调试信息:`true`(与 profile 的 `debug` 相同)、 +- **`dependency_debug`(2026.11.21.1+)** 陈述来自包存储与 git 的依赖的调试信息:`true`(与 profile 的 `debug` 相同)、 `false`(不带)、`"lines"`(只带行表:GCC 的 `-g1`、Clang 的 `-gline-tables-only`,由工具链行给出)。省略时,`dev` 与 `debug` 在其 `debug` 打开时取 `"lines"`,其余 profile 取其 `debug`。工作空间的成员与路径依赖保持 profile 的 `debug`。经由依赖的回溯仍给出文件与行号;在只依赖 mtest 的工程上实测,测试程序由 43.7 MB 降为 8.3 MB,构建由 diff --git a/docs/zh/08-testing.md b/docs/zh/08-testing.md index 54252d63e..ff63b83b2 100644 --- a/docs/zh/08-testing.md +++ b/docs/zh/08-testing.md @@ -114,18 +114,18 @@ note,说明有多少文件仍各自是一个程序;它不形成任何组时 mtest 自身的运行一样读取它们(Markdown 报告、最慢的用例、关键路径与调度)。在 CI 中 `commit` 与 `group` 取 `GITHUB_SHA` 与 `GITHUB_RUN_ID`。`[test] store = ""` 另指一个目录,相对项目;`"none"` 不保存。 -**运行记录的内容**(@NEXT@+,[SPEC-017](../specs/test.md) B4、P5.6、P12.4)。运行的 `run.json` 及其在 `index.ndjson` +**运行记录的内容**(2026.11.21.1+,[SPEC-017](../specs/test.md) B4、P5.6、P12.4)。运行的 `run.json` 及其在 `index.ndjson` 中的一行写出机器的类别(`machine`:处理器的型号)、`processors` 与 `memory`、运行陈述的监测程度(`MTEST_MONITOR`, 环境未陈述时为 `basic`:每个用例的处理器时间)与陈述了的抖动。用例的一行另有其处理器时间(`cpu_ms`)、步骤(端到端 用例运行的每条命令及其用时与状态)、它用到的产品面的项(`exercised`)与它所受的时限(`limit_ms`)。`builds.ndjson` 保存运行中的构建:每条运行的边一行,带包、用时、编译器的处理器时间与最大驻留集,以及是否由单元缓存放置 (`served`)或为缓存提供的包复制(`staged`);缓存提供的每个包一行。调度的估计优先取同类机器的运行。 -**机器的一份预算**(@NEXT@+,SPEC-017 O11)。`mcpp test` 建立每个作业一个槽位文件的目录,并告知其程序 +**机器的一份预算**(2026.11.21.1+,SPEC-017 O11)。`mcpp test` 建立每个作业一个槽位文件的目录,并告知其程序 (`MTEST_BUDGET`,以及工作者持有的槽位 `MTEST_BUDGET_HELD`);测试经由 mcpp 启动的编译器在启动前持有其中一个槽位, 因此构建工程的测试占用本次运行的作业数,而不另加自己的。运行在另一个运行器的预算之内时使用那一份。 -**用例的时限是其作用域陈述的时限**(@NEXT@+)。以 `--case` 选出的用例在其列出用例陈述的 `timeout` 之内运行, +**用例的时限是其作用域陈述的时限**(2026.11.21.1+)。以 `--case` 选出的用例在其列出用例陈述的 `timeout` 之内运行, 而不是运行的缺省时限。 由自身多个源文件编译而成的测试套件也可以是一个包:一个工作区成员,它的 @@ -267,7 +267,7 @@ test result ok. 7 passed; 0 failed; finished in 7.77s (plan 2.10s · programs 1. - 测试受其构建所用文件的变更影响:它自己的文件、它导入的模块单元及这些单元再导入的 单元,以及链接中不属于任何模块的单元; - 清单、锁文件、构建程序或计划中没有的源文件的变更影响所有测试; -- 头文件(@NEXT@+)影响由构建目录上次构建时读取它的单元构建的测试(ninja 的依赖日志与扫描的依赖文件所记录), +- 头文件(2026.11.21.1+)影响由构建目录上次构建时读取它的单元构建的测试(ninja 的依赖日志与扫描的依赖文件所记录), 这样的单元不属于测试自身时还影响运行本包程序的测试;工程的某个单元尚无此记录时(构建目录从未构建过它), 头文件影响所有测试; - 运行本包程序的测试(其构建所用的某个文件写出 `MCPP_BIN_`)受本包中任何不属于测试自身的源文件的 @@ -477,7 +477,7 @@ mcpp test --seed 42 # 用例中随机性的种子 `--message-format json` 的记录带有协议、用例及其失败([50](50-machine-output.md));`--print-launch` 为启动程序的调试器给出协议。 -**运行夹具**(@NEXT@+,SPEC-017 O6)。程序可以声明运行夹具——整次运行共享的一个对象:沙箱化的 home、服务器、 +**运行夹具**(2026.11.21.1+,SPEC-017 O6)。程序可以声明运行夹具——整次运行共享的一个对象:沙箱化的 home、服务器、 索引——其用例在 `uses` 中写出它(mtest 的 `run_fixture` 与 `uses{}`)。`mcpp test` 从程序的列出中读取二者。在所选 用例使用某个夹具的第一个程序启动之前,它以 `MTEST_SERVE` 指明该夹具、以管道为输入启动一个声明它的程序,并至多等待 `--timeout` 读到 `mtest-ready <端点>` 一行;每个使用者以 `MTEST_FIXTURE_` 收到端点。最后一个使用者结束后关闭 @@ -516,7 +516,7 @@ mcpp test unit/math --print-launch -- --gtest_filter=Math.Adds 构建该模式指定的一个测试,并打印 `mcpp test` 启动它的方式(`mcpp.launch` 信封,mcpp 2026.10.31.2+), 供调试器启动:其中包括运行器、运行所加的环境与 `--` 之后的参数。 -### 在 GitHub Actions 的作业中(@NEXT@+) +### 在 GitHub Actions 的作业中(2026.11.21.1+) ```yaml - uses: actions/checkout@v4 diff --git a/docs/zh/26-build-optimizations.md b/docs/zh/26-build-optimizations.md index 18a9ed9c5..d5529dff0 100644 --- a/docs/zh/26-build-optimizations.md +++ b/docs/zh/26-build-optimizations.md @@ -75,7 +75,7 @@ note hub.cppm is the whole of module hub, which 46 units import: its function b 重编译的起点是一个只有一个文件的模块、且导入它的单元不少于 5 个时给出这条说明,每个构建目录一次。 此外,替换或保留了 BMI 的构建说明是哪些([04](04-mcpp-toml.md),`bmi_cutoff`)。 -## home 的缓存放置的内容(@NEXT@+) +## home 的缓存放置的内容(2026.11.21.1+) 在各条目之外,`[build] cache` 为 home 的(缺省)时,home 的构建缓存(`build-cache/v1/`,[25](25-home-indices-and-store.md)) 为 home 中任何工程的下一次构建保存一次构建的产物: @@ -91,7 +91,7 @@ note hub.cppm is the whole of module hub, which 46 units import: its function b 在 `[cache] unused_days` 之内没有构建使用过的条目与其他条目一样被移除。 -## 构建的记录(@NEXT@+) +## 构建的记录(2026.11.21.1+) 构建记录它运行的每条边:所属的包、用时,编译另有编译器的处理器时间与最大驻留集,以及是否由单元缓存放置; 以及缓存为每个包提供的单元数。`MCPP_BUILD_RECORD=<文件>` 把该次调用每次构建的记录追加到该文件,每条边一行 diff --git a/docs/zh/50-machine-output.md b/docs/zh/50-machine-output.md index dfbd153df..b5fdbbc52 100644 --- a/docs/zh/50-machine-output.md +++ b/docs/zh/50-machine-output.md @@ -217,10 +217,10 @@ effects 采用具名的方式而不是一个 `destructive: true|false`,因为 | `test-frameworks` | 2026.11.10.1 | `mcpp test --guide` 写出每个框架的 `capabilities` 与 `session`(SPEC-017 B6) | 在会话与进程之间选择的 IDE | | `test-checks` | 2026.11.10.1 | 在 `test-protocol` 之上:失败的 `source` 与 `fatal`,用例的 `checks`(`held`、`failed`),按程序的报告(SPEC-017 O3) | IDE 的检查点列表 | | `test-output` | 2026.11.10.1 | 在 `test-case-stream` 之上:用例自己的输出 `output`:程序的 `output` 事件,或只运行该用例的程序的全部输出;出现在程序的 `cases` 中,来自事件时也出现在其 `end` 记录中(SPEC-017 O8) | IDE 中单个用例的输出 | -| `test-steps` | @NEXT@ | 在 `test-protocol` 之上:用例的 `steps` 与其用到的面的项(`exercised`)、其 `cpu_ms` 与 `limit_ms`,以及运行的 `builds.ndjson`(SPEC-017 P5.6、B4) | 运行的报告 | -| `test-budget` | @NEXT@ | `mcpp test` 陈述一份槽位的预算(`MTEST_BUDGET`),其程序的编译器从中取槽位;环境陈述了预算时使用那一份(SPEC-017 O11) | `mtest run`,运行 `mcpp test` 的运行器 | -| `test-jitter` | @NEXT@ | 运行陈述的抖动(`MTEST_JITTER`、`MTEST_JITTER_SEED`)到达其程序与记录(SPEC-017 O12) | 抖动的工作流 | -| `test-fixtures` | @NEXT@ | 为用例使用运行夹具的程序服务夹具(`MTEST_SERVE`、`MTEST_FIXTURE_`),运行的 `fixtures.ndjson` 写出它们(SPEC-017 O6) | 框架共享的夹具 | +| `test-steps` | 2026.11.21.1 | 在 `test-protocol` 之上:用例的 `steps` 与其用到的面的项(`exercised`)、其 `cpu_ms` 与 `limit_ms`,以及运行的 `builds.ndjson`(SPEC-017 P5.6、B4) | 运行的报告 | +| `test-budget` | 2026.11.21.1 | `mcpp test` 陈述一份槽位的预算(`MTEST_BUDGET`),其程序的编译器从中取槽位;环境陈述了预算时使用那一份(SPEC-017 O11) | `mtest run`,运行 `mcpp test` 的运行器 | +| `test-jitter` | 2026.11.21.1 | 运行陈述的抖动(`MTEST_JITTER`、`MTEST_JITTER_SEED`)到达其程序与记录(SPEC-017 O12) | 抖动的工作流 | +| `test-fixtures` | 2026.11.21.1 | 为用例使用运行夹具的程序服务夹具(`MTEST_SERVE`、`MTEST_FIXTURE_`),运行的 `fixtures.ndjson` 写出它们(SPEC-017 O6) | 框架共享的夹具 | ### 工具为项目运行 mcpp 的方式 @@ -866,7 +866,7 @@ mcpp test [pattern] [--workspace] --message-format json (每项 `package`、`version`、`entry`、`protocol` 或 `null`、`guide` 或 `null`、`repo`)、`commands`(`list`、 `one`、`records`、`debug`)、`docs`,以及包没有框架时的 `suggest`(mtest 的 dev-dependency 写法与仓库)。 -### `mcpp emit surface` 与一次调用追加的记录 *(mcpp @NEXT@+)* +### `mcpp emit surface` 与一次调用追加的记录 *(mcpp 2026.11.21.1+)* `mcpp emit surface [-o FILE]` 写出 mcpp 的面,每行一项 `<种类> <名字>`,按 `command`、`flag`、`key`、`reason` 的次序, 各自按名字排序: diff --git a/mcpp.toml b/mcpp.toml index 4dcd7c049..406ff6837 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -1,6 +1,6 @@ [package] name = "mcpp" -version = "2026.11.20.1" +version = "2026.11.21.1" description = "Modern C++ build & package management tool" license = "Apache-2.0" authors = ["mcpp-community"] diff --git a/modules/versioning/src/version.cppm b/modules/versioning/src/version.cppm index a25bd7795..fb3449589 100644 --- a/modules/versioning/src/version.cppm +++ b/modules/versioning/src/version.cppm @@ -31,6 +31,6 @@ import std; export namespace mcpp { -inline constexpr std::string_view MCPP_VERSION = "2026.11.20.1"; +inline constexpr std::string_view MCPP_VERSION = "2026.11.21.1"; } // namespace mcpp From 15e4cfaa640d63948531e3f33931b950efc884fa Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 07:01:35 +0900 Subject: [PATCH 32/38] CI: the macOS e2e suite runs on the newest image at a checkpoint (full), as the macOS job does --- .github/workflows/ci-macos-e2e.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-macos-e2e.yml b/.github/workflows/ci-macos-e2e.yml index 9fa3e0be4..d355b341c 100644 --- a/.github/workflows/ci-macos-e2e.yml +++ b/.github/workflows/ci-macos-e2e.yml @@ -41,7 +41,8 @@ jobs: strategy: fail-fast: false matrix: - image: [macos-15, xcode-27] + # The newest image is a checkpoint's (`full`), not each push's (design 2026-11-20 §7.2). + image: ${{ fromJSON(inputs.set == 'full' && '["macos-15", "xcode-27"]' || '["macos-15"]') }} shard: ${{ fromJSON(inputs.shards) }} runs-on: ${{ matrix.image }} timeout-minutes: 60 From 268cbf888cd5fa0951cdb9372f07f33ac5cc5b16 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 07:37:19 +0900 Subject: [PATCH 33/38] CI: e2e-cases prints the run's records when a program fails beside its passing cases --- .github/actions/e2e-cases/action.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/actions/e2e-cases/action.yml b/.github/actions/e2e-cases/action.yml index fc072e44b..ca0fd64d6 100644 --- a/.github/actions/e2e-cases/action.yml +++ b/.github/actions/e2e-cases/action.yml @@ -88,4 +88,13 @@ runs: for c in "${cases[@]}"; do tr -d '\r' < "$log" | grep -qxF " ok $c" || { echo "::error::$c did not pass (a skip or a failure; its line above)"; status=1; } done + # A program can fail beside cases that passed (its exit status, a fixture, a case it did + # not report): the run's records say which, and the job keeps no store. + if [ "$status" != 0 ]; then + run=$(ls -t target/mtest/runs 2>/dev/null | head -1) + for f in programs cases fixtures; do + [ -n "$run" ] && [ -f "target/mtest/runs/$run/$f.ndjson" ] || continue + echo "::group::the run's $f.ndjson"; cat "target/mtest/runs/$run/$f.ndjson"; echo "::endgroup::" + done + fi exit "$status" From beb724370d5bbf3c7b3033923ccf7c9d552b320d Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 09:11:04 +0900 Subject: [PATCH 34/38] mcpp test: a worker that waits for a slot gives up once no test is left (a run inside another runner's budget ran out its time) --- src/build/execute.cppm | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/build/execute.cppm b/src/build/execute.cppm index 2d0cdef3d..33f888763 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -5760,6 +5760,12 @@ static int test_run_member(TestBuild& tb, const TestOptions& testOpts, if (auto l = mcpplibs::os::fs::lock::acquire(f, mcpplibs::os::fs::lock::kind::exclusive, false)) return {mcpp::home::spelling(f), std::move(*l)}; if (slots.empty()) return {{}, std::nullopt}; + // A worker that waits for a slot while the others have taken every test has none + // to run. Inside another runner's budget every other slot can stay held until + // this run ends: the first worker ran the whole list on the held slot, and the + // others waited for a slot forever (measured: a `mcpp test` inside a case of a + // four-worker shard ran out its 600 s). + if (next.load() >= list.size()) return {{}, std::nullopt}; std::this_thread::sleep_for(std::chrono::milliseconds(10)); } }; From 1705f22df397268d048046b40cdce16eb5b54a25 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 09:11:04 +0900 Subject: [PATCH 35/38] The unit cache records what a Clang compile read: its depfile names the BMIs it imports (-fmodule-file-deps), and the files are read after the compile wrote them --- modules/toolchain-model/src/model.cppm | 7 +++++++ modules/unit-cache/src/unit_cache.cppm | 3 +++ src/build/ninja_backend.cppm | 8 +++++--- src/build/schedule/detach_codegen.cppm | 14 +++++++++++--- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/modules/toolchain-model/src/model.cppm b/modules/toolchain-model/src/model.cppm index a227c65d5..33021f00e 100644 --- a/modules/toolchain-model/src/model.cppm +++ b/modules/toolchain-model/src/model.cppm @@ -434,6 +434,12 @@ struct BmiTraits { std::string_view moduleFileUsePrefix; // "" | " -fmodule-file=" | " /reference " std::string_view moduleOutputPrefix; // "" | " -fmodule-output=" | " /ifcOutput " std::string_view bmiSearchPrefix; // "" | " -fprebuilt-module-path=" | " /ifcSearchDir " + // Makes the compile's depfile name the BMIs it read beside its headers. Clang finds an + // imported BMI through the search path, and its depfile names none without it: the home's + // unit cache recorded no import, and placed an object compiled against an earlier BMI + // (measured: an inline function of an edited module kept its old value). GCC's mapper names + // the imports; cl.exe's compiles are not cached. + std::string_view moduleFileDepsFlag; // "" | " -fmodule-file-deps" | "" // How this compiler is TOLD that a translation unit is a module interface. // Emitted UNCONDITIONALLY on every module compile — mcpp never asks // "does this driver recognize this extension?". @@ -733,6 +739,7 @@ BmiTraits bmi_traits(const Toolchain& tc) { .moduleFileUsePrefix = " -fmodule-file=", .moduleOutputPrefix = " -fmodule-output=", .bmiSearchPrefix = " -fprebuilt-module-path=", + .moduleFileDepsFlag = " -fmodule-file-deps", .moduleInterfaceLangFlag = " -x c++-module", .moduleImplLangFlag = " -x c++", .moduleImplPartitionLangFlag = " -x c++-module", diff --git a/modules/unit-cache/src/unit_cache.cppm b/modules/unit-cache/src/unit_cache.cppm index 331560b04..3e04bd30c 100644 --- a/modules/unit-cache/src/unit_cache.cppm +++ b/modules/unit-cache/src/unit_cache.cppm @@ -505,6 +505,9 @@ void Cache::keep(const Unit& u) { if (text.empty()) return forget(u); read = depfile_prerequisites(text); } + // A compile reads its source at least: a unit that names no file read states nothing a hit + // could be checked against. + if (read.empty()) return forget(u); const auto object = u.object.is_absolute() ? u.object : u.cwd / u.object; const auto bmi = u.bmi.empty() ? fs::path{} : (u.bmi.is_absolute() ? u.bmi : u.cwd / u.bmi); // A BMI the unit provides: where it is published, else where the compiler wrote it (phase 1 diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index 3c399163e..fd7042661 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -2271,10 +2271,12 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, const bool gnuDepfile = dial.inputsRecord == std::string_view("make"); const bool needsGnuModuleFilter = gnuDepfile && plan.toolchain.compiler == mcpp::toolchain::CompilerId::GCC; + // A C++ compile's depfile names the BMIs it read too, where the compiler needs to be told + // (BmiTraits::moduleFileDepsFlag): ninja and the unit cache then read the imports from it. const std::string mmd_flag = - gnuDepfile ? (needsGnuModuleFilter ? "-MMD -MF $out.d.raw " - : "-MMD -MF $out.d ") - : ""; + gnuDepfile ? (needsGnuModuleFilter ? std::string("-MMD -MF $out.d.raw ") + : std::format("-MMD -MF $out.d{} ", traits.moduleFileDepsFlag)) + : std::string{}; const std::string mmd_filter = (needsGnuModuleFilter && !mcpplibs::os::is_windows) ? " && awk 'NR==1{print;next} /^[^ ]/{exit} {print}' " diff --git a/src/build/schedule/detach_codegen.cppm b/src/build/schedule/detach_codegen.cppm index a15ff2ee7..475eb0126 100644 --- a/src/build/schedule/detach_codegen.cppm +++ b/src/build/schedule/detach_codegen.cppm @@ -959,14 +959,22 @@ void record_unit(const CompileRequest& req, const CompileRequest& actual, const uc::Unit u{uc::split_command(req.command), std::filesystem::current_path(), req.object, req.bmi}; const auto shape = uc::shape_of(u.argv); if (!shape) return; - std::vector reads; - for (const auto& p : uc::depfile_prerequisites(read(req.depFrom))) reads.emplace_back(p); + // THE FILES READ, WHERE THEY ARE KNOWN BEFORE THE COMPILER RUNS: the scan's depfile + // (`.ddi.dep`) and the BMIs its record imports. A unit whose depfile is the compile's own + // (Clang's `-MMD -MF .d`, which ninja takes into its log and removes) leaves them unset, + // and keeping it reads the depfile this compile writes. Read here, that depfile did not exist + // yet: the entry named no file read, and an edit of a header the module's purview includes + // was placed from it unchanged (measured: case 118 printed 41 after the edit to 42). if (auto ddi = req.depFrom; ddi.extension() == ".dep") { + const auto text = read(req.depFrom); + if (text.empty()) return; + std::vector reads; + for (const auto& p : uc::depfile_prerequisites(text)) reads.emplace_back(p); ddi.replace_extension(); const auto mapper = shape->mapper.empty() ? std::string{} : read(shape->mapper); for (auto& b : uc::imported_bmis(read(ddi), mapper, u.cwd)) reads.push_back(std::move(b)); + u.reads = std::move(reads); } - u.reads = std::move(reads); const auto words = uc::split_command(actual.command); for (std::size_t i = 0; i + 1 < words.size(); ++i) if (words[i] == "-o") u.placeObject = words[i + 1]; From 159edcc2ed7d39fcf0c3a3b911affefb0f468811 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 09:11:04 +0900 Subject: [PATCH 36/38] A build program run in a packing pass runs in the project: its output is a function of the pack's request --- docs/26-build-optimizations.md | 4 +++- docs/specs/manifest-semantics.md | 3 ++- docs/zh/26-build-optimizations.md | 3 ++- src/build/prepare/features.cpp | 10 ++++++++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/26-build-optimizations.md b/docs/26-build-optimizations.md index f55d340f0..84de0de9c 100644 --- a/docs/26-build-optimizations.md +++ b/docs/26-build-optimizations.md @@ -105,7 +105,9 @@ is the home's (the default): git commit, a path dependency's content digest -- runs its build program into `build-cache/v1/out/@//out` (`MCPP_OUT_DIR`), the key being its configuration without that directory: a second project takes the outputs, and runs nothing - ([SPEC-004 §9](specs/manifest-semantics.md) item 18). + ([SPEC-004 §9](specs/manifest-semantics.md) item 18). A program run by `mcpp pack + --pkg-format` runs in the project: what it writes depends on the pack's request and the + member's staged tree. Entries no build used within `[cache] unused_days` are removed as other entries are. diff --git a/docs/specs/manifest-semantics.md b/docs/specs/manifest-semantics.md index c3f2e2fef..267c546c4 100644 --- a/docs/specs/manifest-semantics.md +++ b/docs/specs/manifest-semantics.md @@ -690,7 +690,8 @@ mcpp 2026.9.26.2,#703)。** (索引的版本、git 的提交、路径依赖的内容摘要)的构建程序,其输出目录**必须**位于 `/build-cache/v1/out/<包>@<版本>/<键>/out`,键为该包不含输出目录一项的配置键;同一 身份的输出由所有工程共用,运行时持有该目录的文件锁。构建程序收到 `MCPP_OUT_DIR` 并可以把它 - 写进生成的文件,因此位置是内容的一部分,由身份决定。 + 写进生成的文件,因此位置是内容的一部分,由身份决定。打包的一遍(`mcpp pack` 的 `--pkg-format`)中运行的 + 构建程序**不**在其中:它写出的是打包请求与成员的暂存树的函数,二者不在键中,其输出目录在工程中。 19. **`[build] memory_threshold`。** 编译执行器启动编译器之前读取系统的可用内存:低于该值时等待 已运行的编译器结束,不结束它们;低于该值满一分钟时说明并启动编译器。值为大小(`"2GiB"`、`"512MiB"`、`"1.5GB"`)或字节数,`0` 不等待。取值按 §2 的四层:`MCPP_MEMORY_THRESHOLD`、清单的 `[build] memory_threshold`、机器的 diff --git a/docs/zh/26-build-optimizations.md b/docs/zh/26-build-optimizations.md index d5529dff0..5e3b438ff 100644 --- a/docs/zh/26-build-optimizations.md +++ b/docs/zh/26-build-optimizations.md @@ -87,7 +87,8 @@ note hub.cppm is the whole of module hub, which 46 units import: its function b 288 次编译全部被放置。cl.exe 的编译不进入缓存。 - **构建程序的输出。** 来源被命名的包——索引的版本、git 的提交、路径依赖的内容摘要——其构建程序在 `build-cache/v1/out/<包>@<版本>/<键>/out`(`MCPP_OUT_DIR`)中运行,键为不含该目录的配置:第二个工程取用这些输出, - 不再运行([SPEC-004 §9](../specs/manifest-semantics.md) 第 18 项)。 + 不再运行([SPEC-004 §9](../specs/manifest-semantics.md) 第 18 项)。`mcpp pack --pkg-format` 运行的构建程序在工程中运行: + 它写出的内容取决于打包请求与成员的暂存树。 在 `[cache] unused_days` 之内没有构建使用过的条目与其他条目一样被移除。 diff --git a/src/build/prepare/features.cpp b/src/build/prepare/features.cpp index d34367bea..9c2353f16 100644 --- a/src/build/prepare/features.cpp +++ b/src/build/prepare/features.cpp @@ -2050,9 +2050,15 @@ static std::expected step6_dependency_build_programs(PrepareS // its contract, so every project that builds it in one // configuration states the same include directories and shares // its cache entry. Held while the program runs: a second project - // waits, then replays the directives the first recorded. + // waits, then replays the directives the first recorded. A program + // run in a packing pass is not: what it writes is a function of the + // pack's request and the member's staged tree, which its contract + // does not hold, and two + // projects packing through one provider read each other's archive + // (measured: e2e 868 found its provider's package in the home). std::optional outLock; - if (program && sources_named(state, i) && !state.overrides.describes()) { + if (program && sources_named(state, i) && !state.overrides.describes() && bpEnv.packFormat.empty() + && bpEnv.packStageDir.empty()) { auto label = std::format("{}@{}", pkg.manifest.package.name, pkg.manifest.package.version); for (auto& c : label) if (c == '/' || c == '\\' || c == ':') c = '_'; From d91550eb6dcbce79461d50e9e981366a84a51e9c Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 09:11:04 +0900 Subject: [PATCH 37/38] memory_threshold enters the grouping key of a plan's members, as jobs does --- src/project.cppm | 1 + 1 file changed, 1 insertion(+) diff --git a/src/project.cppm b/src/project.cppm index c504f4cb2..b6ec9a807 100644 --- a/src/project.cppm +++ b/src/project.cppm @@ -1141,6 +1141,7 @@ export std::string root_position_key( field("bmi_schedule", b.bmiSchedule); field("private_fragment", b.privateFragment); field("jobs", b.jobs); + field("memory_threshold", b.memoryThreshold); field("cache", b.cacheMode); field("allow_host_libs", b.allowHostLibs ? "1" : "0"); field("platform_dependencies", b.platformDependencies); From efc91d11221acca439de3aa5a1ace97daf5bf648 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 21 Nov 2026 09:11:11 +0900 Subject: [PATCH 38/38] The kit initializes a case's inherited home with the real xlings; the cases that read the old home's payloads, outputs in the project, a BMI count and one entry of gc read the new ones; the cap test runs outside the run's budget --- .github/tools/waits.baseline | 2 +- CHANGELOG.md | 8 +++++--- tests/e2e/actions/generated.cpp | 5 ++++- tests/e2e/build-program/deps.cpp | 6 ++++-- tests/e2e/cdb/emit.cpp | 7 +++++++ tests/e2e/index/local.cpp | 10 ++++++++-- tests/e2e/modules/programs.cpp | 6 ++++-- tests/e2e/runtime/deploy.cpp | 6 ++++-- tests/e2e/self/homes.cpp | 24 ++++++++++++++++-------- tests/e2e/test/fast.cpp | 10 ++++++++-- tests/e2e/toolchain/management.cpp | 4 ++++ tests/support/src/support.cpp | 13 +++++++++++++ tests/support/src/support.cppm | 7 ++++--- tests/unit/test_detach_status.cpp | 4 +++- tests/unit/test_repo_consistency.cpp | 7 ++++--- 15 files changed, 89 insertions(+), 30 deletions(-) diff --git a/.github/tools/waits.baseline b/.github/tools/waits.baseline index 7951f5e69..947ef956c 100644 --- a/.github/tools/waits.baseline +++ b/.github/tools/waits.baseline @@ -27,7 +27,7 @@ tests/e2e/run/environment.cpp 1 tests/e2e/run/program.cpp 1 tests/e2e/runtime/binding.cpp 2 tests/e2e/self/homes.cpp 2 -tests/e2e/test/fast.cpp 3 +tests/e2e/test/fast.cpp 5 tests/e2e/test/report.cpp 1 tests/e2e/test/streams.cpp 1 tests/e2e/toolchain/llvm.cpp 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index d66d996e1..f0d663729 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,13 +13,15 @@ files an earlier compile of that key read still hold their contents, its object and BMI are placed and no compiler starts. A BMI is named by the key of the compile that made it. A fresh checkout of mcpp on a filled cache built in 4.1 s instead of 164 s, its 288 compiles placed. - The split schedule places its units from the cache too. cl.exe's compiles are not cached. + The split schedule places its units from the cache too. cl.exe's compiles are not cached. A + Clang compile is given `-fmodule-file-deps`, so its depfile names the BMIs it imports beside + its headers, and ninja and the cache read them from it. - **A cacheable package's build program runs in the home** (mcpp#986, mcpp#927, SPEC-004 §9 item 18). A package whose source is named -- an index's version, a git commit, a path dependency's content digest -- runs its build program into `build-cache/v1/out/@//out`, keyed by its configuration, and a second - project takes the outputs. A path enters a cache key by its role (``, ``, - ``), never as written. + project takes the outputs; a program run by `mcpp pack --pkg-format` runs in the project. A + path enters a cache key by its role (``, ``, ``), never as written. - **`[profile.] dependency_debug`** (mcpp#927, SPEC-004 §9 item 20): the debug information of the dependencies from the package store and git, `true`, `false` or `"lines"`. Omitted, `dev` and `debug` take `"lines"` while their `debug` is on. A test program whose only diff --git a/tests/e2e/actions/generated.cpp b/tests/e2e/actions/generated.cpp index aa7b678c2..e1efdf396 100644 --- a/tests/e2e/actions/generated.cpp +++ b/tests/e2e/actions/generated.cpp @@ -189,7 +189,10 @@ int main() { {"src/main.cpp", "#include \nimport proto;\nint main() { std::printf(\"ANSWER=%d\\n\", proto_answer()); }\n"}}}; expect(fatal(e2e::succeeded(app.mcpp({"build"})))) << "a dependency whose action generates its only header did not build"; expect(eq(s::last_line(app.mcpp({"run"}).out), std::string{"ANSWER=42"})) << "expected ANSWER=42 from the generated header"; - const auto hdr = s::find_file(tmp, "proto_generated.h"); + // A path dependency's build program runs into the home's output directory (SPEC-004 §9 + // item 18), else into the project's. + auto hdr = s::find_file(s::shared_home() / "build-cache/v1/out/proto@0.1.0", "proto_generated.h"); + if (!hdr) hdr = s::find_file(tmp, "proto_generated.h"); expect(fatal(hdr.has_value())) << "no proto_generated.h anywhere"; expect(fatal(fs::file_size(*hdr) > 0)) << hdr->string() << " is empty — the placeholder, not the generator's output"; expect(e2e::contains(s::read_file(*hdr), "PROTO_ANSWER")) << hdr->string() << " is not what gen.sh writes"; diff --git a/tests/e2e/build-program/deps.cpp b/tests/e2e/build-program/deps.cpp index 863b8e2b1..93dbd847d 100644 --- a/tests/e2e/build-program/deps.cpp +++ b/tests/e2e/build-program/deps.cpp @@ -792,8 +792,10 @@ int main() { // runs: they are installed first (library deps belong in mcpp.deps; only hook-time tools // are xpm deps). A fake xlings records the order of the installs. const auto home = s::case_home(s::home::inherited); - const auto seed = home / "registry/subos/default/.xlings.json"; - expect(fatal(fs::is_regular_file(seed))) << "inherited toolchain has no default SubOS runtime contract"; + // A real runtime contract for the fake xlings to report: the user's, which the case's home + // names as an upstream; the case's own subos holds one only after an install. + const auto seed = s::shared_home() / "registry/subos/default/.xlings.json"; + expect(fatal(fs::is_regular_file(seed))) << "the shared home has no default SubOS runtime contract"; const auto tmp = s::case_dir(); auto package = [](std::string_view name, std::string_view description, std::string_view source, std::string_view target, std::string_view deps) { diff --git a/tests/e2e/cdb/emit.cpp b/tests/e2e/cdb/emit.cpp index 55332f48f..3abdf17f9 100644 --- a/tests/e2e/cdb/emit.cpp +++ b/tests/e2e/cdb/emit.cpp @@ -204,6 +204,10 @@ E2E emit{"emit", s::home_fixture, [] { expect(fatal(!s::find_files(work, R"(resolution\.json)").empty())) << "L: no resolution record under the work directory " << work; expect(fatal(compiled(work).empty())) << "L: the planning pass wrote compile or link outputs: " << compiled(work).front(); const auto cold = s::case_home(s::home::inherited); + // The toolchain is the home's before it goes offline: it installs it from its upstreams by + // link, and an install is a download to an offline home. Its std cache stays cold. + expect(fatal(e2e::succeeded(s::mcpp({"toolchain", "install", "gcc", s::gcc_version()}, cold, {.where = s::home::inherited})))) + << "L: the cold home did not install its toolchain"; const s::options cold_home{.where = s::home::inherited, .env = {{"MCPP_OFFLINE", "1"}, {"MCPP_COMPILE_COMMANDS", "compile_commands.json"}}}; const auto ce = hello.mcpp({"emit", "build-database", "--format", "json"}, cold_home); expect(fatal(e2e::succeeded(ce))) << "L: emit in a home with a cold std cache: " << ce.err; @@ -414,6 +418,9 @@ int main() { const auto tmp = s::case_dir() / "measured"; const auto out = s::case_dir() / "out"; const auto home = s::case_home(s::home::inherited); + // The toolchain, installed by the real xlings from the upstreams before the stub stands in. + expect(fatal(e2e::succeeded(s::mcpp({"toolchain", "install", "gcc", s::gcc_version()}, s::case_dir(), {.where = s::home::inherited})))) + << "the case's home did not install its toolchain"; s::write_executable(out / "bin/xlings", R"(#!/usr/bin/env bash [[ "${1:-}" == "interface" && "${2:-}" == "--version" ]] && { echo '{"protocol_version":"1.13","version":"9999.1.1.1","capabilities":["install_packages"],"home_layouts":[2],"features":["index_dir","index_store","upstreams","payload_status","sync_index","payload_currency","binding_state","network_diagnostics","download_socket_budgets","read_only_queries"]}'; exit 0; } echo "$*" >> "${STUB_LOG:?}" diff --git a/tests/e2e/index/local.cpp b/tests/e2e/index/local.cpp index fe9ae5207..a0df840ab 100644 --- a/tests/e2e/index/local.cpp +++ b/tests/e2e/index/local.cpp @@ -79,8 +79,10 @@ E2E local{"local", s::home_fixture, [] { // compat:compat.cfg@1.0.0, refreshing no builtin index and writing no xim index. const auto tmp = s::case_dir(); const auto home = s::case_home(s::home::inherited); - const auto seed = home / "registry/subos/default/.xlings.json"; - expect(fatal(fs::is_regular_file(seed))) << "inherited toolchain has no default SubOS runtime contract"; + // A real runtime contract for the fake xlings to report: the user's, which the case's home + // names as an upstream; the case's own subos holds one only after an install. + const auto seed = s::shared_home() / "registry/subos/default/.xlings.json"; + expect(fatal(fs::is_regular_file(seed))) << "the shared home has no default SubOS runtime contract"; s::project index(tmp / "local-index", {{"pkgs/c/compat.cfg.lua", lua_package("compat.cfg", "1.0.0", "Namespaced local path index package", c_lib_segment("src/*.c", "cfg"))}}); const auto indices = std::format("[indices]\ncompat = {{ path = \"{}\" }}\n\n[dependencies.compat]\ncfg = \"1.0.0\"\n", index.dir().generic_string()); s::project app(tmp / "project/app", { @@ -280,6 +282,10 @@ if [[ "${1:-}" == "install" ]]; then fi exit 0 )SH"); + // The sandbox's tools before the fake xlings stands in: the case's home installs them from + // its upstreams on its first build, which the fake cannot do. + auto primer = s::project::scaffold("primer"); + expect(fatal(e2e::succeeded(primer.mcpp({"build"}, inherited())))) << "the case's home did not bootstrap"; fake_xlings_config(home, tmp / "fake-bin/xlings", home / "registry"); const std::string payload = ".mcpp/.xlings/data/xpkgs/compat-x-compat.stale/1.0.0"; s::project app(tmp / "project/app", { diff --git a/tests/e2e/modules/programs.cpp b/tests/e2e/modules/programs.cpp index ccc0c99a3..a4a7321bd 100644 --- a/tests/e2e/modules/programs.cpp +++ b/tests/e2e/modules/programs.cpp @@ -101,7 +101,8 @@ main = "main.cpp" expect(fatal(output_of(*app) == "1")) << "A: the app does not print its own module's value"; expect(fatal(output_of(*upd) == "2")) << "A: the updater does not print its own module's value"; std::size_t nq = 0, nf = 0; - for (const auto& f : s::find_files(a.at("app/target"), R"(boost\..*)")) { + // The BMIs alone: a BMI's producer key beside it (`.mcpp-unit`, the unit cache) is not one. + for (const auto& f : s::find_files(a.at("app/target"), R"(boost\.(gcm|pcm|ifc)$)")) { const auto g = f.generic_string(); if (std::regex_search(g, std::regex{R"(\.cache/.*updater/[0-9a-f]{8}/boost\.)"})) ++nq; if (std::regex_search(g, std::regex{R"(\.cache/boost\.)"})) ++nf; @@ -256,7 +257,8 @@ main = "main.cpp" std::this_thread::sleep_for(std::chrono::seconds{2}); r = s::mcpp({"cache", "gc", "--older-than", "1s"}, nowhere, quiet_home); expect(fatal(e2e::succeeded(r))) << "C: mcpp cache gc failed\n" << s::text(r); - expect(fatal(e2e::contains(s::text(r), "Collected 1 entries"))) << "C: gc did not collect the entry of the bundled module\n" << s::text(r); + // The entry of the bundled module, beside the units the cache kept of the two builds. + expect(fatal(s::has_line(s::text(r), R"(Collected _engine/mcpp-build-module@)"))) << "C: gc did not collect the entry of the bundled module\n" << s::text(r); fs::last_write_time(alpha.at("build.mcpp"), fs::file_time_type::clock::now()); alpha.write("build.mcpp", alpha.read("build.mcpp") + "// edited\n"); r = alpha.mcpp({"build"}, home); diff --git a/tests/e2e/runtime/deploy.cpp b/tests/e2e/runtime/deploy.cpp index e225e8aa6..ea52b43bf 100644 --- a/tests/e2e/runtime/deploy.cpp +++ b/tests/e2e/runtime/deploy.cpp @@ -212,7 +212,9 @@ int main() {{ for (auto& w : split(inputs_text, ' ')) if (!w.empty()) inputs.push_back(std::move(w)); expect(fatal(inputs.size() == 2u)) << "expected exactly two inputs on the stage_file edge, got: " << inputs_text; - static const std::regex dep_src{R"(\.build-mcpp/out/[0-9a-f]*/[a-z.]*dep/gen/shared\.bin$)"}; + // A dependency whose source is named runs its build program into the home's output + // directory (SPEC-004 §9 item 18); one built in the project, into the project's. + static const std::regex dep_src{R"((\.build-mcpp/out/[0-9a-f]*/[a-z.]*dep|build-cache/v1/out/[a-z.]*dep@[^/]+/[0-9a-f]+/out)/gen/shared\.bin$)"}; static const std::regex app_src{R"(\.build-mcpp/out/[0-9a-f]*/app/gen/shared\.bin$)"}; expect(fatal(std::ranges::any_of(inputs, [](const auto& w) { return std::regex_search(w, dep_src); }))) << "the edge does not list the dependency's source: " << inputs_text; expect(fatal(std::ranges::any_of(inputs, [](const auto& w) { return std::regex_search(w, app_src); }))) << "the edge does not also list the app's own source: " << inputs_text; @@ -225,7 +227,7 @@ int main() {{ const auto log = s::text(r); expect(fatal(!r)) << "different bytes for one destination were accepted: " << log; expect(fatal(e2e::contains(log, "disagree"))) << "the refusal does not say the sources disagree: " << log; - expect(fatal(s::has_line(log, R"(\.build-mcpp/out/[0-9a-f]*/[a-z.]*dep/gen/shared\.bin)"))) << "the refusal does not name the dependency's source: " << log; + expect(fatal(s::has_line(log, R"((\.build-mcpp/out/[0-9a-f]*/[a-z.]*dep|build-cache/v1/out/[a-z.]*dep@[^/]+/[0-9a-f]+/out)/gen/shared\.bin)"))) << "the refusal does not name the dependency's source: " << log; expect(fatal(s::has_line(log, R"(\.build-mcpp/out/[0-9a-f]*/app/gen/shared\.bin)"))) << "the refusal does not name the app's own source: " << log; expect(fatal(e2e::contains(log, "shared.bin"))) << "the refusal does not name the destination: " << log; }; diff --git a/tests/e2e/self/homes.cpp b/tests/e2e/self/homes.cpp index 9ddc9e429..348a7fdcd 100644 --- a/tests/e2e/self/homes.cpp +++ b/tests/e2e/self/homes.cpp @@ -39,16 +39,24 @@ E2E homes{"homes", s::home_fixture, [] { expect(fatal(fs::is_regular_file(vendored))) << "the first command did not vendor xlings at " << vendored << ": " << s::text(r); r = s::mcpp({"self", "init"}, tmp, o); expect(fatal(!s::contains_nocase(r.err, "cannot find the path specified"))) << "A: the version probe reached a shell that could not open its redirect: " << r.err; + // Any ninja binary stands in: the case's home's, else the shared home's, which the case's + // home reads as an upstream and so holds it in its own store only once it installed one. + std::vector candidates; + for (const auto& [base, pattern] : {std::pair{home, "*/ninja"}, std::pair{home, "*/bin/ninja"}, + std::pair{s::shared_home(), "*/ninja"}, std::pair{s::shared_home(), "*/bin/ninja"}}) + std::ranges::copy(s::glob(base / "registry/data/xpkgs/xim-x-ninja", std::string(pattern) + (s::on_windows() ? ".exe" : "")), + std::back_inserter(candidates)); std::optional ninja; - for (std::string_view pattern : {"*/ninja", "*/bin/ninja"}) { - for (const auto& c : s::glob(home / "registry/data/xpkgs/xim-x-ninja", std::string(pattern) + (s::on_windows() ? ".exe" : ""))) - if (fs::is_regular_file(c)) { ninja = c; break; } - if (ninja) break; - } - expect(fatal(ninja.has_value())) << "B: no ninja binary to stand in for an older xlings under " << home / "registry"; + if (auto c = std::ranges::find_if(candidates, [](const fs::path& p) { return fs::is_regular_file(p); }); c != candidates.end()) + ninja = *c; + expect(fatal(ninja.has_value())) << "B: no ninja binary to stand in for an older xlings under " << home / "registry" << " or " << s::shared_home() / "registry"; const auto answer = s::run(*ninja, {"--version"}).out; - const auto older = answer.substr(0, answer.find('\n')); - expect(fatal(std::regex_match(older, std::regex{R"([0-9].*\..*)"}))) << "B: the stand-in " << *ninja << " answered '" << older << "', not a dotted version"; + // The version mcpp reads from the answer: its leading dotted numbers (ninja 1.12.1-mcpp.1 + // answers `1.12.1-mcpp.1`, and is read as 1.12.1). + std::smatch dotted; + const auto first = answer.substr(0, answer.find('\n')); + expect(fatal(std::regex_search(first, dotted, std::regex{R"(^[0-9]+(\.[0-9]+)+)"}))) << "B: the stand-in " << *ninja << " answered '" << first << "', not a dotted version"; + const auto older = dotted.str(); const auto real = tmp / "real" / s::exe("xlings"); fs::create_directories(real.parent_path()); fs::copy_file(vendored, real); diff --git a/tests/e2e/test/fast.cpp b/tests/e2e/test/fast.cpp index 714fad101..1a971da9d 100644 --- a/tests/e2e/test/fast.cpp +++ b/tests/e2e/test/fast.cpp @@ -129,8 +129,11 @@ UTest two{{"two", [] {{ expect(eq(2, {})); }}}}; expect(s::lacks(s::text(one), "Resolved mcpplibs.")) << "a selection was planned again"; expect(e2e::contains(s::text(one), "cases: 1 passed, 0 failed")); // An edited test is built, and its result is the edit's. + // A second of the file system's clock, not a time moved forward: the edit must be newer + // than the last build's test.ninja and older than the next one's, which the record is + // compared with (a time two seconds ahead kept the next, unchanged run planning again). + std::this_thread::sleep_for(1s); write_cases(3); - s::touch(p.at("tests/cases.cpp")); auto edited = p.mcpp({"test"}); expect(edited.started && edited.status != 0) << "the edited test passed"; expect(e2e::contains(s::text(edited), "cases: 1 passed, 1 failed")); @@ -139,8 +142,11 @@ UTest two{{"two", [] {{ expect(eq(2, {})); }}}}; expect(s::lacks(s::text(again), "Resolved mcpplibs.")) << "an unchanged failing project was planned again"; expect(e2e::contains(s::text(again), "cases: 1 passed, 1 failed")) << "the record's run lost the failure"; // A test added is planned and run; one removed is no longer run. + // As above, not a time moved forward: the edit must be newer + // than the last build's test.ninja and older than the next one's, which the record is + // compared with (a time two seconds ahead kept the next, unchanged run planning again). + std::this_thread::sleep_for(1s); write_cases(2); - s::touch(p.at("tests/cases.cpp")); p.write("tests/extra.cpp", "int main() { return 0; }\n"); auto added = p.mcpp({"test"}); expect(fatal(e2e::succeeded(added))); diff --git a/tests/e2e/toolchain/management.cpp b/tests/e2e/toolchain/management.cpp index bb3e5a600..0aa27861a 100644 --- a/tests/e2e/toolchain/management.cpp +++ b/tests/e2e/toolchain/management.cpp @@ -177,6 +177,10 @@ main = "src/main.cpp" auto r = s::mcpp({"toolchain", "list"}, s::case_dir(), o1); expect(fatal(e2e::succeeded(r))) << s::text(r); expect(fatal(in_order(s::text(r), "gcc", "16.1.0"))) << "gcc 16.1.0 missing from list: " << s::text(r); + // The list reads the upstreams; a default names a toolchain of this home, which installs it + // from them by link. + r = s::mcpp({"toolchain", "install", "gcc", "16.1.0"}, s::case_dir(), o1); + expect(fatal(e2e::succeeded(r))) << "install 'gcc 16.1.0' from the upstream failed: " << s::text(r); r = s::mcpp({"toolchain", "default", "gcc", "16"}, s::case_dir(), o1); expect(fatal(e2e::succeeded(r))) << "default 'gcc 16' failed: " << s::text(r); expect(fatal(e2e::contains(s::text(r), "gcc@16.1.0"))) << "default 'gcc 16' didn't resolve to 16.1.0: " << s::text(r); diff --git a/tests/support/src/support.cpp b/tests/support/src/support.cpp index cb5394f02..5ebe41831 100644 --- a/tests/support/src/support.cpp +++ b/tests/support/src/support.cpp @@ -452,6 +452,19 @@ void inherit(const fs::path& h, bool whole) { name_upstreams(h); if (fs::is_directory(user / "bin", ec)) link_or_copy(user / "bin", h / "bin"); + // THE SANDBOX, ONCE, WITH THE REAL XLINGS: its runtime contract and the tools every build reads + // (patchelf, ninja), installed from the upstreams by link. A copy of the user's subos and links + // to the payloads gave the home these before; without them a case that runs offline, or that + // puts a stand-in xlings in place, found an uninitialized provider and could install nothing. + // Measured: under a second, and the user's homes unwritten. + std::vector>> env{{"MCPP_HOME", h.string()}, + {"MCPP_VERBOSE", std::nullopt}, + {"MCPP_OFFLINE", std::nullopt}}; + const auto init = e2e::run({.argv = {mcpp_path(), "self", "init"}, .cwd = h, .env = env, + .timeout = std::chrono::minutes{5}}); + if (!init) + std::println(std::cerr, "the case's home {} was not initialized (status {}):\n{}{}", h.string(), init.status, + init.out, init.err); } } // namespace diff --git a/tests/support/src/support.cppm b/tests/support/src/support.cppm index 686603785..d7f85b653 100644 --- a/tests/support/src/support.cppm +++ b/tests/support/src/support.cppm @@ -138,9 +138,10 @@ std::string mcpp_path(); // shared: the machine's home. fresh: an empty home of the case. linked: a home of the case whose // `registry` is a link to the shared home's (its toolchains, not its build cache or state). -// inherited: tests/e2e/_inherit_toolchain.sh's home, a home of the case holding links to each -// payload of the user's homes, their indices, `subos` and `bin`, and a copy of `config.toml`. -// payloads: inherited without config.toml and subos (MCPP_INHERIT_CONFIG=0 MCPP_INHERIT_SUBOS=0). +// inherited: tests/e2e/_inherit_toolchain.sh's home, a home of the case whose `[xlings] upstream` +// names the user's homes, with links to their indices and `bin` and a copy of `config.toml`; it +// installs what it needs from the upstreams by link, into its own subos, and writes nothing there. +// payloads: inherited without the copy of config.toml (MCPP_INHERIT_CONFIG=0). // registry: a home of the case whose `[xlings] home` is the shared home's registry (its toolchains, // indices and sandbox) and whose config.toml is its own. A case that sets the default toolchain // (`mcpp toolchain default`) runs here: the cases that read the machine's default run beside it diff --git a/tests/unit/test_detach_status.cpp b/tests/unit/test_detach_status.cpp index 8f3ced644..f8bc4c073 100644 --- a/tests/unit/test_detach_status.cpp +++ b/tests/unit/test_detach_status.cpp @@ -219,8 +219,10 @@ UTest actual_gcc_mixed_capacity_uses_owned_units{"DetachLifetime/ActualGccMixedC expect(fatal(fs::is_regular_file(engine))); Slot temporary; const auto helper=fs::current_path()/"tests/e2e/helpers/check_shared_compiler_cap.py"; + // The cap is what is measured, not the run's budget (SPEC-017 O11): inside a runner's budget + // the builds' compilers share the slots of the other workers, and the peak is what is free. const auto result=mcpp::run::execute({"python3",helper.string(),engine,(temporary.dir/"managed-gcc").string(), "--owner-driver",mcpp::invocation::self_exe_path().string()}, - {.total=std::chrono::seconds(120)}); + {.unset={"MTEST_BUDGET","MTEST_BUDGET_HELD"},.total=std::chrono::seconds(120)}); expect(eq(result.exit_code, 0))<