diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a11453..5477107 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,16 @@ pull request. ## Unreleased +### Features + +- **Compiling to a module.** `compilerlib::compileToModule`, from + `compilerlib/compile_module.hpp`, gives the module that `compilerlib::compile` writes as text + or bitcode, created in an `llvm::LLVMContext` of the host, instead of writing it for the host + to read it back. Its arguments are those of an LLVM IR compilation; the output file is not + written, and the other files the arguments ask for, such as a dependency file, are. The + context keeps its own setting of value names, except with LLVM 16, where a context that + discards them keeps them after a compilation that names values (#265, #263). + ### Fixes - **A child of `fork()` and the vtable diagnostics, on Linux.** With `--ct-vtable-diag`, a @@ -180,6 +190,10 @@ pull request. - When `sbrk()` fails in the sbrk test of the full allocation table, the test reports `errno`, the program break and the mapping above it, `RLIMIT_DATA`, and the memory of the process and of the system (#239, #233). +- Unit tests check that `compileToModule` gives the module that `compile` writes as text, at -O0 + and -O2, instrumented or not, and the module of a bitcode compilation, whose dependency file is + written and output file is not; that it fails on a compilation to an object and on a broken + source; and that the context keeps its setting of value names (#265, #263). - The Docker tests build and test the image in a job per architecture, each on a runner of its own architecture: arm64 on `ubuntu-24.04-arm` instead of under QEMU on an amd64 runner. Under QEMU's user-mode emulation, `sbrk()` failed now and then in the sbrk test of the full @@ -208,6 +222,7 @@ pull request. ### Documentation +- The README describes compiling to a module with `compileToModule` (#265, #263). - `NOTICE` dates the copyright from 2025, when the code began, credits the CoreTrace contributors alongside Hugo Payet and links the repository. It keeps only these attribution notices, without the license header meant for source files (#248). diff --git a/README.md b/README.md index 8c0121b..1eb32b9 100644 --- a/README.md +++ b/README.md @@ -498,3 +498,27 @@ expects, and waits for it: a host can compile from a thread with a small stack, secondary thread on macOS, whose stack is 512 KiB (#204). The time-trace profiler a host runs on its own thread, started with `llvm::timeTraceProfilerInitialize`, therefore does not record the compilation. With `-ftime-trace`, each job writes its own trace file, as clang does. + +### Compiling to a module + +`compilerlib::compileToModule`, from `compilerlib/compile_module.hpp`, gives the module that +`compilerlib::compile` writes as text or bitcode, created in an `llvm::LLVMContext` of the host, +instead of writing it for the host to read it back: + +```cpp +#include + +llvm::LLVMContext context; +compilerlib::ModuleResult result = + compilerlib::compileToModule({"-g", "-c", "-emit-llvm", "unit.cpp"}, context); +if (result.success) + analyze(*result.module); +``` + +Its arguments are those of an LLVM IR compilation, `-S -emit-llvm` or `-c -emit-llvm`. The output +file is not written; the other files the arguments ask for, such as a dependency file, are. With +`instrument`, the module is the instrumented one. Code generation sets options of the context, as +it does with clang, except whether value names are discarded: the context keeps its own setting, +which applies to the modules read into it later. With LLVM 16, a context that discards names keeps +them after a compilation that names values, since LLVM 16 cannot destroy those values in a context +that discards names. diff --git a/include/compilerlib/compile_module.hpp b/include/compilerlib/compile_module.hpp new file mode 100644 index 0000000..4e052a0 --- /dev/null +++ b/include/compilerlib/compile_module.hpp @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 +#ifndef COMPILERLIB_COMPILE_MODULE_HPP +#define COMPILERLIB_COMPILE_MODULE_HPP + +#include "compilerlib/attributes.hpp" + +#include +#include + +#include +#include +#include + +namespace compilerlib +{ + + struct ModuleResult + { + bool success = false; + std::string diagnostics; + /// The module, in the context compileToModule was given; null when the compilation + /// failed. + std::unique_ptr module; + }; + + /// Compiles one source to the module that compile() writes, created in `context`, for a + /// host that works on the module itself: it is neither written as text or bitcode nor read + /// back. `args` are those of an LLVM IR compilation, `-S -emit-llvm` or `-c -emit-llvm`; + /// the output file is not written, and the other files the arguments ask for, such as a + /// dependency file, are. With `instrument`, the module is the instrumented one. + /// + /// Code generation sets options of `context` as it does with clang, except whether value + /// names are discarded, which is restored: modules read into `context` later keep theirs. + /// With LLVM 16, a context that discards names keeps them after a compilation that names + /// values, which LLVM 16 could not destroy in a context that discards names. + /// No other thread may use `context` during the compilation. + CT_NODISCARD ModuleResult compileToModule(const std::vector& args, + llvm::LLVMContext& context, bool instrument = false); + +} // namespace compilerlib + +#endif // COMPILERLIB_COMPILE_MODULE_HPP diff --git a/src/compilerlib/clang_compat.hpp b/src/compilerlib/clang_compat.hpp index c47168a..a286ef1 100644 --- a/src/compilerlib/clang_compat.hpp +++ b/src/compilerlib/clang_compat.hpp @@ -68,6 +68,19 @@ namespace compilerlib::clang_compat return file; } + // Removes from `context` the record that Clang's frontend action set up on it, for + // -fsave-optimization-record: before LLVM 22, the action closes the file and leaves the + // streamer, which would write into it; LLVM 22's handle removes the streamer itself. + inline void releaseFrontendOptimizationRecord(llvm::LLVMContext& context) + { +#if LLVM_VERSION_MAJOR >= 22 + (void)context; +#else + context.setLLVMRemarkStreamer(nullptr); + context.setMainRemarkStreamer(nullptr); +#endif + } + // Removes the record's streamer from `context`, which must not write into `file` once // it is released. inline void releaseOptimizationRecord(llvm::LLVMContext& context, RemarkFile& file) diff --git a/src/compilerlib/compiler.cpp b/src/compilerlib/compiler.cpp index 025a8c1..da4984a 100644 --- a/src/compilerlib/compiler.cpp +++ b/src/compilerlib/compiler.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 #include "compilerlib/compiler.h" +#include "compilerlib/compile_module.hpp" #include "compilerlib/attributes.hpp" #include "compilerlib/toolchain.hpp" @@ -75,6 +76,9 @@ namespace compilerlib class LocatingCodegenAction : public clang::EmitLLVMOnlyAction { public: + // The module is created in the given context, or in one of the action's own. + using clang::EmitLLVMOnlyAction::EmitLLVMOnlyAction; + emit::FunctionLocations takeFunctionLocations(void) { return std::move(functions_); @@ -223,6 +227,11 @@ namespace compilerlib std::string clang_sysroot; RuntimeArchives runtime_archives; llvm::IntrusiveRefCntPtr fs; + // compileToModule's context, where code generation creates the module, which the + // compilation then leaves in `module`; null for compile(), whose actions create + // contexts of their own. + llvm::LLVMContext* module_context = nullptr; + std::unique_ptr module; // Options of the driver's diagnostics engine, which must outlive it. clang_compat::DiagnosticOptionsStorage driver_diagnostic_options; DiagsSaver dc; @@ -699,7 +708,103 @@ namespace compilerlib return result; } + // compileToModule's job: the module stays in ctx_.module. + CompileResult runToModule(const clang::driver::Command& job) + { + CompileResult result; + if (!runOnClangStack([&] { result = runToModuleOnThisThread(job); })) + { + ctx_.module.reset(); + return {false, "compilation crashed", {}}; + } + return result; + } + private: + // What code generation changes in compileToModule's context, which the host + // keeps: whether value names are discarded, which would apply to the modules the + // host reads into it later, and the optimization record of + // -fsave-optimization-record, whose file is closed once the job ends. + class ModuleContextScope + { + public: + ModuleContextScope(llvm::LLVMContext& context, bool optimizationRecord) + : context_(context), discardValueNames_(context.shouldDiscardValueNames()), + optimizationRecord_(optimizationRecord) + { + } + + ModuleContextScope(const ModuleContextScope&) = delete; + ModuleContextScope& operator=(const ModuleContextScope&) = delete; + + ~ModuleContextScope() + { +#if LLVM_VERSION_MAJOR < 17 + // LLVM 16 cannot destroy a value with a name in a context that discards + // names: Value::setName("") then leaves the name of a function's argument + // in the function's symbol table, which frees it a second time. A context + // that discards names keeps them, once a compilation gave its module some. + if (!discardValueNames_ || context_.shouldDiscardValueNames()) +#endif + context_.setDiscardValueNames(discardValueNames_); + if (optimizationRecord_) + clang_compat::releaseFrontendOptimizationRecord(context_); + } + + private: + llvm::LLVMContext& context_; + bool discardValueNames_; + bool optimizationRecord_; + }; + + CompileResult runToModuleOnThisThread(const clang::driver::Command& job) + { + auto ci = makeCompilerInstance(job.getArguments()); + if (!ci) + return {false, "failed to create compiler instance", {}}; + switch (ci->getFrontendOpts().ProgramAction) + { + case clang::frontend::EmitLLVM: + case clang::frontend::EmitBC: + break; + default: + return {false, + mergeDiagnostics(ctx_.driver_diagnostics, + "a module needs an LLVM IR compilation: pass -S " + "-emit-llvm or -c -emit-llvm"), + {}}; + } + Cc1TimeTrace trace(*ci, job.getExecutable()); + + std::string error; + auto keepModule = [&](std::unique_ptr module) -> bool + { + if (ctx_.instrument && !instrument(*module, error)) + return false; + ctx_.module = std::move(module); + return true; + }; + bool ok = false; + { + ModuleContextScope scope(*ctx_.module_context, + !ci->getCodeGenOpts().OptRecordFile.empty()); + ok = ctx_.runtimeConfig.optnone_enabled + ? runCodegenWithModule< + frontend::OptNoneAction>( + *ci, keepModule, error) + : runCodegenWithModule(*ci, keepModule, + error); + } + if (!ok) + { + ctx_.module.reset(); + return {false, mergeDiagnostics(ctx_.driver_diagnostics, error), {}}; + } + // Before the diagnostics are collected: writing the trace may report one. + trace.finish(); + return {true, mergeDiagnostics(ctx_.driver_diagnostics, ctx_.dc.message), {}}; + } + CT_NODISCARD bool runInstrumentedOnThisThread(const clang::driver::Command& job, std::string& error) { @@ -974,7 +1079,8 @@ namespace compilerlib options.OptRecordFile.clear(); } - Action action; + // compileToModule's context, or one of the action's own. + Action action(ctx_.module_context); resetDiagnostics(); if (!ci.ExecuteAction(action)) { @@ -1294,72 +1400,98 @@ namespace compilerlib return {true, mergeDiagnostics(ctx.driver_diagnostics, cc1_diags), {}}; } - } // namespace - - CT_NODISCARD CompileResult compile(const std::vector& input_args, OutputMode mode, - bool instrument) - { - CompileContext ctx(input_args, mode, instrument); - ArgBuilder argBuilder(ctx); - std::string error; + // The compilation `ctx` describes, from its arguments to its result. + CT_NODISCARD CompileResult runCompilation(CompileContext& ctx) + { + ArgBuilder argBuilder(ctx); + std::string error; - if (!argBuilder.build(error)) - return {false, std::move(error), {}}; + if (!argBuilder.build(error)) + return {false, std::move(error), {}}; - auto diags = createDriverDiagnostics(ctx); + auto diags = createDriverDiagnostics(ctx); - DriverSession driver(ctx, *diags); - std::unique_ptr comp = driver.buildCompilation(error); - if (!comp) - { - if (error.empty()) - error = ctx.dc.message.empty() ? "failed to build compilation" - : std::move(ctx.dc.message); - return {false, std::move(error), {}}; - } + DriverSession driver(ctx, *diags); + std::unique_ptr comp = driver.buildCompilation(error); + if (!comp) + { + if (error.empty()) + error = ctx.dc.message.empty() ? "failed to build compilation" + : std::move(ctx.dc.message); + return {false, std::move(error), {}}; + } - if (comp->getJobs().empty()) - return { - false, ctx.dc.message.empty() ? "no jobs to run" : std::move(ctx.dc.message), {}}; - - // Instrumented LTO is not supported (#163): the bitcode would be written without its - // summary, and the link-time pipeline would optimize the instrumented code again, - // which no test covers. The driver resolves -flto, -flto= and -fno-lto. - if (instrument && clang_compat::usesLto(*comp)) - return {false, - "error: ct: --instrument does not support -flto: compile instrumented code " - "without link-time optimization", - {}}; + if (comp->getJobs().empty()) + return {false, + ctx.dc.message.empty() ? "no jobs to run" : std::move(ctx.dc.message), + {}}; + + // Instrumented LTO is not supported (#163): the bitcode would be written without its + // summary, and the link-time pipeline would optimize the instrumented code again, + // which no test covers. The driver resolves -flto, -flto= and -fno-lto. + if (ctx.instrument && clang_compat::usesLto(*comp)) + return {false, + "error: ct: --instrument does not support -flto: compile instrumented code " + "without link-time optimization", + {}}; + + if (!ctx.instrument && ctx.mode == OutputMode::ToFile && + !ctx.runtimeConfig.optnone_enabled) + return runNonInstrumentedCompilation(ctx, driver, *comp); + + JobPlan plan = buildJobPlan(*comp); + if (!validateJobPlan(plan, ctx.mode, error)) + return {false, std::move(error), {}}; + + if (!ctx.dc.message.empty()) + { + ctx.driver_diagnostics = std::move(ctx.dc.message); + ctx.dc.message.clear(); + ctx.dc.os.flush(); + } - if (!instrument && mode == OutputMode::ToFile && !ctx.runtimeConfig.optnone_enabled) - return runNonInstrumentedCompilation(ctx, driver, *comp); + Cc1Runner cc1(ctx, *diags); + if (ctx.module_context) + return cc1.runToModule(*plan.cc1Jobs.front()); + if (ctx.mode == OutputMode::ToMemory || ctx.mode == OutputMode::ToMemoryBitcode) + return cc1.runSingle(*plan.cc1Jobs.front()); - JobPlan plan = buildJobPlan(*comp); - if (!validateJobPlan(plan, mode, error)) - return {false, std::move(error), {}}; + if (plan.cc1Jobs.empty()) + { + Linker linker(hasArg(ctx.filtered_args, "-v")); + if (!linker.run(plan.otherJobs, error)) + return {false, mergeDiagnostics(ctx.driver_diagnostics, error), {}}; + return {true, mergeDiagnostics(ctx.driver_diagnostics, {}), {}}; + } - if (!ctx.dc.message.empty()) - { - ctx.driver_diagnostics = std::move(ctx.dc.message); - ctx.dc.message.clear(); - ctx.dc.os.flush(); + if (ctx.instrument) + return runInstrumentedToFile(ctx, cc1, plan, error); + return runPlainToFile(ctx, cc1, plan, error); } - Cc1Runner cc1(ctx, *diags); - if (mode == OutputMode::ToMemory || mode == OutputMode::ToMemoryBitcode) - return cc1.runSingle(*plan.cc1Jobs.front()); + } // namespace - if (plan.cc1Jobs.empty()) - { - Linker linker(hasArg(ctx.filtered_args, "-v")); - if (!linker.run(plan.otherJobs, error)) - return {false, mergeDiagnostics(ctx.driver_diagnostics, error), {}}; - return {true, mergeDiagnostics(ctx.driver_diagnostics, {}), {}}; - } + CT_NODISCARD CompileResult compile(const std::vector& input_args, OutputMode mode, + bool instrument) + { + CompileContext ctx(input_args, mode, instrument); + return runCompilation(ctx); + } - if (instrument) - return runInstrumentedToFile(ctx, cc1, plan, error); - return runPlainToFile(ctx, cc1, plan, error); + CT_NODISCARD ModuleResult compileToModule(const std::vector& args, + llvm::LLVMContext& context, bool instrument) + { + // The modes of in-memory output: a single -cc1 job, and -c kept, which a bitcode + // compilation needs. + CompileContext ctx(args, OutputMode::ToMemoryBitcode, instrument); + ctx.module_context = &context; + CompileResult result = runCompilation(ctx); + ModuleResult moduleResult; + moduleResult.success = result.success && ctx.module != nullptr; + moduleResult.diagnostics = std::move(result.diagnostics); + if (moduleResult.success) + moduleResult.module = std::move(ctx.module); + return moduleResult; } } // namespace compilerlib diff --git a/test/unit/compile_test.cpp b/test/unit/compile_test.cpp index 4c6e741..e9e3fd0 100644 --- a/test/unit/compile_test.cpp +++ b/test/unit/compile_test.cpp @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +#include "compilerlib/compile_module.hpp" #include "compilerlib/compiler.h" #include @@ -31,6 +32,7 @@ #include #include #include +#include #include #ifndef _WIN32 @@ -687,6 +689,153 @@ int main(void) EXPECT_NE(memory.diagnostics.find("missing"), std::string::npos) << memory.diagnostics; } + std::string printed(const llvm::Module& module) + { + std::string text; + llvm::raw_string_ostream stream(text); + module.print(stream, nullptr); + stream.flush(); + return text; + } + + // The text of `module` as the IR parser gives it back: a module read from text does not + // keep the order of each value's uses, which orders the predecessors listed for a block. + std::string normalizedText(const llvm::Module& module) + { + llvm::LLVMContext context; + llvm::SMDiagnostic error; + std::unique_ptr parsed = + llvm::parseAssemblyString(printed(module), error, context); + if (!parsed) + return "unreadable module: " + error.getMessage().str(); + parsed->setModuleIdentifier(module.getModuleIdentifier()); + return printed(*parsed); + } + + // An allocation, a loop and accesses for the instrumentation to rewrite. + constexpr const char* kModuleUnit = R"(#include + +int sum(int count) +{ + int* values = malloc(sizeof(int) * count); + int total = 0; + for (int i = 0; i < count; ++i) + { + values[i] = i; + total += values[i]; + } + free(values); + return total; +} +)"; + + // compileToModule gives the module that compile() writes as text, without the text. + class ModuleOutputTest : public CompileTest, + public ::testing::WithParamInterface> + { + }; + + TEST_P(ModuleOutputTest, IsTheTextualOutput) + { + const auto [instrument, optimization] = GetParam(); + const std::vector args{optimization, "-g", "-S", "-emit-llvm", + writeSource("unit.c", kModuleUnit)}; + compilerlib::CompileResult text = + compilerlib::compile(args, compilerlib::OutputMode::ToMemory, instrument); + ASSERT_TRUE(text.success) << text.diagnostics; + + llvm::LLVMContext context; + compilerlib::ModuleResult result = compilerlib::compileToModule(args, context, instrument); + ASSERT_TRUE(result.success) << result.diagnostics; + ASSERT_NE(result.module, nullptr); + EXPECT_EQ(&result.module->getContext(), &context); + EXPECT_EQ(result.diagnostics, text.diagnostics); + + llvm::LLVMContext textContext; + llvm::SMDiagnostic error; + std::unique_ptr parsed = + llvm::parseAssemblyString(text.llvmIR, error, textContext); + ASSERT_NE(parsed, nullptr) << error.getMessage().str(); + parsed->setModuleIdentifier(result.module->getModuleIdentifier()); + EXPECT_EQ(normalizedText(*result.module), normalizedText(*parsed)); + } + + INSTANTIATE_TEST_SUITE_P(Modes, ModuleOutputTest, + ::testing::Combine(::testing::Bool(), ::testing::Values("-O0", "-O2")), + [](const ::testing::TestParamInfo>& info) + { + return std::string(std::get<0>(info.param) ? "Instrumented" + : "Plain") + + (std::string(std::get<1>(info.param)) == "-O0" ? "O0" + : "O2"); + }); + + // A bitcode compilation gives its module too. Its output file is not written; the + // dependency file it asks for is. + TEST_F(CompileTest, ModuleOfABitcodeCompilation) + { + llvm::LLVMContext context; + compilerlib::ModuleResult result = + compilerlib::compileToModule({"-emit-llvm", "-c", writeSource("unit.c", kBitcodeUnit), + "-o", path("unit.bc"), "-MD", "-MF", path("unit.d")}, + context); + ASSERT_TRUE(result.success) << result.diagnostics; + ASSERT_NE(result.module, nullptr); + EXPECT_NE(result.module->getFunction("main"), nullptr); + EXPECT_FALSE(fs::exists(path("unit.bc"))); + EXPECT_NE(readFile(path("unit.d")).find("unit.c"), std::string::npos); + } + + TEST_F(CompileTest, ModuleNeedsAnIRCompilation) + { + llvm::LLVMContext context; + compilerlib::ModuleResult result = compilerlib::compileToModule( + {"-c", writeSource("unit.c", kBitcodeUnit), "-o", path("unit.o")}, context); + EXPECT_FALSE(result.success); + EXPECT_EQ(result.module, nullptr); + EXPECT_NE(result.diagnostics.find("-emit-llvm"), std::string::npos) << result.diagnostics; + EXPECT_FALSE(fs::exists(path("unit.o"))); + } + + TEST_F(CompileTest, ModuleReportsCompileErrors) + { + llvm::LLVMContext context; + compilerlib::ModuleResult result = compilerlib::compileToModule( + {"-S", "-emit-llvm", writeSource("broken.c", "int main(void) { return missing; }\n")}, + context); + EXPECT_FALSE(result.success); + EXPECT_EQ(result.module, nullptr); + EXPECT_NE(result.diagnostics.find("missing"), std::string::npos) << result.diagnostics; + } + + // Code generation names the values as -f[no-]discard-value-names asks, and the context + // keeps its own setting, which applies to the modules read into it later. + TEST_F(CompileTest, ModuleLeavesTheValueNamesSettingOfTheContext) + { + const std::string source = writeSource("unit.c", kBitcodeUnit); + for (bool discard : {false, true}) + { + llvm::LLVMContext context; + context.setDiscardValueNames(discard); + for (bool names : {false, true}) + { + compilerlib::ModuleResult result = compilerlib::compileToModule( + {names ? "-fno-discard-value-names" : "-fdiscard-value-names", "-S", + "-emit-llvm", source}, + context); + ASSERT_TRUE(result.success) << result.diagnostics; + EXPECT_EQ(printed(*result.module).find("%call ") != std::string::npos, names); +#if LLVM_VERSION_MAJOR < 17 + // LLVM 16 could not destroy the module's named values once the context + // discards names again: the context keeps them. + EXPECT_EQ(context.shouldDiscardValueNames(), discard && !names); +#else + EXPECT_EQ(context.shouldDiscardValueNames(), discard); +#endif + } + } + } + // Like clang, a failed compilation leaves no output behind: neither a partial object // nor an earlier one that would look up to date. class FailedCompilationTest : public CompileTest,