diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4bf1b5..b39c0bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -186,6 +186,44 @@ jobs: ${{ runner.temp }}/ct_autofree_tests/*.log ${{ runner.temp }}/ct_runtime_tests/*.log + # compilerlib and its unit tests built with ThreadSanitizer: compilations from several threads + # at once, each in a working directory of its own, race on none of compilerlib's state. LLVM + # and Clang come from packages built without ThreadSanitizer: a race between two + # compilations inside their code goes unseen. + tsan-compilerlib: + name: compilerlib with ThreadSanitizer (ubuntu-24.04, LLVM 20) + runs-on: ubuntu-24.04 + env: + TSAN_OPTIONS: halt_on_error=1:exitcode=66 + + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake ninja-build + sudo bash scripts/ci/install-llvm-apt.sh 20 llvm-20 llvm-20-dev clang-20 libclang-20-dev \ + libclang-rt-20-dev + # The kernel randomizes mmap addresses over more bits than ThreadSanitizer supports. + sudo sysctl vm.mmap_rnd_bits=28 + + - name: Build + run: | + cmake -S . -B build -G Ninja \ + -DCMAKE_C_COMPILER=clang-20 -DCMAKE_CXX_COMPILER=clang++-20 \ + "-DCMAKE_C_FLAGS=-fsanitize=thread -g" "-DCMAKE_CXX_FLAGS=-fsanitize=thread -g" \ + -DCMAKE_EXE_LINKER_FLAGS=-fsanitize=thread \ + -DLLVM_DIR=/usr/lib/llvm-20/lib/cmake/llvm \ + -DClang_DIR=/usr/lib/llvm-20/lib/cmake/clang \ + -DUSE_SHARED_LIB=OFF + cmake --build build --target compilerlib_unit_tests + + - name: Run the compilations with a working directory, concurrent ones included + run: | + build/test/unit/compilerlib_unit_tests \ + --gtest_filter='*WorkingDirectory*:*ConcurrentCompilationTest*' + # Line coverage of compilerlib, cc and the runtime (ENABLE_COVERAGE), from the unit tests, # the smoke tests and the runtime fixture suites. Informational: no threshold is enforced. # The suites build their programs with -fprofile-instr-generate, so that the runtime they diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a11453..952aa55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,23 @@ pull request. ## Unreleased +### Features + +- **Compilations from several threads.** `compilerlib::compile` may run from several threads at + once: each compilation reads and writes files through a file system of its own, whose working + directory `-working-directory ` moves while the process's stays. A host compiles each + command of a compilation database in its directory that way, instead of moving the process + under a lock. A compilation with a job that runs as a program, such as a link, and one that + passes `-mllvm`, `-mdebug-pass`, `-mlimit-float-precision` or `-ftime-report` must not run + while another one does (#262, #261). + +### Behaviour changes + +- **`-working-directory` leaves the process where it is** when every job of the compilation runs + in the process: a compilation to a file or to memory, instrumented or not. The process was + left in ``. A compilation that links or assembles still moves the process to ``, as + clang does (#262, #261). + ### Fixes - **A child of `fork()` and the vtable diagnostics, on Linux.** With `--ct-vtable-diag`, a @@ -119,6 +136,12 @@ pull request. - **`CT_DEBUG_AUTOFREE` read from several threads.** The allocation pass read the variable into two plain statics on its first use, on which two threads instrumenting modules at once raced. It is now read once, into a constant (#251). +- **A relative `-working-directory`** names a directory relative to the process's working + directory. The compilation failed: the `-cc1` job resolved it again from the directory the + driver had moved to (#262, #261). +- **Objects linked alone with `-working-directory`** are read in that directory, to choose + between the C and C++ drivers. They were looked up in the process's working directory, and + the link failed with `failed to inspect object` (#262, #261). ### Performance @@ -177,6 +200,10 @@ pull request. Ubuntu 26.04, where clang-format 19 ran and apt.llvm.org has no LLVM 20. `scripts/format.sh` and `scripts/format-check.sh` run the binary that `CLANG_FORMAT` names and print its version (#238, #237). +- A job of the Build workflow builds compilerlib and its unit tests with ThreadSanitizer, and + runs the tests of `-working-directory`, eight threads compiling at once included. LLVM and + Clang come from packages built without ThreadSanitizer, so a race inside their code goes + unseen (#262, #261). - When `sbrk()` fails in the sbrk test of the full allocation table, the test reports `errno`, the program break and the mapping above it, `RLIMIT_DATA`, and the memory of the process and of the system (#239, #233). @@ -211,6 +238,8 @@ pull request. - `NOTICE` dates the copyright from 2025, when the code began, credits the CoreTrace contributors alongside Hugo Payet and links the repository. It keeps only these attribution notices, without the license header meant for source files (#248). +- The README and `compilerlib/compiler.h` describe compiling from several threads, with + `-working-directory`, and its limits (#262, #261). ## v0.12.0 (2026-10-22) diff --git a/README.md b/README.md index 8c0121b..c3f2cc2 100644 --- a/README.md +++ b/README.md @@ -498,3 +498,22 @@ expects, and waits for it: a host can compile from a thread with a small stack, secondary thread on macOS, whose stack is 512 KiB (#204). The time-trace profiler a host runs on its own thread, started with `llvm::timeTraceProfilerInitialize`, therefore does not record the compilation. With `-ftime-trace`, each job writes its own trace file, as clang does. + +### Compiling from several threads + +`compilerlib::compile` may run from several threads at once, as for a host that compiles each +command of a compilation database. Each compilation reads and writes files through a file system +of its own, whose working directory `-working-directory ` sets without moving the process: +the sources, the include directories and the files the compilation writes, its output, its +dependency file and the others, are relative to ``, and a relative `` is relative to the +process's working directory. A host passes each command's directory that way, instead of changing +the process's working directory under a lock. + +Two kinds of compilations must not run while another one does: + +- **Jobs that run as programs.** The linker, the assembler and, with `-fno-integrated-cc1`, every + job run as programs of their own, which start in the process's working directory. With + `-working-directory`, such a compilation moves the whole process to ``, as clang does, and + leaves it there. +- **Options of LLVM's own.** Clang applies `-mllvm`, `-mdebug-pass`, `-mlimit-float-precision` + and `-ftime-report` to LLVM's state, which the whole process shares. diff --git a/include/compilerlib/compiler.h b/include/compilerlib/compiler.h index 645d26e..b1c5fcd 100644 --- a/include/compilerlib/compiler.h +++ b/include/compilerlib/compiler.h @@ -30,6 +30,18 @@ namespace compilerlib }; // std::pair compile(const std::vector& args); + /// Compiles as clang does with `args`, in this process. + /// + /// `-working-directory ` gives the compilation a working directory of its own: the + /// sources, the include directories and the files it writes are relative to ``, and a + /// relative `` to the process's working directory, which stays where it is. A + /// compilation with a job that runs as a program of its own, as linking and assembling do, + /// moves the whole process to `` instead, as clang does, and leaves it there. + /// + /// Compilations may run from several threads at once. A compilation that moves the process + /// must not run while another compilation of the process does, nor one that passes an + /// option that Clang applies to LLVM's state of the whole process: `-mllvm`, `-mdebug-pass`, + /// `-mlimit-float-precision` or `-ftime-report`. CompileResult compile(const std::vector& args, OutputMode mode = OutputMode::ToFile, bool instrument = false); diff --git a/src/compilerlib/args.cpp b/src/compilerlib/args.cpp index e3f1455..d24b692 100644 --- a/src/compilerlib/args.cpp +++ b/src/compilerlib/args.cpp @@ -2,10 +2,16 @@ #include "args_internal.hpp" #include "clang_driver_compat.hpp" +#include +#include #include #include +#include +#include #include +#include +#include namespace compilerlib { @@ -107,6 +113,157 @@ namespace compilerlib args.swap(out); } + std::string normalizeWorkingDirectoryArgs(std::vector& args, + llvm::vfs::FileSystem& fs) + { + std::vector argv; + argv.reserve(args.size()); + for (const auto& arg : args) + argv.push_back(arg.c_str()); + unsigned missingIndex = 0; + unsigned missingCount = 0; + // The driver's own parse: the value of another option, or an input after "--", is + // not an option. + const llvm::opt::InputArgList parsed = + clang_compat::driverOptTable().ParseArgs(argv, missingIndex, missingCount); + + // Index of each -working-directory, either spelling, and its absolute directory. + std::map directories; + std::string last; + for (const llvm::opt::Arg* arg : + parsed.filtered(clang_compat::options::OPT_working_directory)) + { + llvm::SmallString<256> directory(arg->getValue()); + (void)fs.makeAbsolute(directory); + last = directory.str().str(); + directories.emplace(arg->getIndex(), last); + } + if (directories.empty()) + return {}; + + std::vector out; + out.reserve(args.size() + directories.size()); + for (unsigned i = 0; i < args.size(); ++i) + { + auto found = directories.find(i); + if (found == directories.end()) + { + out.push_back(std::move(args[i])); + continue; + } + // The separate spelling has its directory in the next argument. + if (args[i] == "-working-directory") + ++i; + out.push_back("-working-directory"); + out.push_back(std::move(found->second)); + } + args.swap(out); + return last; + } + + llvm::opt::ArgStringList + resolveCc1OutputPaths(const llvm::opt::ArgStringList& args, llvm::StringRef directory, + llvm::function_ref save) + { + // The -cc1 spellings the driver gives these options, LLVM 16 to 23. + static constexpr llvm::StringLiteral kSeparate[] = { + "-o", + "-dependency-file", + "-header-include-file", + "-opt-record-file", + "-split-dwarf-output", + "-stack-usage-file", + "-serialize-diagnostic-file", + "-dependency-dot", + }; + static constexpr llvm::StringLiteral kJoined[] = { + "-ftime-trace=", "-stats-file=", "-fmodules-cache-path="}; + + // nullptr when `path` needs no change: absolute, or "-" for a standard stream. A + // relative path is appended to the directory, as clang's FileManager does. + auto absolute = [&](llvm::StringRef prefix, llvm::StringRef path) -> const char* + { + if (path.empty() || path == "-" || llvm::sys::path::is_absolute(path)) + return nullptr; + llvm::SmallString<256> resolved(directory); + llvm::sys::path::append(resolved, path); + std::string argument = prefix.str(); + argument += resolved.str(); + return save(argument); + }; + + llvm::opt::ArgStringList out; + out.reserve(args.size()); + for (size_t i = 0; i < args.size(); ++i) + { + const llvm::StringRef arg = args[i]; + out.push_back(args[i]); + if (llvm::is_contained(kSeparate, arg) && i + 1 < args.size()) + { + ++i; + const char* resolved = absolute("", args[i]); + out.push_back(resolved ? resolved : args[i]); + continue; + } + for (llvm::StringRef prefix : kJoined) + { + if (!arg.starts_with(prefix)) + continue; + if (const char* resolved = absolute(prefix, arg.drop_front(prefix.size()))) + out.back() = resolved; + break; + } + } + return out; + } + + void resolveDriverOutputPaths(std::vector& args, llvm::StringRef directory) + { + std::vector argv; + argv.reserve(args.size()); + for (const auto& arg : args) + argv.push_back(arg.c_str()); + unsigned missingIndex = 0; + unsigned missingCount = 0; + // The driver's own parse, as for -working-directory. + const llvm::opt::InputArgList parsed = + clang_compat::driverOptTable().ParseArgs(argv, missingIndex, missingCount); + + // Index of each option, either spelling, with its spelling and its absolute path. + std::map> resolved; + for (const llvm::opt::Arg* arg : parsed.filtered( + clang_compat::options::OPT_MJ, clang_compat::options::OPT_gen_cdb_fragment_path)) + { + const llvm::StringRef value = arg->getValue(); + if (value.empty() || llvm::sys::path::is_absolute(value)) + continue; + llvm::SmallString<256> path(directory); + llvm::sys::path::append(path, value); + resolved.emplace(arg->getIndex(), + std::make_pair(arg->getSpelling().str(), path.str().str())); + } + if (resolved.empty()) + return; + + std::vector out; + out.reserve(args.size() + resolved.size()); + for (unsigned i = 0; i < args.size(); ++i) + { + auto found = resolved.find(i); + if (found == resolved.end()) + { + out.push_back(std::move(args[i])); + continue; + } + // The separate spelling has its path in the next argument. + if (args[i] == found->second.first) + ++i; + out.push_back(std::move(found->second.first)); + out.push_back(std::move(found->second.second)); + } + args.swap(out); + } + void appendDiagnostics(std::string& out, const std::string& extra) { if (extra.empty()) diff --git a/src/compilerlib/args_internal.hpp b/src/compilerlib/args_internal.hpp index 94aac8b..a93c188 100644 --- a/src/compilerlib/args_internal.hpp +++ b/src/compilerlib/args_internal.hpp @@ -7,6 +7,7 @@ #include "compilerlib/attributes.hpp" +#include #include #include #include @@ -14,6 +15,11 @@ #include #include +namespace llvm::vfs +{ + class FileSystem; +} // namespace llvm::vfs + namespace compilerlib { // Value of `opt` given either as a separate argument (`-o out`) or joined with @@ -36,6 +42,29 @@ namespace compilerlib // Rewrites the `-o=path` and `-x=lang` spellings into the two-argument forms. void normalizeEqualsArgs(std::vector& args); + // Rewrites each -working-directory of a driver command line, `-working-directory ` or + // `-working-directory=`, into the two-argument form, with made absolute by `fs`. + // Returns the directory of the last one, which the driver uses; empty without one. + CT_NODISCARD std::string normalizeWorkingDirectoryArgs(std::vector& args, + llvm::vfs::FileSystem& fs); + + // The arguments of a -cc1 job, with the relative path of each file it writes made absolute + // against `directory`: its output, dependency file and graph, header list, optimization + // record, split DWARF, stack usage, serialized diagnostics, time trace, statistics and + // module cache. A job resolves the files it reads, and its output, against its + // -working-directory, but the other files it writes against the process's working + // directory, which clang itself moves to that directory first. `save` keeps a new argument + // alive as long as the job. + CT_NODISCARD llvm::opt::ArgStringList + resolveCc1OutputPaths(const llvm::opt::ArgStringList& args, llvm::StringRef directory, + llvm::function_ref save); + + // Rewrites each option of a driver command line that names what the driver itself writes, + // `-MJ ` and `-gen-cdb-fragment-path `, into the two-argument form, with a + // relative path made absolute against `directory`. The driver writes them relative to the + // process's working directory, which clang itself moves to that directory first. + void resolveDriverOutputPaths(std::vector& args, llvm::StringRef directory); + // Appends `extra` to `out` with exactly one newline between them; no-op when empty. void appendDiagnostics(std::string& out, const std::string& extra); diff --git a/src/compilerlib/compiler.cpp b/src/compilerlib/compiler.cpp index 025a8c1..cfd7151 100644 --- a/src/compilerlib/compiler.cpp +++ b/src/compilerlib/compiler.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -222,6 +223,12 @@ namespace compilerlib std::string clang_resource_dir; std::string clang_sysroot; RuntimeArchives runtime_archives; + // The directory of -working-directory, absolute; empty without the option. + std::string working_directory; + // The file system of this compilation alone: its working directory is the + // process's when the compilation starts, then -working-directory's once the + // driver reads it, while the process's stays where it is. Compilations from + // several threads thus each have a working directory of their own. llvm::IntrusiveRefCntPtr fs; // Options of the driver's diagnostics engine, which must outlive it. clang_compat::DiagnosticOptionsStorage driver_diagnostic_options; @@ -230,7 +237,7 @@ namespace compilerlib CompileContext(const std::vector& args, OutputMode mode, bool instrument) : mode(mode), instrument(instrument), input_args(args), - fs(llvm::vfs::getRealFileSystem()) + fs(llvm::vfs::createPhysicalFileSystem()) { } }; @@ -259,6 +266,14 @@ namespace compilerlib { extractRuntimeConfig(ctx_.input_args, ctx_.filtered_args, ctx_.runtimeConfig); normalizeEqualsArgs(ctx_.filtered_args); + // Absolute, so that it names one directory to the driver and to the jobs it + // gives the option to. + ctx_.working_directory = + normalizeWorkingDirectoryArgs(ctx_.filtered_args, *ctx_.fs); + // The driver writes the files of -MJ and -gen-cdb-fragment-path relative to the + // process's working directory, which stays where it is. + if (!ctx_.working_directory.empty()) + resolveDriverOutputPaths(ctx_.filtered_args, ctx_.working_directory); if (ctx_.runtimeConfig.bounds_without_alloc) { ctx_.dc.os << "warning: ct: bounds instrumentation requires alloc tracking; " @@ -580,6 +595,14 @@ namespace compilerlib bool succeeded = false; if (parsed) { +#if LLVM_VERSION_MAJOR < 22 + // The compilation's file system, with the invocation's overlays, as LLVM 22 + // sets it above: the job reads some files through it rather than through its + // file manager, such as a sanitizer's ignore list, and a relative path then + // names a file of the compilation's working directory. + ci->createFileManager(clang::createVFSFromCompilerInvocation( + ci->getInvocation(), ci->getDiagnostics(), ctx.fs)); +#endif Cc1TimeTrace trace(*ci, argv[0]); succeeded = clang::ExecuteCompilerInvocation(ci.get()); } @@ -629,7 +652,20 @@ namespace compilerlib clang_compat::options::OPT_fno_integrated_cc1, /*Default=*/true); for (clang::driver::Command& job : ownedComp->getJobs()) + { job.InProcess = integrated && isCc1Command(job.getArguments()); + // clang moves the whole process to -working-directory, so the files + // its jobs write land there; here the process stays, and the paths + // of the files a -cc1 job writes are resolved beforehand. + if (!ctx_.working_directory.empty() && isCc1Command(job.getArguments())) + { + const llvm::opt::DerivedArgList& owner = ownedComp->getArgs(); + job.replaceArguments( + resolveCc1OutputPaths(job.getArguments(), ctx_.working_directory, + [&](llvm::StringRef argument) + { return owner.MakeArgString(argument); })); + } + } } else { @@ -1207,6 +1243,22 @@ namespace compilerlib return result; } + // Whether every job of `comp` runs in this process: the -cc1 jobs, unless the driver + // executes them with -fno-integrated-cc1, and none of the others, which run clang's + // tools, such as the linker or the assembler, as programs of their own. + CT_NODISCARD bool allJobsRunInProcess(const clang::driver::Compilation& comp, + bool driverExecutes) + { + for (const clang::driver::Command& job : comp.getJobs()) + { + const bool inProcess = + driverExecutes ? job.InProcess : isCc1Command(job.getArguments()); + if (!inProcess) + return false; + } + return true; + } + CT_NODISCARD bool validateJobPlan(const JobPlan& plan, OutputMode mode, std::string& error) { if (mode == OutputMode::ToMemory || mode == OutputMode::ToMemoryBitcode) @@ -1331,7 +1383,27 @@ namespace compilerlib "without link-time optimization", {}}; - if (!instrument && mode == OutputMode::ToFile && !ctx.runtimeConfig.optnone_enabled) + // Plain file output runs the jobs as clang's driver does; the other modes run the -cc1 + // jobs here and the others, such as the link, as programs. + const bool driverExecutes = + !instrument && mode == OutputMode::ToFile && !ctx.runtimeConfig.optnone_enabled; + + // A program, such as the linker or the assembler, starts in the working directory of + // this process. With -working-directory, the process moves to that directory before + // such jobs run, as clang does, and stays there: the compilation changes the working + // directory of the whole process, so it must not run while another compilation of the + // process does. -### only prints the jobs. + if (!ctx.working_directory.empty() && !allJobsRunInProcess(*comp, driverExecutes) && + !comp->getArgs().hasArg(clang_compat::options::OPT__HASH_HASH_HASH)) + { + if (const std::error_code ec = llvm::sys::fs::set_current_path(ctx.working_directory)) + return {false, + "error: unable to set working directory: " + ctx.working_directory + ": " + + ec.message(), + {}}; + } + + if (driverExecutes) return runNonInstrumentedCompilation(ctx, driver, *comp); JobPlan plan = buildJobPlan(*comp); diff --git a/src/compilerlib/toolchain.cpp b/src/compilerlib/toolchain.cpp index 4a16695..c287f8d 100644 --- a/src/compilerlib/toolchain.cpp +++ b/src/compilerlib/toolchain.cpp @@ -3,6 +3,7 @@ #include "clang_driver_compat.hpp" #include "toolchain_internal.hpp" +#include #include #include #include @@ -180,8 +181,20 @@ namespace compilerlib bool has_source_inputs = false; bool has_object_inputs = false; std::vector inputs; + // The -working-directory value, when the arguments give one. + std::string working_directory; }; + // An input where the driver reads it: relative to -working-directory, when given. + CT_NODISCARD std::string inputPath(const ArgScan& scan, const std::string& input) + { + if (scan.working_directory.empty() || llvm::sys::path::is_absolute(input)) + return input; + llvm::SmallString<256> path(scan.working_directory); + llvm::sys::path::append(path, input); + return path.str().str(); + } + CT_NODISCARD ArgScan scanArgs(const std::vector& args) { ArgScan scan; @@ -214,6 +227,15 @@ namespace compilerlib scan.resource_dir = arg.drop_front(sizeof("-resource-dir=") - 1).str(); continue; } + if (arg == "-working-directory" || arg.starts_with("-working-directory=")) + { + if (arg == "-working-directory" && i + 1 < args.size()) + scan.working_directory = args[++i]; + else if (arg != "-working-directory") + scan.working_directory = + arg.drop_front(sizeof("-working-directory=") - 1).str(); + continue; + } if (arg == "-isysroot") { scan.has_sysroot = true; @@ -505,7 +527,7 @@ namespace compilerlib llvm::StringRef ext = llvm::sys::path::extension(path); if (!isObjectExt(ext) && !isArchiveExt(ext)) continue; - llvm::Expected looksCxx = pathLooksLikeCxxObject(path); + llvm::Expected looksCxx = pathLooksLikeCxxObject(inputPath(scan, path)); if (!looksCxx) { error = "failed to inspect object: " + path + ": " + diff --git a/test/unit/args_test.cpp b/test/unit/args_test.cpp index 0fdaded..f44e7ec 100644 --- a/test/unit/args_test.cpp +++ b/test/unit/args_test.cpp @@ -1,7 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 #include "compilerlib/args_internal.hpp" +#include #include +#include +#include +#include +#include #include #include @@ -149,6 +154,129 @@ namespace EXPECT_EQ(args, before); } + // The directory that `fs` makes of `path`, as normalizeWorkingDirectoryArgs does. + std::string absoluteIn(llvm::vfs::FileSystem& fs, const char* path) + { + llvm::SmallString<256> directory(path); + EXPECT_FALSE(fs.makeAbsolute(directory)); + return directory.str().str(); + } + + TEST(NormalizeWorkingDirectoryArgs, MakesEverySpellingAbsoluteAndReturnsTheLast) + { + auto fs = llvm::vfs::createPhysicalFileSystem(); + Args args{"-working-directory", "one", "-c", "a.c", "-working-directory=two"}; + const std::string last = compilerlib::normalizeWorkingDirectoryArgs(args, *fs); + EXPECT_EQ(last, absoluteIn(*fs, "two")); + EXPECT_TRUE(llvm::sys::path::is_absolute(last)); + EXPECT_EQ(args, (Args{"-working-directory", absoluteIn(*fs, "one"), "-c", "a.c", + "-working-directory", last})); + } + + TEST(NormalizeWorkingDirectoryArgs, LeavesTheValuesOfOtherOptionsAlone) + { + auto fs = llvm::vfs::createPhysicalFileSystem(); + Args args{"-Xclang", "-working-directory", "-Xclang", "dir", "-c", "a.c"}; + const Args before = args; + EXPECT_EQ(compilerlib::normalizeWorkingDirectoryArgs(args, *fs), ""); + EXPECT_EQ(args, before); + } + + // The directory a -cc1 job's relative paths are resolved against, absolute on the host. + std::string jobDirectory(void) + { +#ifdef _WIN32 + return "C:\\work"; +#else + return "/work"; +#endif + } + + std::string inJobDirectory(const char* path) + { + llvm::SmallString<64> resolved(jobDirectory()); + llvm::sys::path::append(resolved, path); + return resolved.str().str(); + } + + Args resolveOutputs(const Args& args) + { + llvm::opt::ArgStringList list; + for (const std::string& arg : args) + list.push_back(arg.c_str()); + llvm::BumpPtrAllocator allocator; + llvm::StringSaver saver(allocator); + const llvm::opt::ArgStringList resolved = + compilerlib::resolveCc1OutputPaths(list, jobDirectory(), [&](llvm::StringRef argument) + { return saver.save(argument).data(); }); + return Args(resolved.begin(), resolved.end()); + } + + TEST(ResolveCc1OutputPaths, ResolvesTheFilesTheJobWrites) + { + const Args args{"-cc1", "-o", + "out/a.o", "-dependency-file", + "a.d", "-opt-record-file", + "a.yaml", "-split-dwarf-output", + "a.dwo", "-stack-usage-file", + "a.su", "-serialize-diagnostic-file", + "a.dia", "-header-include-file", + "a.h.txt", "-ftime-trace=a.json"}; + EXPECT_EQ(resolveOutputs(args), + (Args{"-cc1", "-o", inJobDirectory("out/a.o"), "-dependency-file", + inJobDirectory("a.d"), "-opt-record-file", inJobDirectory("a.yaml"), + "-split-dwarf-output", inJobDirectory("a.dwo"), "-stack-usage-file", + inJobDirectory("a.su"), "-serialize-diagnostic-file", + inJobDirectory("a.dia"), "-header-include-file", inJobDirectory("a.h.txt"), + "-ftime-trace=" + inJobDirectory("a.json")})); + } + + // The inputs, which the job resolves itself; the name of the split DWARF file, which + // the object records as given; absolute paths; and "-", a standard stream. + TEST(ResolveCc1OutputPaths, LeavesEverythingElseAlone) + { + const Args args{"-cc1", + "-I", + "inc", + "-split-dwarf-file", + "a.dwo", + "-dependency-file", + "-", + "-o", + inJobDirectory("elsewhere.o"), + "-x", + "c", + "a.c"}; + EXPECT_EQ(resolveOutputs(args), args); + } + + TEST(ResolveCc1OutputPaths, ResolvesTheStatisticsDependencyGraphAndModuleCache) + { + const Args args{"-cc1", "-stats-file=a.stats", "-dependency-dot", "a.dot", + "-fmodules-cache-path=modules"}; + EXPECT_EQ( + resolveOutputs(args), + (Args{"-cc1", "-stats-file=" + inJobDirectory("a.stats"), "-dependency-dot", + inJobDirectory("a.dot"), "-fmodules-cache-path=" + inJobDirectory("modules")})); + } + + TEST(ResolveDriverOutputPaths, ResolvesTheFilesTheDriverWrites) + { + Args args{"-MJ", "a.json", "-MJb.json", "-gen-cdb-fragment-path", "fragments", "-c", "a.c"}; + compilerlib::resolveDriverOutputPaths(args, jobDirectory()); + EXPECT_EQ(args, (Args{"-MJ", inJobDirectory("a.json"), "-MJ", inJobDirectory("b.json"), + "-gen-cdb-fragment-path", inJobDirectory("fragments"), "-c", "a.c"})); + } + + // An absolute path, and the value of another option. + TEST(ResolveDriverOutputPaths, LeavesEverythingElseAlone) + { + Args args{"-MJ", inJobDirectory("a.json"), "-Xclang", "-MJ", "-o", "a.o", "-c", "a.c"}; + const Args before = args; + compilerlib::resolveDriverOutputPaths(args, jobDirectory()); + EXPECT_EQ(args, before); + } + TEST(MergeDiagnostics, JoinsWithASingleNewline) { EXPECT_EQ(compilerlib::mergeDiagnostics("driver", "cc1"), "driver\ncc1"); diff --git a/test/unit/compile_test.cpp b/test/unit/compile_test.cpp index 4c6e741..cbbbdba 100644 --- a/test/unit/compile_test.cpp +++ b/test/unit/compile_test.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #ifndef _WIN32 @@ -1859,4 +1860,237 @@ int once(int x) { return x; } EXPECT_TRUE(function->hasFnAttribute(llvm::Attribute::OptimizeNone)) << name; } } + + // -working-directory, as clang reads it: the sources, the include directories and the + // files a compilation writes are relative to it. + class WorkingDirectoryTest : public CompileTest + { + protected: + // A project whose source includes its header through a relative include directory. + std::string writeProject(const std::string& name, int value) const + { + writeSource((name + "/inc/value.h").c_str(), + ("#define VALUE " + std::to_string(value) + "\n").c_str()); + writeSource((name + "/unit.c").c_str(), + "#include \"value.h\"\nint value(void) { return VALUE; }\n"); + return path(name.c_str()); + } + }; + + // The linker runs as a program of its own, which starts in the process's working + // directory: as clang does, a link with -working-directory moves the process to that + // directory, and leaves it there. The test moves it back. + TEST_F(WorkingDirectoryTest, LinkingRunsInTheDirectory) + { + const fs::path project = writeProject("project", 0); + writeSource("project/main.c", "int value(void);\nint main(void) { return value(); }\n"); + const fs::path processDirectory = fs::current_path(); + compilerlib::CompileResult result = compilerlib::compile( + {"-working-directory", project.string(), "-Iinc", "unit.c", "main.c", "-o", "program"}, + compilerlib::OutputMode::ToFile); + const fs::path linkDirectory = fs::current_path(); + fs::current_path(processDirectory); + ASSERT_TRUE(result.success) << result.diagnostics; + EXPECT_TRUE(fs::exists(project / "program") || fs::exists(project / "program.exe")); + EXPECT_EQ(fs::canonical(linkDirectory), fs::canonical(project)); + } + + // The assembler runs as a program of its own too. An empty source, which every target + // assembles, gives its object in the directory. + TEST_F(WorkingDirectoryTest, AssemblingRunsInTheDirectory) + { + writeSource("project/empty.s", ""); + const fs::path project = path("project"); + const fs::path processDirectory = fs::current_path(); + compilerlib::CompileResult result = compilerlib::compile( + {"-working-directory", project.string(), "-c", "empty.s", "-o", "empty.o"}, + compilerlib::OutputMode::ToFile); + fs::current_path(processDirectory); + ASSERT_TRUE(result.success) << result.diagnostics; + EXPECT_TRUE(fs::exists(project / "empty.o")); + } + + class WorkingDirectoryOutputTest : public WorkingDirectoryTest, + public ::testing::WithParamInterface + { + }; + + // The directory resolves the source and the include directory, and receives the output + // and the dependency file, instrumented or not. + TEST_P(WorkingDirectoryOutputTest, ResolvesTheSourcesAndTheFilesWritten) + { + const fs::path project = writeProject("project", 42); + const fs::path processDirectory = fs::current_path(); + compilerlib::CompileResult result = + compilerlib::compile({"-working-directory", project.string(), "-Iinc", "-c", "unit.c", + "-MD", "-MF", "ct_wd_unit.d", "-o", "ct_wd_unit.o"}, + compilerlib::OutputMode::ToFile, /*instrument=*/GetParam()); + fs::current_path(processDirectory); + ASSERT_TRUE(result.success) << result.diagnostics; + EXPECT_TRUE(fs::exists(project / "ct_wd_unit.o")); + EXPECT_NE(readFile((project / "ct_wd_unit.d").string()).find("value.h"), std::string::npos); + EXPECT_FALSE(fs::exists(processDirectory / "ct_wd_unit.o")); + EXPECT_FALSE(fs::exists(processDirectory / "ct_wd_unit.d")); + } + + // A compilation whose jobs all run in this process leaves the process's working + // directory where it is, instrumented or not. + TEST_P(WorkingDirectoryOutputTest, ProcessStaysInItsDirectory) + { + const fs::path project = writeProject("project", 42); + const fs::path processDirectory = fs::current_path(); + compilerlib::CompileResult result = compilerlib::compile( + {"-working-directory", project.string(), "-Iinc", "-c", "unit.c", "-o", "unit.o"}, + compilerlib::OutputMode::ToFile, /*instrument=*/GetParam()); + const fs::path directoryAfter = fs::current_path(); + fs::current_path(processDirectory); + ASSERT_TRUE(result.success) << result.diagnostics; + EXPECT_EQ(directoryAfter, processDirectory); + EXPECT_TRUE(fs::exists(project / "unit.o")); + } + + INSTANTIATE_TEST_SUITE_P(Instrumentation, WorkingDirectoryOutputTest, ::testing::Bool(), + [](const ::testing::TestParamInfo& info) + { return info.param ? "Instrumented" : "Plain"; }); + + // A relative directory is relative to the process's working directory, for the driver + // and for the jobs it gives the option to. + TEST_F(WorkingDirectoryTest, RelativeDirectoryIsRelativeToTheProcess) + { + const fs::path processDirectory = fs::current_path(); + const fs::path relative = fs::relative(writeProject("project", 42), processDirectory); + if (relative.empty()) + GTEST_SKIP() << "no relative path from the working directory to the test's"; + compilerlib::CompileResult result = compilerlib::compile( + {"-working-directory=" + relative.string(), "-Iinc", "-S", "-emit-llvm", "unit.c"}, + compilerlib::OutputMode::ToMemory); + fs::current_path(processDirectory); + ASSERT_TRUE(result.success) << result.diagnostics; + EXPECT_NE(result.llvmIR.find("ret i32 42"), std::string::npos) << result.llvmIR; + } + + // Objects alone are read before the link, to choose between the C and C++ drivers: they + // are read in the directory too. + TEST_F(WorkingDirectoryTest, ObjectsAreLinkedFromTheDirectory) + { + const fs::path project = writeProject("project", 0); + writeSource("project/main.c", "int value(void);\nint main(void) { return value(); }\n"); + const fs::path processDirectory = fs::current_path(); + for (const char* name : {"unit", "main"}) + { + const std::string source = std::string(name) + ".c"; + const std::string object = std::string(name) + ".o"; + compilerlib::CompileResult compiled = compilerlib::compile( + {"-working-directory", project.string(), "-Iinc", "-c", source, "-o", object}, + compilerlib::OutputMode::ToFile); + fs::current_path(processDirectory); + ASSERT_TRUE(compiled.success) << compiled.diagnostics; + } + + compilerlib::CompileResult result = compilerlib::compile( + {"-working-directory", project.string(), "unit.o", "main.o", "-o", "program"}, + compilerlib::OutputMode::ToFile); + fs::current_path(processDirectory); + ASSERT_TRUE(result.success) << result.diagnostics; + EXPECT_TRUE(fs::exists(project / "program") || fs::exists(project / "program.exe")); + } + + // A -cc1 job reads some files through its file system rather than through its file + // manager, such as a sanitizer's ignore list, which the driver also reads first: both + // read it in the directory. + TEST_F(WorkingDirectoryTest, IgnoreListIsReadInTheDirectory) + { + const fs::path project = writeProject("project", 42); + writeSource("project/ignore.txt", "fun:value\n"); + const fs::path processDirectory = fs::current_path(); + compilerlib::CompileResult result = compilerlib::compile( + {"-working-directory", project.string(), "-Iinc", "-fsanitize=undefined", + "-fsanitize-ignorelist=ignore.txt", "-c", "unit.c", "-o", "unit.o"}, + compilerlib::OutputMode::ToFile); + fs::current_path(processDirectory); + ASSERT_TRUE(result.success) << result.diagnostics; + EXPECT_TRUE(fs::exists(project / "unit.o")); + } + + // The driver itself writes the fragment of a compilation database that -MJ asks for: in + // the directory, as clang does. + TEST_F(WorkingDirectoryTest, CompilationDatabaseFragmentIsWrittenInTheDirectory) + { + const fs::path project = writeProject("project", 42); + const fs::path processDirectory = fs::current_path(); + compilerlib::CompileResult result = + compilerlib::compile({"-working-directory", project.string(), "-Iinc", "-c", "unit.c", + "-o", "unit.o", "-MJ", "unit.json"}, + compilerlib::OutputMode::ToFile); + fs::current_path(processDirectory); + ASSERT_TRUE(result.success) << result.diagnostics; + EXPECT_NE(readFile((project / "unit.json").string()).find("\"unit.c\""), std::string::npos); + EXPECT_FALSE(fs::exists(processDirectory / "unit.json")); + } + + // The -cc1 job writes the statistics file of -save-stats=cwd: in the directory too. + TEST_F(WorkingDirectoryTest, StatisticsAreWrittenInTheDirectory) + { + const fs::path project = writeProject("project", 42); + const fs::path processDirectory = fs::current_path(); + compilerlib::CompileResult result = + compilerlib::compile({"-working-directory", project.string(), "-Iinc", "-c", "unit.c", + "-o", "unit.o", "-save-stats=cwd"}, + compilerlib::OutputMode::ToFile); + fs::current_path(processDirectory); + ASSERT_TRUE(result.success) << result.diagnostics; + EXPECT_TRUE(fs::exists(project / "unit.stats")); + EXPECT_FALSE(fs::exists(processDirectory / "unit.stats")); + } + + // Compilations from several threads at once, each in a project of its own whose header + // has the same name: each reads its own, and the process's working directory stays + // where it is. + class ConcurrentCompilationTest : public WorkingDirectoryTest, + public ::testing::WithParamInterface + { + }; + + TEST_P(ConcurrentCompilationTest, EachReadsItsOwnWorkingDirectory) + { + constexpr int kThreads = 8; + constexpr int kRounds = 3; + std::vector projects; + for (int i = 0; i < kThreads; ++i) + projects.push_back(writeProject("project" + std::to_string(i), 100 + i)); + const fs::path processDirectory = fs::current_path(); + + std::vector failures(kThreads); + std::vector threads; + for (int i = 0; i < kThreads; ++i) + { + threads.emplace_back( + [&, i] + { + const std::string expected = "ret i32 " + std::to_string(100 + i); + for (int round = 0; round < kRounds && failures[i].empty(); ++round) + { + compilerlib::CompileResult result = compilerlib::compile( + {"-working-directory", projects[i], "-Iinc", "-S", "-emit-llvm", + "unit.c"}, + compilerlib::OutputMode::ToMemory, /*instrument=*/GetParam()); + if (!result.success) + failures[i] = result.diagnostics; + else if (result.llvmIR.find(expected) == std::string::npos) + failures[i] = "no \"" + expected + "\" in\n" + result.llvmIR; + } + }); + } + for (std::thread& thread : threads) + thread.join(); + const fs::path directoryAfter = fs::current_path(); + fs::current_path(processDirectory); + for (int i = 0; i < kThreads; ++i) + EXPECT_EQ(failures[i], "") << "thread " << i; + EXPECT_EQ(directoryAfter, processDirectory); + } + + INSTANTIATE_TEST_SUITE_P(Instrumentation, ConcurrentCompilationTest, ::testing::Bool(), + [](const ::testing::TestParamInfo& info) + { return info.param ? "Instrumented" : "Plain"; }); } // namespace