From ba04ad691782a8f38c3bca80aac3e8cf9467e2a7 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 27 Jun 2023 15:33:15 -0700 Subject: [PATCH] [DependencyScanning] Add ability to scan TU with a buffer input Update Dependency scanner so it can scan the dependency of a TU with a provided buffer rather than relying on the on disk file system to provide the input file. --- .../DependencyScanningTool.h | 13 +- .../DependencyScanningWorker.h | 25 ++-- .../DependencyScanningTool.cpp | 7 +- .../DependencyScanningWorker.cpp | 35 ++++-- clang/test/ClangScanDeps/tu-with-modules.c | 111 ++++++++++++++++++ clang/tools/clang-scan-deps/ClangScanDeps.cpp | 38 ++++-- clang/tools/clang-scan-deps/Opts.td | 2 + 7 files changed, 191 insertions(+), 40 deletions(-) create mode 100644 clang/test/ClangScanDeps/tu-with-modules.c diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h index c747bcc3907644..c251d65142667d 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -174,14 +174,17 @@ class DependencyScanningTool { /// \param LookupModuleOutput This function is called to fill in /// "-fmodule-file=", "-o" and other output /// arguments for dependencies. + /// \param TUBuffer Optional memory buffer for translation unit input. If + /// TUBuffer is nullopt, the input should be included in the + /// Commandline already. /// /// \returns a \c StringError with the diagnostic output if clang errors /// occurred, \c TranslationUnitDeps otherwise. - llvm::Expected - getTranslationUnitDependencies(const std::vector &CommandLine, - StringRef CWD, - const llvm::DenseSet &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput); + llvm::Expected getTranslationUnitDependencies( + const std::vector &CommandLine, StringRef CWD, + const llvm::DenseSet &AlreadySeen, + LookupModuleOutputCallback LookupModuleOutput, + std::optional TUBuffer = std::nullopt); /// Given a compilation context specified via the Clang driver command-line, /// gather modular dependencies of module with the given name, and return the diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index 52c89e96269252..7f5d6e1be9e9b9 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -18,6 +18,7 @@ #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBufferRef.h" #include #include @@ -119,24 +120,28 @@ class DependencyScanningWorker { llvm::IntrusiveRefCntPtr FS); /// Run the dependency scanning tool for a given clang driver command-line, - /// and report the discovered dependencies to the provided consumer. If \p - /// ModuleName isn't empty, this function reports the dependencies of module - /// \p ModuleName. + /// and report the discovered dependencies to the provided consumer. If + /// TUBuffer is not nullopt, it is used as TU input for the dependency + /// scanning. Otherwise, the input should be included as part of the + /// command-line. ModuleNames is a list of additional modules to be imported + /// in addition to what is referenced in TU. /// /// \returns false if clang errors occurred (with diagnostics reported to /// \c DiagConsumer), true otherwise. - bool computeDependencies(StringRef WorkingDirectory, - const std::vector &CommandLine, - DependencyConsumer &DepConsumer, - DependencyActionController &Controller, - DiagnosticConsumer &DiagConsumer, - std::optional ModuleName = std::nullopt); + bool computeDependencies( + StringRef WorkingDirectory, const std::vector &CommandLine, + DependencyConsumer &DepConsumer, DependencyActionController &Controller, + DiagnosticConsumer &DiagConsumer, + std::optional> + AdditionalInfo = std::nullopt); + /// \returns A \c StringError with the diagnostic output if clang errors /// occurred, success otherwise. llvm::Error computeDependencies( StringRef WorkingDirectory, const std::vector &CommandLine, DependencyConsumer &Consumer, DependencyActionController &Controller, - std::optional ModuleName = std::nullopt); + std::optional> + AdditionalInfo = std::nullopt); /// Scan from a compiler invocation. /// If \p DiagGenerationAsCompilation is true it will generate error diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp index 123603e0dc525e..6d100f9cfaf047 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -268,11 +268,12 @@ llvm::Expected DependencyScanningTool::getTranslationUnitDependencies( const std::vector &CommandLine, StringRef CWD, const llvm::DenseSet &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput) { + LookupModuleOutputCallback LookupModuleOutput, + std::optional TUBuffer) { FullDependencyConsumer Consumer(AlreadySeen); auto Controller = createActionController(LookupModuleOutput); - llvm::Error Result = - Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller); + llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, + *Controller, TUBuffer); if (Result) return std::move(Result); return Consumer.takeTranslationUnitDeps(); diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 47936613eae1c0..cfb7eddbe6a3ea 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -34,6 +34,7 @@ #include "llvm/CAS/ObjectStore.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/PrefixMapper.h" #include "llvm/TargetParser/Host.h" #include @@ -747,7 +748,8 @@ DependencyScanningWorker::getOrCreateFileManager() const { llvm::Error DependencyScanningWorker::computeDependencies( StringRef WorkingDirectory, const std::vector &CommandLine, DependencyConsumer &Consumer, DependencyActionController &Controller, - std::optional ModuleName) { + std::optional> + AdditionalInfo) { std::vector CLI; for (const std::string &Arg : CommandLine) CLI.push_back(Arg.c_str()); @@ -761,7 +763,7 @@ llvm::Error DependencyScanningWorker::computeDependencies( TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts.release()); if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller, - DiagPrinter, ModuleName)) + DiagPrinter, AdditionalInfo)) return llvm::Error::success(); return llvm::make_error(DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); @@ -831,30 +833,41 @@ static bool createAndRunToolInvocation( bool DependencyScanningWorker::computeDependencies( StringRef WorkingDirectory, const std::vector &CommandLine, DependencyConsumer &Consumer, DependencyActionController &Controller, - DiagnosticConsumer &DC, std::optional ModuleName) { + DiagnosticConsumer &DC, + std::optional> + AdditionalInfo) { // Reset what might have been modified in the previous worker invocation. BaseFS->setCurrentWorkingDirectory(WorkingDirectory); std::optional> ModifiedCommandLine; llvm::IntrusiveRefCntPtr ModifiedFS; + std::optional ModuleName; // If we're scanning based on a module name alone, we don't expect the client // to provide us with an input file. However, the driver really wants to have // one. Let's just make it up to make the driver happy. - if (ModuleName) { + if (AdditionalInfo) { auto OverlayFS = llvm::makeIntrusiveRefCnt(BaseFS); auto InMemoryFS = llvm::makeIntrusiveRefCnt(); InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); - SmallString<128> FakeInputPath; - // TODO: We should retry the creation if the path already exists. - llvm::sys::fs::createUniquePath(*ModuleName + "-%%%%%%%%.input", - FakeInputPath, - /*MakeAbsolute=*/false); - InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer("")); - + if (const auto *Name = std::get_if(&*AdditionalInfo)) { + ModuleName = *Name; + // TODO: We should retry the creation if the path already exists. + llvm::sys::fs::createUniquePath(*Name + "-%%%%%%%%.input", + FakeInputPath, + /*MakeAbsolute=*/false); + InMemoryFS->addFile(FakeInputPath, 0, + llvm::MemoryBuffer::getMemBuffer("")); + } else if (const auto *TU = + std::get_if(&*AdditionalInfo)) { + FakeInputPath = TU->getBufferIdentifier(); + InMemoryFS->addFile( + FakeInputPath, 0, + llvm::MemoryBuffer::getMemBufferCopy(TU->getBuffer())); + } llvm::IntrusiveRefCntPtr InMemoryOverlay = InMemoryFS; // If we are using a CAS but not dependency CASFS, we need to provide the diff --git a/clang/test/ClangScanDeps/tu-with-modules.c b/clang/test/ClangScanDeps/tu-with-modules.c new file mode 100644 index 00000000000000..22360f6980e9ef --- /dev/null +++ b/clang/test/ClangScanDeps/tu-with-modules.c @@ -0,0 +1,111 @@ +// UNSUPPORTED: target=powerpc64-ibm-aix{{.*}} + +// RUN: rm -rf %t +// RUN: split-file %s %t + +//--- module.modulemap +module root { header "root.h" } +module direct { header "direct.h" } +module transitive { header "transitive.h" } +module addition { header "addition.h" } +//--- root.h +#include "direct.h" +#include "root/textual.h" +//--- direct.h +#include "transitive.h" +//--- transitive.h +// empty + +//--- addition.h +// empty + +//--- tu.c +#include "root.h" + +//--- root/textual.h +// This is here to verify that the "root" directory doesn't clash with name of +// the "root" module. + +//--- cdb.json.template +[{ + "file": "", + "directory": "DIR", + "command": "clang -fmodules -fmodules-cache-path=DIR/cache -I DIR -x c -c" +}] + +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -tu-path %t/tu.c > %t/result.json +// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s --check-prefix=CHECK + +// CHECK: { +// CHECK-NEXT: "modules": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "module-name": "transitive" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: "[[PREFIX]]/direct.h" +// CHECK-NEXT: ], +// CHECK: "name": "direct" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "module-name": "direct" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: "[[PREFIX]]/root.h" +// CHECK-NEXT: "[[PREFIX]]/root/textual.h" +// CHECK-NEXT: ], +// CHECK: "name": "root" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: "[[PREFIX]]/transitive.h" +// CHECK-NEXT: ], +// CHECK: "name": "transitive" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "{{.*}}", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "module-name": "root" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [ +// CHECK: ], +// CHECK: "file-deps": [ +// CHECK-NEXT: [[PREFIX]]/tu.c +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "" +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index c9a9f05b927e74..accbfadfb3aaff 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -29,6 +29,7 @@ #include "llvm/Support/Format.h" #include "llvm/Support/JSON.h" #include "llvm/Support/LLVMDriver.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Program.h" #include "llvm/Support/Signals.h" #include "llvm/Support/TargetSelect.h" @@ -36,6 +37,7 @@ #include "llvm/Support/Threading.h" #include "llvm/Support/Timer.h" #include "llvm/TargetParser/Host.h" +#include #include #include #include @@ -89,8 +91,9 @@ static std::string ModuleFilesDir; static bool EagerLoadModules; static unsigned NumThreads = 0; static std::string CompilationDB; -static std::string ModuleName; +static std::optional ModuleName; static std::vector ModuleDepTargets; +static std::string TranslationUnitFile; static bool DeprecatedDriverCommand; static ResourceDirRecipeKind ResourceDirRecipe; static bool Verbose; @@ -220,6 +223,9 @@ static void ParseArgs(int argc, char **argv) { for (const llvm::opt::Arg *A : Args.filtered(OPT_dependency_target_EQ)) ModuleDepTargets.emplace_back(A->getValue()); + if (const llvm::opt::Arg *A = Args.getLastArg(OPT_tu_path_EQ)) + TranslationUnitFile = A->getValue(); + DeprecatedDriverCommand = Args.hasArg(OPT_deprecated_driver_command); if (const llvm::opt::Arg *A = Args.getLastArg(OPT_resource_dir_recipe_EQ)) { @@ -1269,7 +1275,7 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { if (Format == ScanningOutputFormat::Full || Format == ScanningOutputFormat::FullTree || Format == ScanningOutputFormat::FullIncludeTree) - FD.emplace(ModuleName.empty() ? Inputs.size() : 0); + FD.emplace(!ModuleName ? Inputs.size() : 0); struct DepTreeResult { size_t Index; @@ -1290,7 +1296,7 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { if (Format == ScanningOutputFormat::Full || Format == ScanningOutputFormat::FullTree) - FD.emplace(ModuleName.empty() ? Inputs.size() : 0); + FD.emplace(!ModuleName ? Inputs.size() : 0); std::atomic NumStatusCalls = 0; std::atomic NumOpenFileForReadCalls = 0; @@ -1312,10 +1318,6 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { std::string Filename = std::move(Input->Filename); std::string CWD = std::move(Input->Directory); - std::optional MaybeModuleName; - if (!ModuleName.empty()) - MaybeModuleName = ModuleName; - std::string OutputDir(ModuleFilesDir); if (OutputDir.empty()) OutputDir = getModuleCachePath(Input->CommandLine); @@ -1382,16 +1384,30 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { MakeformatOS, Errs)) HadErrors = true; } - } else if (MaybeModuleName) { + } else if (ModuleName) { auto MaybeModuleDepsGraph = WorkerTool.getModuleDependencies( - *MaybeModuleName, Input->CommandLine, CWD, AlreadySeenModules, + *ModuleName, Input->CommandLine, CWD, AlreadySeenModules, LookupOutput); - if (handleModuleResult(*MaybeModuleName, MaybeModuleDepsGraph, *FD, + if (handleModuleResult(*ModuleName, MaybeModuleDepsGraph, *FD, LocalIndex, DependencyOS, Errs)) HadErrors = true; } else { + std::unique_ptr TU; + std::optional TUBuffer; + if (!TranslationUnitFile.empty()) { + auto MaybeTU = llvm::MemoryBuffer::getFile(TranslationUnitFile); + if (!MaybeTU) { + llvm::errs() << "cannot open input translation unit: " + << MaybeTU.getError().message() << "\n"; + HadErrors = true; + continue; + } + TU = std::move(*MaybeTU); + TUBuffer = TU->getMemBufferRef(); + } auto MaybeTUDeps = WorkerTool.getTranslationUnitDependencies( - Input->CommandLine, CWD, AlreadySeenModules, LookupOutput); + Input->CommandLine, CWD, AlreadySeenModules, LookupOutput, + TUBuffer); if (handleTranslationUnitResult(Filename, MaybeTUDeps, *FD, LocalIndex, DependencyOS, Errs)) HadErrors = true; diff --git a/clang/tools/clang-scan-deps/Opts.td b/clang/tools/clang-scan-deps/Opts.td index cc479c12c965d5..03374a7d7182f0 100644 --- a/clang/tools/clang-scan-deps/Opts.td +++ b/clang/tools/clang-scan-deps/Opts.td @@ -29,6 +29,8 @@ defm compilation_database : Eq<"compilation-database", "Compilation database">; defm module_name : Eq<"module-name", "the module of which the dependencies are to be computed">; defm dependency_target : Eq<"dependency-target", "The names of dependency targets for the dependency file">; +defm tu_path: Eq<"tu-path", "The path to the translation unit for depscan. Not compatible with -module-name">; + def deprecated_driver_command : F<"deprecated-driver-command", "use a single driver command to build the tu (deprecated)">; defm resource_dir_recipe : Eq<"resource-dir-recipe", "How to produce missing '-resource-dir' argument">;