20 #include "clang/AST/ASTContext.h" 21 #include "clang/Index/IndexingAction.h" 22 #include "clang/Index/IndexingOptions.h" 23 #include "clang/Lex/MacroInfo.h" 24 #include "clang/Lex/Preprocessor.h" 25 #include "llvm/ADT/DenseMap.h" 26 #include "llvm/ADT/DenseSet.h" 27 #include "llvm/ADT/STLExtras.h" 28 #include "llvm/ADT/StringRef.h" 35 llvm::ArrayRef<Decl *> DeclsToIndex,
38 bool IsIndexMainAST) {
45 index::IndexingOptions IndexOpts;
47 IndexOpts.SystemSymbolFilter =
48 index::IndexingOptions::SystemSymbolFilterKind::DeclarationsOnly;
49 IndexOpts.IndexFunctionLocals =
false;
57 IndexOpts.IndexMacrosInPreprocessor =
true;
66 index::indexTopLevelDecls(AST, *PP, DeclsToIndex, Collector, IndexOpts);
68 const auto &SM = AST.getSourceManager();
69 const auto *MainFileEntry = SM.getFileEntryForID(SM.getMainFileID());
70 std::string
FileName = MainFileEntry ? MainFileEntry->getName() :
"";
76 vlog(
"index AST for {0} (main={1}): \n" 77 " symbol slab: {2} symbols, {3} bytes\n" 78 " ref slab: {4} symbols, {5} refs, {6} bytes\n" 79 " relations slab: {7} relations, {8} bytes",
80 FileName, IsIndexMainAST, Syms.size(), Syms.bytes(),
Refs.
size(),
82 return std::make_tuple(std::move(Syms), std::move(
Refs),
83 std::move(Relations));
95 std::vector<Decl *> DeclsToIndex(
96 AST.getTranslationUnitDecl()->decls().begin(),
97 AST.getTranslationUnitDecl()->decls().end());
104 std::unique_ptr<RefSlab>
Refs,
105 std::unique_ptr<RelationSlab> Relations,
106 bool CountReferences) {
107 std::lock_guard<std::mutex> Lock(Mutex);
109 FileToSymbols.erase(Path);
111 FileToSymbols[
Path] = std::move(Symbols);
113 FileToRefs.erase(Path);
115 RefSlabAndCountReferences Item;
116 Item.CountReferences = CountReferences;
117 Item.Slab = std::move(Refs);
118 FileToRefs[
Path] = std::move(Item);
121 FileToRelations.erase(Path);
123 FileToRelations[
Path] = std::move(Relations);
126 std::unique_ptr<SymbolIndex>
128 std::vector<std::shared_ptr<SymbolSlab>> SymbolSlabs;
129 std::vector<std::shared_ptr<RefSlab>> RefSlabs;
130 std::vector<std::shared_ptr<RelationSlab>> RelationSlabs;
131 std::vector<RefSlab *> MainFileRefs;
133 std::lock_guard<std::mutex> Lock(Mutex);
134 for (
const auto &FileAndSymbols : FileToSymbols)
135 SymbolSlabs.push_back(FileAndSymbols.second);
136 for (
const auto &FileAndRefs : FileToRefs) {
137 RefSlabs.push_back(FileAndRefs.second.Slab);
138 if (FileAndRefs.second.CountReferences)
139 MainFileRefs.push_back(RefSlabs.back().get());
141 for (
const auto &FileAndRelations : FileToRelations)
142 RelationSlabs.push_back(FileAndRelations.second);
144 std::vector<const Symbol *> AllSymbols;
145 std::vector<Symbol> SymsStorage;
146 switch (DuplicateHandle) {
148 llvm::DenseMap<SymbolID, Symbol> Merged;
149 for (
const auto &Slab : SymbolSlabs) {
150 for (
const auto &Sym : *Slab) {
151 assert(Sym.References == 0 &&
152 "Symbol with non-zero references sent to FileSymbols");
153 auto I = Merged.try_emplace(Sym.ID, Sym);
155 I.first->second =
mergeSymbol(I.first->second, Sym);
159 for (
const auto &Sym : *
Refs) {
160 auto It = Merged.find(Sym.first);
162 if (It == Merged.end())
164 It->getSecond().References += Sym.second.size();
166 SymsStorage.reserve(Merged.size());
167 for (
auto &Sym : Merged) {
168 SymsStorage.push_back(std::move(Sym.second));
169 AllSymbols.push_back(&SymsStorage.back());
174 llvm::DenseSet<SymbolID> AddedSymbols;
175 for (
const auto &Slab : SymbolSlabs)
176 for (
const auto &Sym : *Slab) {
177 assert(Sym.References == 0 &&
178 "Symbol with non-zero references sent to FileSymbols");
179 if (AddedSymbols.insert(Sym.ID).second)
180 AllSymbols.push_back(&Sym);
186 std::vector<Ref> RefsStorage;
187 llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> AllRefs;
189 llvm::DenseMap<SymbolID, llvm::SmallVector<Ref, 4>> MergedRefs;
191 for (
const auto &
RefSlab : RefSlabs)
192 for (
const auto &Sym : *
RefSlab) {
193 MergedRefs[Sym.first].append(Sym.second.begin(), Sym.second.end());
194 Count += Sym.second.size();
196 RefsStorage.reserve(Count);
197 AllRefs.reserve(MergedRefs.size());
198 for (
auto &Sym : MergedRefs) {
199 auto &SymRefs = Sym.second;
202 llvm::copy(SymRefs, back_inserter(RefsStorage));
205 llvm::ArrayRef<Ref>(&RefsStorage[RefsStorage.size() - SymRefs.size()],
210 std::vector<Relation> AllRelations;
213 AllRelations.push_back(R);
217 RefsStorage.size() *
sizeof(
Ref) + SymsStorage.size() *
sizeof(
Symbol);
218 for (
const auto &Slab : SymbolSlabs)
219 StorageSize += Slab->bytes();
220 for (
const auto &
RefSlab : RefSlabs)
228 return std::make_unique<MemIndex>(
229 llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
230 std::move(AllRelations),
231 std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
232 std::move(RefsStorage), std::move(SymsStorage)),
235 return std::make_unique<dex::Dex>(
236 llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
237 std::move(AllRelations),
238 std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
239 std::move(RefsStorage), std::move(SymsStorage)),
242 llvm_unreachable(
"Unknown clangd::IndexType");
246 :
MergedIndex(&MainFileIndex, &PreambleIndex), UseDex(UseDex),
247 PreambleIndex(std::make_unique<
MemIndex>()),
248 MainFileIndex(std::make_unique<
MemIndex>()) {}
251 std::shared_ptr<Preprocessor>
PP,
255 Path, std::make_unique<SymbolSlab>(std::move(std::get<0>(Slabs))),
256 std::make_unique<RefSlab>(),
257 std::make_unique<RelationSlab>(std::move(std::get<2>(Slabs))),
267 Path, std::make_unique<SymbolSlab>(std::move(std::get<0>(
Contents))),
268 std::make_unique<RefSlab>(std::move(std::get<1>(
Contents))),
269 std::make_unique<RelationSlab>(std::move(std::get<2>(
Contents))),
std::tuple< SymbolSlab, RefSlab, RelationSlab > SlabTuple
bool CollectMacro
Collect macros.
SymbolCollector::Options CollectorOpts
An efficient structure of storing large set of symbol references in memory.
This defines Dex - a symbol index implementation based on query iterators over symbol tokens...
IndexType
Select between in-memory index implementations, which have tradeoffs.
Represents a symbol occurrence in the source file.
SlabTuple indexMainDecls(ParsedAST &AST)
Retrieves symbols and refs of local top level decls in AST (i.e.
Collect declarations (symbols) from an AST.
void updatePreamble(PathRef Path, ASTContext &AST, std::shared_ptr< Preprocessor > PP, const CanonicalIncludes &Includes)
Update preamble symbols of file Path with all declarations in AST and macros in PP.
llvm::StringRef PathRef
A typedef to represent a ref to file path.
ArrayRef< Decl * > getLocalTopLevelDecls()
This function returns top-level decls present in the main file of the AST.
void vlog(const char *Fmt, Ts &&... Vals)
void updateMain(PathRef Path, ParsedAST &AST)
Update symbols and references from main file Path with indexMainDecls.
ASTContext & getASTContext()
Note that the returned ast will not contain decls from the preamble that were not deserialized during...
DuplicateHandling
How to handle duplicated symbols across multiple files.
Maps a definition location onto an #include file, based on a set of filename rules.
const CanonicalIncludes & getCanonicalIncludes() const
bool StoreAllDocumentation
If set to true, SymbolCollector will collect doc for all symbols.
MemIndex is a naive in-memory index suitable for a small set of symbols.
std::shared_ptr< Preprocessor > getPreprocessorPtr()
std::string Path
A typedef to represent a file path.
size_t size() const
Gets the number of symbols.
std::unique_ptr< SymbolIndex > buildIndex(IndexType, DuplicateHandling DuplicateHandle=DuplicateHandling::PickOne)
The index keeps the symbols alive.
Symbol mergeSymbol(const Symbol &L, const Symbol &R)
std::shared_ptr< SymbolCollector > Collector
void handleMacros(const MainFileMacros &MacroRefsToIndex)
Stores and provides access to parsed AST.
FileIndex(bool UseDex=true)
static SlabTuple indexSymbols(ASTContext &AST, std::shared_ptr< Preprocessor > PP, llvm::ArrayRef< Decl *> DeclsToIndex, const MainFileMacros *MacroRefsToIndex, const CanonicalIncludes &Includes, bool IsIndexMainAST)
const MainFileMacros & getMacros() const
Gets all macro references (definition, expansions) present in the main file, including those in the p...
The class presents a C++ symbol, e.g.
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
RelationSlab takeRelations()
void setPreprocessor(std::shared_ptr< Preprocessor > PP) override
void update(PathRef Path, std::unique_ptr< SymbolSlab > Slab, std::unique_ptr< RefSlab > Refs, std::unique_ptr< RelationSlab > Relations, bool CountReferences)
Updates all symbols and refs in a file.
const CanonicalIncludes * Includes
If set, this is used to map symbol #include path to a potentially different #include path...
void reset(std::unique_ptr< SymbolIndex >)
SlabTuple indexHeaderSymbols(ASTContext &AST, std::shared_ptr< Preprocessor > PP, const CanonicalIncludes &Includes)
Index declarations from AST and macros from PP that are declared in included headers.
RefKind RefFilter
The symbol ref kinds that will be collected.