23 #include "llvm/ADT/StringExtras.h"
24 #include "llvm/ADT/StringMap.h"
25 #include "llvm/Support/EndianStream.h"
26 #include "llvm/Support/FileSystem.h"
27 #include "llvm/Support/MemoryBuffer.h"
28 #include "llvm/Support/OnDiskHashTable.h"
29 #include "llvm/Support/Path.h"
30 #include "llvm/Support/raw_ostream.h"
34 #define S_ISDIR(x) (((x)&_S_IFDIR)!=0)
37 using namespace clang;
47 Offset TokenData, PPCondData;
53 : TokenData(td), PPCondData(ppcd) {}
55 Offset getTokenOffset()
const {
return TokenData; }
56 Offset getPPCondTableOffset()
const {
return PPCondData; }
60 class PTHEntryKeyVariant {
61 union {
const FileEntry* FE;
const char* Path; };
62 enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 }
Kind;
66 PTHEntryKeyVariant(
const FileEntry *fe) : FE(fe),
Kind(IsFE), Data(
nullptr) {}
68 PTHEntryKeyVariant(
FileData *Data,
const char *path)
71 explicit PTHEntryKeyVariant(
const char *path)
72 : Path(path),
Kind(IsNoExist), Data(
nullptr) {}
74 bool isFile()
const {
return Kind == IsFE; }
76 StringRef getString()
const {
77 return Kind == IsFE ? FE->getName() : Path;
80 unsigned getKind()
const {
return (
unsigned)
Kind; }
82 void EmitData(raw_ostream& Out) {
83 using namespace llvm::support;
84 endian::Writer<little> LE(Out);
88 llvm::sys::fs::UniqueID UID = FE->getUniqueID();
89 LE.write<uint64_t>(UID.getFile());
90 LE.write<uint64_t>(UID.getDevice());
91 LE.write<uint64_t>(FE->getModificationTime());
92 LE.write<uint64_t>(FE->getSize());
96 LE.write<uint64_t>(Data->
UniqueID.getFile());
97 LE.write<uint64_t>(Data->
UniqueID.getDevice());
98 LE.write<uint64_t>(Data->
ModTime);
99 LE.write<uint64_t>(Data->
Size);
107 unsigned getRepresentationLength()
const {
108 return Kind == IsNoExist ? 0 : 4 + 4 + 2 + 8 + 8;
112 class FileEntryPTHEntryInfo {
114 typedef PTHEntryKeyVariant key_type;
115 typedef key_type key_type_ref;
117 typedef PTHEntry data_type;
118 typedef const PTHEntry& data_type_ref;
120 typedef unsigned hash_value_type;
121 typedef unsigned offset_type;
123 static hash_value_type
ComputeHash(PTHEntryKeyVariant V) {
124 return llvm::HashString(V.getString());
127 static std::pair<unsigned,unsigned>
128 EmitKeyDataLength(raw_ostream& Out, PTHEntryKeyVariant V,
130 using namespace llvm::support;
131 endian::Writer<little> LE(Out);
133 unsigned n = V.getString().size() + 1 + 1;
134 LE.write<uint16_t>(n);
136 unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0);
137 LE.write<uint8_t>(m);
139 return std::make_pair(n, m);
142 static void EmitKey(raw_ostream& Out, PTHEntryKeyVariant V,
unsigned n){
143 using namespace llvm::support;
145 endian::Writer<little>(Out).write<uint8_t>((
unsigned)V.getKind());
147 Out.write(V.getString().data(), n - 1);
150 static void EmitData(raw_ostream& Out, PTHEntryKeyVariant V,
151 const PTHEntry& E,
unsigned) {
152 using namespace llvm::support;
153 endian::Writer<little> LE(Out);
158 LE.write<uint32_t>(E.getTokenOffset());
159 LE.write<uint32_t>(E.getPPCondTableOffset());
171 OffsetOpt() : valid(
false) {}
172 bool hasOffset()
const {
return valid; }
173 Offset getOffset()
const { assert(valid);
return off; }
174 void setOffset(
Offset o) { off = o; valid =
true; }
178 typedef llvm::OnDiskChainedHashTableGenerator<FileEntryPTHEntryInfo>
PTHMap;
182 typedef llvm::DenseMap<const IdentifierInfo*,uint32_t> IDMap;
183 typedef llvm::StringMap<OffsetOpt, llvm::BumpPtrAllocator> CachedStrsTy;
186 raw_pwrite_stream &Out;
190 CachedStrsTy CachedStrs;
192 std::vector<llvm::StringMapEntry<OffsetOpt>*> StrEntries;
198 void EmitToken(
const Token& T);
200 void Emit8(uint32_t V) {
201 using namespace llvm::support;
202 endian::Writer<little>(Out).write<uint8_t>(V);
205 void Emit16(uint32_t V) {
206 using namespace llvm::support;
207 endian::Writer<little>(Out).write<uint16_t>(V);
210 void Emit32(uint32_t V) {
211 using namespace llvm::support;
212 endian::Writer<little>(Out).write<uint32_t>(V);
215 void EmitBuf(
const char *Ptr,
unsigned NumBytes) {
216 Out.write(Ptr, NumBytes);
220 using namespace llvm::support;
221 endian::Writer<little>(Out).write<uint16_t>(V.size());
222 EmitBuf(V.data(), V.size());
229 std::pair<Offset, Offset> EmitIdentifierTable();
233 Offset EmitFileTable() {
return PM.Emit(Out); }
235 PTHEntry LexTokens(
Lexer& L);
236 Offset EmitCachedSpellings();
240 : Out(out), PP(pp), idcount(0), CurStrOffset(0) {}
242 PTHMap &getPM() {
return PM; }
252 IDMap::iterator I = IM.find(II);
260 void PTHWriter::EmitToken(
const Token& T) {
263 (((uint32_t) T.getLength()) << 16));
273 auto &E = *CachedStrs.insert(std::make_pair(s, OffsetOpt())).first;
276 if (!E.second.hasOffset()) {
277 E.second.setOffset(CurStrOffset);
278 StrEntries.push_back(&E);
279 CurStrOffset += s.size() + 1;
283 Emit32(E.second.getOffset());
288 Emit32(PP.getSourceManager().getFileOffset(T.
getLocation()));
291 PTHEntry PTHWriter::LexTokens(
Lexer& L) {
294 using namespace llvm::support;
295 endian::Writer<little> LE(Out);
296 uint32_t TokenOff = Out.tell();
297 for (uint64_t N = llvm::OffsetToAlignment(TokenOff, 4); N; --N, ++TokenOff)
298 LE.write<uint8_t>(0);
301 typedef std::vector<std::pair<Offset, unsigned> > PPCondTable;
303 std::vector<unsigned> PPStartCond;
304 bool ParsingPreprocessorDirective =
false;
311 if ((Tok.isAtStartOfLine() || Tok.is(
tok::eof)) &&
312 ParsingPreprocessorDirective) {
322 ParsingPreprocessorDirective =
false;
325 if (Tok.is(tok::raw_identifier)) {
326 PP.LookUpIdentifierInfo(Tok);
331 if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
334 assert(!ParsingPreprocessorDirective);
351 if (Tok.isNot(tok::raw_identifier)) {
359 ParsingPreprocessorDirective =
true;
362 case tok::pp_not_keyword:
368 case tok::pp_include:
370 case tok::pp_include_next: {
377 assert(!Tok.isAtStartOfLine());
378 if (Tok.is(tok::raw_identifier))
379 PP.LookUpIdentifierInfo(Tok);
385 case tok::pp_ifndef: {
388 PPStartCond.push_back(PPCond.size());
389 PPCond.push_back(std::make_pair(HashOff, 0U));
392 case tok::pp_endif: {
396 unsigned index = PPCond.size();
398 assert(!PPStartCond.empty());
399 assert(PPCond.size() > PPStartCond.back());
400 assert(PPCond[PPStartCond.back()].second == 0);
401 PPCond[PPStartCond.back()].second = index;
402 PPStartCond.pop_back();
404 PPCond.push_back(std::make_pair(HashOff, index));
411 while (Tok.isNot(
tok::eof) && !Tok.isAtStartOfLine());
421 unsigned index = PPCond.size();
423 assert(!PPStartCond.empty());
424 assert(PPCond.size() > PPStartCond.back());
425 assert(PPCond[PPStartCond.back()].second == 0);
426 PPCond[PPStartCond.back()].second = index;
427 PPStartCond.pop_back();
429 PPCond.push_back(std::make_pair(HashOff, 0U));
430 PPStartCond.push_back(index);
440 assert(PPStartCond.empty() &&
"Error: imblanced preprocessor conditionals.");
446 Emit32(PPCond.size());
448 for (
unsigned i = 0, e = PPCond.size(); i!=e; ++i) {
449 Emit32(PPCond[i].first - TokenOff);
450 uint32_t x = PPCond[i].second;
451 assert(x != 0 &&
"PPCond entry not backpatched.");
454 Emit32(x == i ? 0 : x);
457 return PTHEntry(TokenOff, PPCondOff);
460 Offset PTHWriter::EmitCachedSpellings() {
462 Offset SpellingsOff = Out.tell();
464 for (std::vector<llvm::StringMapEntry<OffsetOpt>*>::iterator
465 I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I)
466 EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 );
472 return llvm::support::endian::byte_swap<uint32_t, llvm::support::little>(
X);
475 static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off) {
477 OS.pwrite(reinterpret_cast<const char *>(&LEVal), 4, Off);
483 Out <<
"cfe-pth" <<
'\0';
487 Offset PrologueOffset = Out.tell();
488 for (
unsigned i = 0; i < 4; ++i)
492 if (!MainFile.empty()) {
507 const SrcMgr::ContentCache &
C = *I->second;
511 if (llvm::sys::path::is_relative(FE->
getName()))
514 const llvm::MemoryBuffer *B = C.getBuffer(PP.getDiagnostics(),
SM);
518 const llvm::MemoryBuffer *FromFile = SM.
getBuffer(FID);
519 Lexer L(FID, FromFile, SM, LOpts);
520 PM.insert(FE, LexTokens(L));
524 const std::pair<Offset,Offset> &IdTableOff = EmitIdentifierTable();
527 Offset SpellingOff = EmitCachedSpellings();
530 Offset FileTableOff = EmitFileTable();
533 uint64_t Off = PrologueOffset;
549 StatListener(
PTHMap &pm) : PM(pm) {}
550 ~StatListener()
override {}
553 std::unique_ptr<vfs::File> *F,
557 if (Result == CacheMissing)
558 PM.insert(PTHEntryKeyVariant(Path), PTHEntry());
561 if (llvm::sys::path::is_relative(Path))
564 PM.insert(PTHEntryKeyVariant(&Data, Path), PTHEntry());
578 llvm::sys::fs::make_absolute(MainFilePath);
581 PTHWriter PW(*OS, PP);
584 auto StatCacheOwner = llvm::make_unique<StatListener>(PW.getPM());
585 StatListener *StatCache = StatCacheOwner.get();
597 PW.GeneratePTH(MainFilePath.str());
609 class PTHIdentifierTableTrait {
611 typedef PTHIdKey* key_type;
612 typedef key_type key_type_ref;
614 typedef uint32_t data_type;
615 typedef data_type data_type_ref;
617 typedef unsigned hash_value_type;
618 typedef unsigned offset_type;
620 static hash_value_type
ComputeHash(PTHIdKey* key) {
621 return llvm::HashString(key->II->getName());
624 static std::pair<unsigned,unsigned>
625 EmitKeyDataLength(raw_ostream& Out,
const PTHIdKey* key, uint32_t) {
626 using namespace llvm::support;
627 unsigned n = key->II->getLength() + 1;
628 endian::Writer<little>(Out).write<uint16_t>(n);
629 return std::make_pair(n,
sizeof(uint32_t));
632 static void EmitKey(raw_ostream& Out, PTHIdKey* key,
unsigned n) {
635 key->FileOffset = Out.tell();
636 Out.write(key->II->getNameStart(), n);
639 static void EmitData(raw_ostream& Out, PTHIdKey*, uint32_t pID,
641 using namespace llvm::support;
642 endian::Writer<little>(Out).write<uint32_t>(pID);
652 std::pair<Offset,Offset> PTHWriter::EmitIdentifierTable() {
658 PTHIdKey *IIDMap = (PTHIdKey*)calloc(idcount,
sizeof(PTHIdKey));
661 llvm::OnDiskChainedHashTableGenerator<PTHIdentifierTableTrait> IIOffMap;
664 for (IDMap::iterator I = IM.begin(), E = IM.end(); I != E; ++I) {
667 assert(I->second > 0);
668 assert(I->second-1 < idcount);
669 unsigned idx = I->second-1;
672 IIDMap[idx].II = I->first;
675 IIOffMap.insert(&IIDMap[idx], I->second);
681 Offset StringTableOffset = IIOffMap.Emit(Out);
684 Offset IDOff = Out.tell();
686 for (
unsigned i = 0 ; i < idcount; ++i)
687 Emit32(IIDMap[i].FileOffset);
692 return std::make_pair(IDOff, StringTableOffset);
bool isAtStartOfLine() const
SourceManager & getSourceManager() const
Defines the clang::FileManager interface and associated types.
bool LexFromRawLexer(Token &Result)
Defines the SourceManager interface.
Defines the FileSystemStatCache interface.
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
fileinfo_iterator fileinfo_begin() const
The virtual file system interface.
static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off)
llvm::OnDiskChainedHashTableGenerator< FileEntryPTHEntryInfo > PTHMap
void setKind(tok::TokenKind K)
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
FileManager & getFileManager() const
Abstract interface for introducing a FileManager cache for 'stat' system calls, which is used by prec...
Represents the results of name lookup.
void setParsingPreprocessorDirective(bool f)
Inform the lexer whether or not we are currently lexing a preprocessor directive. ...
void CacheTokens(Preprocessor &PP, raw_pwrite_stream *OS)
Cache tokens for use with PCH. Note that this requires a seekable stream.
tok::TokenKind getKind() const
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
Defines the clang::Preprocessor interface.
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool isNot(tok::TokenKind K) const
The result type of a method or function.
const char * getLiteralData() const
static uint32_t swap32le(uint32_t X)
void addStatCache(std::unique_ptr< FileSystemStatCache > statCache, bool AtBeginning=false)
Installs the provided FileSystemStatCache object within the FileManager.
const char * getName() const
Encodes a location in the source. The SourceManager can decode this to get at the full include stack...
llvm::sys::fs::UniqueID UniqueID
Cached information about one file (either on disk or in the virtual file system). ...
void setIdentifierInfo(IdentifierInfo *II)
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
FileID getMainFileID() const
Returns the FileID of the main source file.
unsigned ComputeHash(Selector Sel)
unsigned getFlags() const
Return the internal represtation of the flags.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T-> getSizeExpr()))
void removeStatCache(FileSystemStatCache *statCache)
Removes the specified FileSystemStatCache object from the manager.
fileinfo_iterator fileinfo_end() const
llvm::DenseMap< const FileEntry *, SrcMgr::ContentCache * >::const_iterator fileinfo_iterator
Defines the Diagnostic-related interfaces.
void LexIncludeFilename(Token &Result)
After the preprocessor has parsed a #include, lex and (potentially) macro expand the filename...
raw_ostream & EmitString(raw_ostream &o, StringRef s)
static Decl::Kind getKind(const Decl *D)
unsigned getLength() const
Generate pre-tokenized header.
void clearFlag(TokenFlags Flag)
Unset the specified flag.
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
IdentifierInfo * getIdentifierInfo() const
tok::PPKeywordKind getPPKeywordID() const
Return the preprocessor keyword ID for this identifier.