clang  3.7.0
VirtualFileSystem.cpp
Go to the documentation of this file.
1 //===- VirtualFileSystem.cpp - Virtual File System Layer --------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 // This file implements the VirtualFileSystem interface.
10 //===----------------------------------------------------------------------===//
11 
13 #include "llvm/ADT/DenseMap.h"
14 #include "llvm/ADT/STLExtras.h"
15 #include "llvm/ADT/StringExtras.h"
16 #include "llvm/ADT/StringSet.h"
17 #include "llvm/ADT/iterator_range.h"
18 #include "llvm/Support/Errc.h"
19 #include "llvm/Support/MemoryBuffer.h"
20 #include "llvm/Support/Path.h"
21 #include "llvm/Support/YAMLParser.h"
22 #include <atomic>
23 #include <memory>
24 
25 using namespace clang;
26 using namespace clang::vfs;
27 using namespace llvm;
28 using llvm::sys::fs::file_status;
29 using llvm::sys::fs::file_type;
30 using llvm::sys::fs::perms;
31 using llvm::sys::fs::UniqueID;
32 
33 Status::Status(const file_status &Status)
34  : UID(Status.getUniqueID()), MTime(Status.getLastModificationTime()),
35  User(Status.getUser()), Group(Status.getGroup()), Size(Status.getSize()),
36  Type(Status.type()), Perms(Status.permissions()), IsVFSMapped(false) {}
37 
38 Status::Status(StringRef Name, StringRef ExternalName, UniqueID UID,
39  sys::TimeValue MTime, uint32_t User, uint32_t Group,
40  uint64_t Size, file_type Type, perms Perms)
41  : Name(Name), UID(UID), MTime(MTime), User(User), Group(Group), Size(Size),
42  Type(Type), Perms(Perms), IsVFSMapped(false) {}
43 
44 bool Status::equivalent(const Status &Other) const {
45  return getUniqueID() == Other.getUniqueID();
46 }
47 bool Status::isDirectory() const {
48  return Type == file_type::directory_file;
49 }
50 bool Status::isRegularFile() const {
51  return Type == file_type::regular_file;
52 }
53 bool Status::isOther() const {
54  return exists() && !isRegularFile() && !isDirectory() && !isSymlink();
55 }
56 bool Status::isSymlink() const {
57  return Type == file_type::symlink_file;
58 }
59 bool Status::isStatusKnown() const {
60  return Type != file_type::status_error;
61 }
62 bool Status::exists() const {
63  return isStatusKnown() && Type != file_type::file_not_found;
64 }
65 
66 File::~File() {}
67 
68 FileSystem::~FileSystem() {}
69 
70 ErrorOr<std::unique_ptr<MemoryBuffer>>
71 FileSystem::getBufferForFile(const llvm::Twine &Name, int64_t FileSize,
72  bool RequiresNullTerminator, bool IsVolatile) {
73  auto F = openFileForRead(Name);
74  if (!F)
75  return F.getError();
76 
77  return (*F)->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile);
78 }
79 
80 //===-----------------------------------------------------------------------===/
81 // RealFileSystem implementation
82 //===-----------------------------------------------------------------------===/
83 
84 namespace {
85 /// \brief Wrapper around a raw file descriptor.
86 class RealFile : public File {
87  int FD;
88  Status S;
89  friend class RealFileSystem;
90  RealFile(int FD) : FD(FD) {
91  assert(FD >= 0 && "Invalid or inactive file descriptor");
92  }
93 
94 public:
95  ~RealFile() override;
96  ErrorOr<Status> status() override;
97  ErrorOr<std::unique_ptr<MemoryBuffer>>
98  getBuffer(const Twine &Name, int64_t FileSize = -1,
99  bool RequiresNullTerminator = true,
100  bool IsVolatile = false) override;
101  std::error_code close() override;
102  void setName(StringRef Name) override;
103 };
104 } // end anonymous namespace
105 RealFile::~RealFile() { close(); }
106 
107 ErrorOr<Status> RealFile::status() {
108  assert(FD != -1 && "cannot stat closed file");
109  if (!S.isStatusKnown()) {
110  file_status RealStatus;
111  if (std::error_code EC = sys::fs::status(FD, RealStatus))
112  return EC;
113  Status NewS(RealStatus);
114  NewS.setName(S.getName());
115  S = std::move(NewS);
116  }
117  return S;
118 }
119 
120 ErrorOr<std::unique_ptr<MemoryBuffer>>
121 RealFile::getBuffer(const Twine &Name, int64_t FileSize,
122  bool RequiresNullTerminator, bool IsVolatile) {
123  assert(FD != -1 && "cannot get buffer for closed file");
124  return MemoryBuffer::getOpenFile(FD, Name, FileSize, RequiresNullTerminator,
125  IsVolatile);
126 }
127 
128 // FIXME: This is terrible, we need this for ::close.
129 #if !defined(_MSC_VER) && !defined(__MINGW32__)
130 #include <unistd.h>
131 #include <sys/uio.h>
132 #else
133 #include <io.h>
134 #ifndef S_ISFIFO
135 #define S_ISFIFO(x) (0)
136 #endif
137 #endif
138 std::error_code RealFile::close() {
139  if (::close(FD))
140  return std::error_code(errno, std::generic_category());
141  FD = -1;
142  return std::error_code();
143 }
144 
145 void RealFile::setName(StringRef Name) {
146  S.setName(Name);
147 }
148 
149 namespace {
150 /// \brief The file system according to your operating system.
151 class RealFileSystem : public FileSystem {
152 public:
153  ErrorOr<Status> status(const Twine &Path) override;
154  ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
155  directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
156 };
157 } // end anonymous namespace
158 
159 ErrorOr<Status> RealFileSystem::status(const Twine &Path) {
160  sys::fs::file_status RealStatus;
161  if (std::error_code EC = sys::fs::status(Path, RealStatus))
162  return EC;
163  Status Result(RealStatus);
164  Result.setName(Path.str());
165  return Result;
166 }
167 
168 ErrorOr<std::unique_ptr<File>>
169 RealFileSystem::openFileForRead(const Twine &Name) {
170  int FD;
171  if (std::error_code EC = sys::fs::openFileForRead(Name, FD))
172  return EC;
173  std::unique_ptr<File> Result(new RealFile(FD));
174  Result->setName(Name.str());
175  return std::move(Result);
176 }
177 
179  static IntrusiveRefCntPtr<FileSystem> FS = new RealFileSystem();
180  return FS;
181 }
182 
183 namespace {
184 class RealFSDirIter : public clang::vfs::detail::DirIterImpl {
185  std::string Path;
186  llvm::sys::fs::directory_iterator Iter;
187 public:
188  RealFSDirIter(const Twine &_Path, std::error_code &EC)
189  : Path(_Path.str()), Iter(Path, EC) {
190  if (!EC && Iter != llvm::sys::fs::directory_iterator()) {
191  llvm::sys::fs::file_status S;
192  EC = Iter->status(S);
193  if (!EC) {
194  CurrentEntry = Status(S);
195  CurrentEntry.setName(Iter->path());
196  }
197  }
198  }
199 
200  std::error_code increment() override {
201  std::error_code EC;
202  Iter.increment(EC);
203  if (EC) {
204  return EC;
205  } else if (Iter == llvm::sys::fs::directory_iterator()) {
206  CurrentEntry = Status();
207  } else {
208  llvm::sys::fs::file_status S;
209  EC = Iter->status(S);
210  CurrentEntry = Status(S);
211  CurrentEntry.setName(Iter->path());
212  }
213  return EC;
214  }
215 };
216 }
217 
218 directory_iterator RealFileSystem::dir_begin(const Twine &Dir,
219  std::error_code &EC) {
220  return directory_iterator(std::make_shared<RealFSDirIter>(Dir, EC));
221 }
222 
223 //===-----------------------------------------------------------------------===/
224 // OverlayFileSystem implementation
225 //===-----------------------------------------------------------------------===/
226 OverlayFileSystem::OverlayFileSystem(IntrusiveRefCntPtr<FileSystem> BaseFS) {
227  pushOverlay(BaseFS);
228 }
229 
230 void OverlayFileSystem::pushOverlay(IntrusiveRefCntPtr<FileSystem> FS) {
231  FSList.push_back(FS);
232 }
233 
234 ErrorOr<Status> OverlayFileSystem::status(const Twine &Path) {
235  // FIXME: handle symlinks that cross file systems
236  for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) {
237  ErrorOr<Status> Status = (*I)->status(Path);
238  if (Status || Status.getError() != llvm::errc::no_such_file_or_directory)
239  return Status;
240  }
241  return make_error_code(llvm::errc::no_such_file_or_directory);
242 }
243 
244 ErrorOr<std::unique_ptr<File>>
245 OverlayFileSystem::openFileForRead(const llvm::Twine &Path) {
246  // FIXME: handle symlinks that cross file systems
247  for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) {
248  auto Result = (*I)->openFileForRead(Path);
249  if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
250  return Result;
251  }
252  return make_error_code(llvm::errc::no_such_file_or_directory);
253 }
254 
256 
257 namespace {
258 class OverlayFSDirIterImpl : public clang::vfs::detail::DirIterImpl {
259  OverlayFileSystem &Overlays;
260  std::string Path;
261  OverlayFileSystem::iterator CurrentFS;
262  directory_iterator CurrentDirIter;
263  llvm::StringSet<> SeenNames;
264 
265  std::error_code incrementFS() {
266  assert(CurrentFS != Overlays.overlays_end() && "incrementing past end");
267  ++CurrentFS;
268  for (auto E = Overlays.overlays_end(); CurrentFS != E; ++CurrentFS) {
269  std::error_code EC;
270  CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC);
271  if (EC && EC != errc::no_such_file_or_directory)
272  return EC;
273  if (CurrentDirIter != directory_iterator())
274  break; // found
275  }
276  return std::error_code();
277  }
278 
279  std::error_code incrementDirIter(bool IsFirstTime) {
280  assert((IsFirstTime || CurrentDirIter != directory_iterator()) &&
281  "incrementing past end");
282  std::error_code EC;
283  if (!IsFirstTime)
284  CurrentDirIter.increment(EC);
285  if (!EC && CurrentDirIter == directory_iterator())
286  EC = incrementFS();
287  return EC;
288  }
289 
290  std::error_code incrementImpl(bool IsFirstTime) {
291  while (true) {
292  std::error_code EC = incrementDirIter(IsFirstTime);
293  if (EC || CurrentDirIter == directory_iterator()) {
294  CurrentEntry = Status();
295  return EC;
296  }
297  CurrentEntry = *CurrentDirIter;
298  StringRef Name = llvm::sys::path::filename(CurrentEntry.getName());
299  if (SeenNames.insert(Name).second)
300  return EC; // name not seen before
301  }
302  llvm_unreachable("returned above");
303  }
304 
305 public:
306  OverlayFSDirIterImpl(const Twine &Path, OverlayFileSystem &FS,
307  std::error_code &EC)
308  : Overlays(FS), Path(Path.str()), CurrentFS(Overlays.overlays_begin()) {
309  CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC);
310  EC = incrementImpl(true);
311  }
312 
313  std::error_code increment() override { return incrementImpl(false); }
314 };
315 } // end anonymous namespace
316 
317 directory_iterator OverlayFileSystem::dir_begin(const Twine &Dir,
318  std::error_code &EC) {
319  return directory_iterator(
320  std::make_shared<OverlayFSDirIterImpl>(Dir, *this, EC));
321 }
322 
323 //===-----------------------------------------------------------------------===/
324 // VFSFromYAML implementation
325 //===-----------------------------------------------------------------------===/
326 
327 namespace {
328 
329 enum EntryKind {
330  EK_Directory,
331  EK_File
332 };
333 
334 /// \brief A single file or directory in the VFS.
335 class Entry {
336  EntryKind Kind;
337  std::string Name;
338 
339 public:
340  virtual ~Entry();
341  Entry(EntryKind K, StringRef Name) : Kind(K), Name(Name) {}
342  StringRef getName() const { return Name; }
343  EntryKind getKind() const { return Kind; }
344 };
345 
346 class DirectoryEntry : public Entry {
347  std::vector<Entry *> Contents;
348  Status S;
349 
350 public:
351  ~DirectoryEntry() override;
352  DirectoryEntry(StringRef Name, std::vector<Entry *> Contents, Status S)
353  : Entry(EK_Directory, Name), Contents(std::move(Contents)),
354  S(std::move(S)) {}
355  Status getStatus() { return S; }
356  typedef std::vector<Entry *>::iterator iterator;
357  iterator contents_begin() { return Contents.begin(); }
358  iterator contents_end() { return Contents.end(); }
359  static bool classof(const Entry *E) { return E->getKind() == EK_Directory; }
360 };
361 
362 class FileEntry : public Entry {
363 public:
364  enum NameKind {
365  NK_NotSet,
366  NK_External,
367  NK_Virtual
368  };
369 private:
370  std::string ExternalContentsPath;
371  NameKind UseName;
372 public:
373  FileEntry(StringRef Name, StringRef ExternalContentsPath, NameKind UseName)
374  : Entry(EK_File, Name), ExternalContentsPath(ExternalContentsPath),
375  UseName(UseName) {}
376  StringRef getExternalContentsPath() const { return ExternalContentsPath; }
377  /// \brief whether to use the external path as the name for this file.
378  bool useExternalName(bool GlobalUseExternalName) const {
379  return UseName == NK_NotSet ? GlobalUseExternalName
380  : (UseName == NK_External);
381  }
382  static bool classof(const Entry *E) { return E->getKind() == EK_File; }
383 };
384 
385 class VFSFromYAML;
386 
387 class VFSFromYamlDirIterImpl : public clang::vfs::detail::DirIterImpl {
388  std::string Dir;
389  VFSFromYAML &FS;
390  DirectoryEntry::iterator Current, End;
391 public:
392  VFSFromYamlDirIterImpl(const Twine &Path, VFSFromYAML &FS,
393  DirectoryEntry::iterator Begin,
394  DirectoryEntry::iterator End, std::error_code &EC);
395  std::error_code increment() override;
396 };
397 
398 /// \brief A virtual file system parsed from a YAML file.
399 ///
400 /// Currently, this class allows creating virtual directories and mapping
401 /// virtual file paths to existing external files, available in \c ExternalFS.
402 ///
403 /// The basic structure of the parsed file is:
404 /// \verbatim
405 /// {
406 /// 'version': <version number>,
407 /// <optional configuration>
408 /// 'roots': [
409 /// <directory entries>
410 /// ]
411 /// }
412 /// \endverbatim
413 ///
414 /// All configuration options are optional.
415 /// 'case-sensitive': <boolean, default=true>
416 /// 'use-external-names': <boolean, default=true>
417 ///
418 /// Virtual directories are represented as
419 /// \verbatim
420 /// {
421 /// 'type': 'directory',
422 /// 'name': <string>,
423 /// 'contents': [ <file or directory entries> ]
424 /// }
425 /// \endverbatim
426 ///
427 /// The default attributes for virtual directories are:
428 /// \verbatim
429 /// MTime = now() when created
430 /// Perms = 0777
431 /// User = Group = 0
432 /// Size = 0
433 /// UniqueID = unspecified unique value
434 /// \endverbatim
435 ///
436 /// Re-mapped files are represented as
437 /// \verbatim
438 /// {
439 /// 'type': 'file',
440 /// 'name': <string>,
441 /// 'use-external-name': <boolean> # Optional
442 /// 'external-contents': <path to external file>)
443 /// }
444 /// \endverbatim
445 ///
446 /// and inherit their attributes from the external contents.
447 ///
448 /// In both cases, the 'name' field may contain multiple path components (e.g.
449 /// /path/to/file). However, any directory that contains more than one child
450 /// must be uniquely represented by a directory entry.
451 class VFSFromYAML : public vfs::FileSystem {
452  std::vector<Entry *> Roots; ///< The root(s) of the virtual file system.
453  /// \brief The file system to use for external references.
455 
456  /// @name Configuration
457  /// @{
458 
459  /// \brief Whether to perform case-sensitive comparisons.
460  ///
461  /// Currently, case-insensitive matching only works correctly with ASCII.
462  bool CaseSensitive;
463 
464  /// \brief Whether to use to use the value of 'external-contents' for the
465  /// names of files. This global value is overridable on a per-file basis.
466  bool UseExternalNames;
467  /// @}
468 
469  friend class VFSFromYAMLParser;
470 
471 private:
472  VFSFromYAML(IntrusiveRefCntPtr<FileSystem> ExternalFS)
473  : ExternalFS(ExternalFS), CaseSensitive(true), UseExternalNames(true) {}
474 
475  /// \brief Looks up \p Path in \c Roots.
476  ErrorOr<Entry *> lookupPath(const Twine &Path);
477 
478  /// \brief Looks up the path <tt>[Start, End)</tt> in \p From, possibly
479  /// recursing into the contents of \p From if it is a directory.
480  ErrorOr<Entry *> lookupPath(sys::path::const_iterator Start,
481  sys::path::const_iterator End, Entry *From);
482 
483  /// \brief Get the status of a given an \c Entry.
484  ErrorOr<Status> status(const Twine &Path, Entry *E);
485 
486 public:
487  ~VFSFromYAML() override;
488 
489  /// \brief Parses \p Buffer, which is expected to be in YAML format and
490  /// returns a virtual file system representing its contents.
491  static VFSFromYAML *create(std::unique_ptr<MemoryBuffer> Buffer,
492  SourceMgr::DiagHandlerTy DiagHandler,
493  void *DiagContext,
494  IntrusiveRefCntPtr<FileSystem> ExternalFS);
495 
496  ErrorOr<Status> status(const Twine &Path) override;
497  ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
498 
499  directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override{
500  ErrorOr<Entry *> E = lookupPath(Dir);
501  if (!E) {
502  EC = E.getError();
503  return directory_iterator();
504  }
505  ErrorOr<Status> S = status(Dir, *E);
506  if (!S) {
507  EC = S.getError();
508  return directory_iterator();
509  }
510  if (!S->isDirectory()) {
511  EC = std::error_code(static_cast<int>(errc::not_a_directory),
512  std::system_category());
513  return directory_iterator();
514  }
515 
516  DirectoryEntry *D = cast<DirectoryEntry>(*E);
517  return directory_iterator(std::make_shared<VFSFromYamlDirIterImpl>(Dir,
518  *this, D->contents_begin(), D->contents_end(), EC));
519  }
520 };
521 
522 /// \brief A helper class to hold the common YAML parsing state.
523 class VFSFromYAMLParser {
524  yaml::Stream &Stream;
525 
526  void error(yaml::Node *N, const Twine &Msg) {
527  Stream.printError(N, Msg);
528  }
529 
530  // false on error
531  bool parseScalarString(yaml::Node *N, StringRef &Result,
532  SmallVectorImpl<char> &Storage) {
533  yaml::ScalarNode *S = dyn_cast<yaml::ScalarNode>(N);
534  if (!S) {
535  error(N, "expected string");
536  return false;
537  }
538  Result = S->getValue(Storage);
539  return true;
540  }
541 
542  // false on error
543  bool parseScalarBool(yaml::Node *N, bool &Result) {
544  SmallString<5> Storage;
545  StringRef Value;
546  if (!parseScalarString(N, Value, Storage))
547  return false;
548 
549  if (Value.equals_lower("true") || Value.equals_lower("on") ||
550  Value.equals_lower("yes") || Value == "1") {
551  Result = true;
552  return true;
553  } else if (Value.equals_lower("false") || Value.equals_lower("off") ||
554  Value.equals_lower("no") || Value == "0") {
555  Result = false;
556  return true;
557  }
558 
559  error(N, "expected boolean value");
560  return false;
561  }
562 
563  struct KeyStatus {
564  KeyStatus(bool Required=false) : Required(Required), Seen(false) {}
565  bool Required;
566  bool Seen;
567  };
568  typedef std::pair<StringRef, KeyStatus> KeyStatusPair;
569 
570  // false on error
571  bool checkDuplicateOrUnknownKey(yaml::Node *KeyNode, StringRef Key,
572  DenseMap<StringRef, KeyStatus> &Keys) {
573  if (!Keys.count(Key)) {
574  error(KeyNode, "unknown key");
575  return false;
576  }
577  KeyStatus &S = Keys[Key];
578  if (S.Seen) {
579  error(KeyNode, Twine("duplicate key '") + Key + "'");
580  return false;
581  }
582  S.Seen = true;
583  return true;
584  }
585 
586  // false on error
587  bool checkMissingKeys(yaml::Node *Obj, DenseMap<StringRef, KeyStatus> &Keys) {
588  for (DenseMap<StringRef, KeyStatus>::iterator I = Keys.begin(),
589  E = Keys.end();
590  I != E; ++I) {
591  if (I->second.Required && !I->second.Seen) {
592  error(Obj, Twine("missing key '") + I->first + "'");
593  return false;
594  }
595  }
596  return true;
597  }
598 
599  Entry *parseEntry(yaml::Node *N) {
600  yaml::MappingNode *M = dyn_cast<yaml::MappingNode>(N);
601  if (!M) {
602  error(N, "expected mapping node for file or directory entry");
603  return nullptr;
604  }
605 
606  KeyStatusPair Fields[] = {
607  KeyStatusPair("name", true),
608  KeyStatusPair("type", true),
609  KeyStatusPair("contents", false),
610  KeyStatusPair("external-contents", false),
611  KeyStatusPair("use-external-name", false),
612  };
613 
614  DenseMap<StringRef, KeyStatus> Keys(
615  &Fields[0], Fields + sizeof(Fields)/sizeof(Fields[0]));
616 
617  bool HasContents = false; // external or otherwise
618  std::vector<Entry *> EntryArrayContents;
619  std::string ExternalContentsPath;
620  std::string Name;
621  FileEntry::NameKind UseExternalName = FileEntry::NK_NotSet;
622  EntryKind Kind;
623 
624  for (yaml::MappingNode::iterator I = M->begin(), E = M->end(); I != E;
625  ++I) {
626  StringRef Key;
627  // Reuse the buffer for key and value, since we don't look at key after
628  // parsing value.
629  SmallString<256> Buffer;
630  if (!parseScalarString(I->getKey(), Key, Buffer))
631  return nullptr;
632 
633  if (!checkDuplicateOrUnknownKey(I->getKey(), Key, Keys))
634  return nullptr;
635 
636  StringRef Value;
637  if (Key == "name") {
638  if (!parseScalarString(I->getValue(), Value, Buffer))
639  return nullptr;
640  Name = Value;
641  } else if (Key == "type") {
642  if (!parseScalarString(I->getValue(), Value, Buffer))
643  return nullptr;
644  if (Value == "file")
645  Kind = EK_File;
646  else if (Value == "directory")
647  Kind = EK_Directory;
648  else {
649  error(I->getValue(), "unknown value for 'type'");
650  return nullptr;
651  }
652  } else if (Key == "contents") {
653  if (HasContents) {
654  error(I->getKey(),
655  "entry already has 'contents' or 'external-contents'");
656  return nullptr;
657  }
658  HasContents = true;
659  yaml::SequenceNode *Contents =
660  dyn_cast<yaml::SequenceNode>(I->getValue());
661  if (!Contents) {
662  // FIXME: this is only for directories, what about files?
663  error(I->getValue(), "expected array");
664  return nullptr;
665  }
666 
667  for (yaml::SequenceNode::iterator I = Contents->begin(),
668  E = Contents->end();
669  I != E; ++I) {
670  if (Entry *E = parseEntry(&*I))
671  EntryArrayContents.push_back(E);
672  else
673  return nullptr;
674  }
675  } else if (Key == "external-contents") {
676  if (HasContents) {
677  error(I->getKey(),
678  "entry already has 'contents' or 'external-contents'");
679  return nullptr;
680  }
681  HasContents = true;
682  if (!parseScalarString(I->getValue(), Value, Buffer))
683  return nullptr;
684  ExternalContentsPath = Value;
685  } else if (Key == "use-external-name") {
686  bool Val;
687  if (!parseScalarBool(I->getValue(), Val))
688  return nullptr;
689  UseExternalName = Val ? FileEntry::NK_External : FileEntry::NK_Virtual;
690  } else {
691  llvm_unreachable("key missing from Keys");
692  }
693  }
694 
695  if (Stream.failed())
696  return nullptr;
697 
698  // check for missing keys
699  if (!HasContents) {
700  error(N, "missing key 'contents' or 'external-contents'");
701  return nullptr;
702  }
703  if (!checkMissingKeys(N, Keys))
704  return nullptr;
705 
706  // check invalid configuration
707  if (Kind == EK_Directory && UseExternalName != FileEntry::NK_NotSet) {
708  error(N, "'use-external-name' is not supported for directories");
709  return nullptr;
710  }
711 
712  // Remove trailing slash(es), being careful not to remove the root path
713  StringRef Trimmed(Name);
714  size_t RootPathLen = sys::path::root_path(Trimmed).size();
715  while (Trimmed.size() > RootPathLen &&
716  sys::path::is_separator(Trimmed.back()))
717  Trimmed = Trimmed.slice(0, Trimmed.size()-1);
718  // Get the last component
719  StringRef LastComponent = sys::path::filename(Trimmed);
720 
721  Entry *Result = nullptr;
722  switch (Kind) {
723  case EK_File:
724  Result = new FileEntry(LastComponent, std::move(ExternalContentsPath),
725  UseExternalName);
726  break;
727  case EK_Directory:
728  Result = new DirectoryEntry(LastComponent, std::move(EntryArrayContents),
729  Status("", "", getNextVirtualUniqueID(), sys::TimeValue::now(), 0, 0,
730  0, file_type::directory_file, sys::fs::all_all));
731  break;
732  }
733 
734  StringRef Parent = sys::path::parent_path(Trimmed);
735  if (Parent.empty())
736  return Result;
737 
738  // if 'name' contains multiple components, create implicit directory entries
739  for (sys::path::reverse_iterator I = sys::path::rbegin(Parent),
740  E = sys::path::rend(Parent);
741  I != E; ++I) {
742  Result = new DirectoryEntry(*I, llvm::makeArrayRef(Result),
743  Status("", "", getNextVirtualUniqueID(), sys::TimeValue::now(), 0, 0,
744  0, file_type::directory_file, sys::fs::all_all));
745  }
746  return Result;
747  }
748 
749 public:
750  VFSFromYAMLParser(yaml::Stream &S) : Stream(S) {}
751 
752  // false on error
753  bool parse(yaml::Node *Root, VFSFromYAML *FS) {
754  yaml::MappingNode *Top = dyn_cast<yaml::MappingNode>(Root);
755  if (!Top) {
756  error(Root, "expected mapping node");
757  return false;
758  }
759 
760  KeyStatusPair Fields[] = {
761  KeyStatusPair("version", true),
762  KeyStatusPair("case-sensitive", false),
763  KeyStatusPair("use-external-names", false),
764  KeyStatusPair("roots", true),
765  };
766 
767  DenseMap<StringRef, KeyStatus> Keys(
768  &Fields[0], Fields + sizeof(Fields)/sizeof(Fields[0]));
769 
770  // Parse configuration and 'roots'
771  for (yaml::MappingNode::iterator I = Top->begin(), E = Top->end(); I != E;
772  ++I) {
773  SmallString<10> KeyBuffer;
774  StringRef Key;
775  if (!parseScalarString(I->getKey(), Key, KeyBuffer))
776  return false;
777 
778  if (!checkDuplicateOrUnknownKey(I->getKey(), Key, Keys))
779  return false;
780 
781  if (Key == "roots") {
782  yaml::SequenceNode *Roots = dyn_cast<yaml::SequenceNode>(I->getValue());
783  if (!Roots) {
784  error(I->getValue(), "expected array");
785  return false;
786  }
787 
788  for (yaml::SequenceNode::iterator I = Roots->begin(), E = Roots->end();
789  I != E; ++I) {
790  if (Entry *E = parseEntry(&*I))
791  FS->Roots.push_back(E);
792  else
793  return false;
794  }
795  } else if (Key == "version") {
796  StringRef VersionString;
797  SmallString<4> Storage;
798  if (!parseScalarString(I->getValue(), VersionString, Storage))
799  return false;
800  int Version;
801  if (VersionString.getAsInteger<int>(10, Version)) {
802  error(I->getValue(), "expected integer");
803  return false;
804  }
805  if (Version < 0) {
806  error(I->getValue(), "invalid version number");
807  return false;
808  }
809  if (Version != 0) {
810  error(I->getValue(), "version mismatch, expected 0");
811  return false;
812  }
813  } else if (Key == "case-sensitive") {
814  if (!parseScalarBool(I->getValue(), FS->CaseSensitive))
815  return false;
816  } else if (Key == "use-external-names") {
817  if (!parseScalarBool(I->getValue(), FS->UseExternalNames))
818  return false;
819  } else {
820  llvm_unreachable("key missing from Keys");
821  }
822  }
823 
824  if (Stream.failed())
825  return false;
826 
827  if (!checkMissingKeys(Top, Keys))
828  return false;
829  return true;
830  }
831 };
832 } // end of anonymous namespace
833 
834 Entry::~Entry() {}
835 DirectoryEntry::~DirectoryEntry() { llvm::DeleteContainerPointers(Contents); }
836 
837 VFSFromYAML::~VFSFromYAML() { llvm::DeleteContainerPointers(Roots); }
838 
839 VFSFromYAML *VFSFromYAML::create(std::unique_ptr<MemoryBuffer> Buffer,
840  SourceMgr::DiagHandlerTy DiagHandler,
841  void *DiagContext,
842  IntrusiveRefCntPtr<FileSystem> ExternalFS) {
843 
844  SourceMgr SM;
845  yaml::Stream Stream(Buffer->getMemBufferRef(), SM);
846 
847  SM.setDiagHandler(DiagHandler, DiagContext);
848  yaml::document_iterator DI = Stream.begin();
849  yaml::Node *Root = DI->getRoot();
850  if (DI == Stream.end() || !Root) {
851  SM.PrintMessage(SMLoc(), SourceMgr::DK_Error, "expected root node");
852  return nullptr;
853  }
854 
855  VFSFromYAMLParser P(Stream);
856 
857  std::unique_ptr<VFSFromYAML> FS(new VFSFromYAML(ExternalFS));
858  if (!P.parse(Root, FS.get()))
859  return nullptr;
860 
861  return FS.release();
862 }
863 
864 ErrorOr<Entry *> VFSFromYAML::lookupPath(const Twine &Path_) {
865  SmallString<256> Path;
866  Path_.toVector(Path);
867 
868  // Handle relative paths
869  if (std::error_code EC = sys::fs::make_absolute(Path))
870  return EC;
871 
872  if (Path.empty())
873  return make_error_code(llvm::errc::invalid_argument);
874 
875  sys::path::const_iterator Start = sys::path::begin(Path);
876  sys::path::const_iterator End = sys::path::end(Path);
877  for (std::vector<Entry *>::iterator I = Roots.begin(), E = Roots.end();
878  I != E; ++I) {
879  ErrorOr<Entry *> Result = lookupPath(Start, End, *I);
880  if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
881  return Result;
882  }
883  return make_error_code(llvm::errc::no_such_file_or_directory);
884 }
885 
886 ErrorOr<Entry *> VFSFromYAML::lookupPath(sys::path::const_iterator Start,
887  sys::path::const_iterator End,
888  Entry *From) {
889  if (Start->equals("."))
890  ++Start;
891 
892  // FIXME: handle ..
893  if (CaseSensitive ? !Start->equals(From->getName())
894  : !Start->equals_lower(From->getName()))
895  // failure to match
896  return make_error_code(llvm::errc::no_such_file_or_directory);
897 
898  ++Start;
899 
900  if (Start == End) {
901  // Match!
902  return From;
903  }
904 
905  DirectoryEntry *DE = dyn_cast<DirectoryEntry>(From);
906  if (!DE)
907  return make_error_code(llvm::errc::not_a_directory);
908 
909  for (DirectoryEntry::iterator I = DE->contents_begin(),
910  E = DE->contents_end();
911  I != E; ++I) {
912  ErrorOr<Entry *> Result = lookupPath(Start, End, *I);
913  if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
914  return Result;
915  }
916  return make_error_code(llvm::errc::no_such_file_or_directory);
917 }
918 
919 ErrorOr<Status> VFSFromYAML::status(const Twine &Path, Entry *E) {
920  assert(E != nullptr);
921  std::string PathStr(Path.str());
922  if (FileEntry *F = dyn_cast<FileEntry>(E)) {
923  ErrorOr<Status> S = ExternalFS->status(F->getExternalContentsPath());
924  assert(!S || S->getName() == F->getExternalContentsPath());
925  if (S && !F->useExternalName(UseExternalNames))
926  S->setName(PathStr);
927  if (S)
928  S->IsVFSMapped = true;
929  return S;
930  } else { // directory
931  DirectoryEntry *DE = cast<DirectoryEntry>(E);
932  Status S = DE->getStatus();
933  S.setName(PathStr);
934  return S;
935  }
936 }
937 
938 ErrorOr<Status> VFSFromYAML::status(const Twine &Path) {
939  ErrorOr<Entry *> Result = lookupPath(Path);
940  if (!Result)
941  return Result.getError();
942  return status(Path, *Result);
943 }
944 
945 ErrorOr<std::unique_ptr<File>> VFSFromYAML::openFileForRead(const Twine &Path) {
946  ErrorOr<Entry *> E = lookupPath(Path);
947  if (!E)
948  return E.getError();
949 
950  FileEntry *F = dyn_cast<FileEntry>(*E);
951  if (!F) // FIXME: errc::not_a_file?
952  return make_error_code(llvm::errc::invalid_argument);
953 
954  auto Result = ExternalFS->openFileForRead(F->getExternalContentsPath());
955  if (!Result)
956  return Result;
957 
958  if (!F->useExternalName(UseExternalNames))
959  (*Result)->setName(Path.str());
960 
961  return Result;
962 }
963 
965 vfs::getVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
966  SourceMgr::DiagHandlerTy DiagHandler, void *DiagContext,
967  IntrusiveRefCntPtr<FileSystem> ExternalFS) {
968  return VFSFromYAML::create(std::move(Buffer), DiagHandler, DiagContext,
969  ExternalFS);
970 }
971 
973  static std::atomic<unsigned> UID;
974  unsigned ID = ++UID;
975  // The following assumes that uint64_t max will never collide with a real
976  // dev_t value from the OS.
977  return UniqueID(std::numeric_limits<uint64_t>::max(), ID);
978 }
979 
980 #ifndef NDEBUG
981 static bool pathHasTraversal(StringRef Path) {
982  using namespace llvm::sys;
983  for (StringRef Comp : llvm::make_range(path::begin(Path), path::end(Path)))
984  if (Comp == "." || Comp == "..")
985  return true;
986  return false;
987 }
988 #endif
989 
990 void YAMLVFSWriter::addFileMapping(StringRef VirtualPath, StringRef RealPath) {
991  assert(sys::path::is_absolute(VirtualPath) && "virtual path not absolute");
992  assert(sys::path::is_absolute(RealPath) && "real path not absolute");
993  assert(!pathHasTraversal(VirtualPath) && "path traversal is not supported");
994  Mappings.emplace_back(VirtualPath, RealPath);
995 }
996 
997 namespace {
998 class JSONWriter {
999  llvm::raw_ostream &OS;
1000  SmallVector<StringRef, 16> DirStack;
1001  inline unsigned getDirIndent() { return 4 * DirStack.size(); }
1002  inline unsigned getFileIndent() { return 4 * (DirStack.size() + 1); }
1003  bool containedIn(StringRef Parent, StringRef Path);
1004  StringRef containedPart(StringRef Parent, StringRef Path);
1005  void startDirectory(StringRef Path);
1006  void endDirectory();
1007  void writeEntry(StringRef VPath, StringRef RPath);
1008 
1009 public:
1010  JSONWriter(llvm::raw_ostream &OS) : OS(OS) {}
1011  void write(ArrayRef<YAMLVFSEntry> Entries, Optional<bool> IsCaseSensitive);
1012 };
1013 }
1014 
1015 bool JSONWriter::containedIn(StringRef Parent, StringRef Path) {
1016  using namespace llvm::sys;
1017  // Compare each path component.
1018  auto IParent = path::begin(Parent), EParent = path::end(Parent);
1019  for (auto IChild = path::begin(Path), EChild = path::end(Path);
1020  IParent != EParent && IChild != EChild; ++IParent, ++IChild) {
1021  if (*IParent != *IChild)
1022  return false;
1023  }
1024  // Have we exhausted the parent path?
1025  return IParent == EParent;
1026 }
1027 
1028 StringRef JSONWriter::containedPart(StringRef Parent, StringRef Path) {
1029  assert(!Parent.empty());
1030  assert(containedIn(Parent, Path));
1031  return Path.slice(Parent.size() + 1, StringRef::npos);
1032 }
1033 
1034 void JSONWriter::startDirectory(StringRef Path) {
1035  StringRef Name =
1036  DirStack.empty() ? Path : containedPart(DirStack.back(), Path);
1037  DirStack.push_back(Path);
1038  unsigned Indent = getDirIndent();
1039  OS.indent(Indent) << "{\n";
1040  OS.indent(Indent + 2) << "'type': 'directory',\n";
1041  OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(Name) << "\",\n";
1042  OS.indent(Indent + 2) << "'contents': [\n";
1043 }
1044 
1045 void JSONWriter::endDirectory() {
1046  unsigned Indent = getDirIndent();
1047  OS.indent(Indent + 2) << "]\n";
1048  OS.indent(Indent) << "}";
1049 
1050  DirStack.pop_back();
1051 }
1052 
1053 void JSONWriter::writeEntry(StringRef VPath, StringRef RPath) {
1054  unsigned Indent = getFileIndent();
1055  OS.indent(Indent) << "{\n";
1056  OS.indent(Indent + 2) << "'type': 'file',\n";
1057  OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(VPath) << "\",\n";
1058  OS.indent(Indent + 2) << "'external-contents': \""
1059  << llvm::yaml::escape(RPath) << "\"\n";
1060  OS.indent(Indent) << "}";
1061 }
1062 
1063 void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
1064  Optional<bool> IsCaseSensitive) {
1065  using namespace llvm::sys;
1066 
1067  OS << "{\n"
1068  " 'version': 0,\n";
1069  if (IsCaseSensitive.hasValue())
1070  OS << " 'case-sensitive': '"
1071  << (IsCaseSensitive.getValue() ? "true" : "false") << "',\n";
1072  OS << " 'roots': [\n";
1073 
1074  if (!Entries.empty()) {
1075  const YAMLVFSEntry &Entry = Entries.front();
1076  startDirectory(path::parent_path(Entry.VPath));
1077  writeEntry(path::filename(Entry.VPath), Entry.RPath);
1078 
1079  for (const auto &Entry : Entries.slice(1)) {
1080  StringRef Dir = path::parent_path(Entry.VPath);
1081  if (Dir == DirStack.back())
1082  OS << ",\n";
1083  else {
1084  while (!DirStack.empty() && !containedIn(DirStack.back(), Dir)) {
1085  OS << "\n";
1086  endDirectory();
1087  }
1088  OS << ",\n";
1089  startDirectory(Dir);
1090  }
1091  writeEntry(path::filename(Entry.VPath), Entry.RPath);
1092  }
1093 
1094  while (!DirStack.empty()) {
1095  OS << "\n";
1096  endDirectory();
1097  }
1098  OS << "\n";
1099  }
1100 
1101  OS << " ]\n"
1102  << "}\n";
1103 }
1104 
1105 void YAMLVFSWriter::write(llvm::raw_ostream &OS) {
1106  std::sort(Mappings.begin(), Mappings.end(),
1107  [](const YAMLVFSEntry &LHS, const YAMLVFSEntry &RHS) {
1108  return LHS.VPath < RHS.VPath;
1109  });
1110 
1111  JSONWriter(OS).write(Mappings, IsCaseSensitive);
1112 }
1113 
1114 VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl(const Twine &_Path,
1115  VFSFromYAML &FS,
1116  DirectoryEntry::iterator Begin,
1117  DirectoryEntry::iterator End,
1118  std::error_code &EC)
1119  : Dir(_Path.str()), FS(FS), Current(Begin), End(End) {
1120  if (Current != End) {
1121  SmallString<128> PathStr(Dir);
1122  llvm::sys::path::append(PathStr, (*Current)->getName());
1123  llvm::ErrorOr<vfs::Status> S = FS.status(PathStr);
1124  if (S)
1125  CurrentEntry = *S;
1126  else
1127  EC = S.getError();
1128  }
1129 }
1130 
1131 std::error_code VFSFromYamlDirIterImpl::increment() {
1132  assert(Current != End && "cannot iterate past end");
1133  if (++Current != End) {
1134  SmallString<128> PathStr(Dir);
1135  llvm::sys::path::append(PathStr, (*Current)->getName());
1136  llvm::ErrorOr<vfs::Status> S = FS.status(PathStr);
1137  if (!S)
1138  return S.getError();
1139  CurrentEntry = *S;
1140  } else {
1141  CurrentEntry = Status();
1142  }
1143  return std::error_code();
1144 }
1145 
1147  const Twine &Path,
1148  std::error_code &EC)
1149  : FS(&FS_) {
1150  directory_iterator I = FS->dir_begin(Path, EC);
1151  if (!EC && I != directory_iterator()) {
1152  State = std::make_shared<IterState>();
1153  State->push(I);
1154  }
1155 }
1156 
1159  assert(FS && State && !State->empty() && "incrementing past end");
1160  assert(State->top()->isStatusKnown() && "non-canonical end iterator");
1162  if (State->top()->isDirectory()) {
1163  vfs::directory_iterator I = FS->dir_begin(State->top()->getName(), EC);
1164  if (EC)
1165  return *this;
1166  if (I != End) {
1167  State->push(I);
1168  return *this;
1169  }
1170  }
1171 
1172  while (!State->empty() && State->top().increment(EC) == End)
1173  State->pop();
1174 
1175  if (State->empty())
1176  State.reset(); // end iterator
1177 
1178  return *this;
1179 }
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
std::error_code make_error_code(ParseError e)
Definition: Format.cpp:322
The virtual file system interface.
IntrusiveRefCntPtr< FileSystem > getVFSFromYAML(std::unique_ptr< llvm::MemoryBuffer > Buffer, llvm::SourceMgr::DiagHandlerTy DiagHandler, void *DiagContext=nullptr, IntrusiveRefCntPtr< FileSystem > ExternalFS=getRealFileSystem())
Gets a FileSystem for a virtual file system described in YAML format.
An input iterator over the recursive contents of a virtual path, similar to llvm::sys::fs::recursive_...
A file system that allows overlaying one AbstractFileSystem on top of another.
FileSystemList::reverse_iterator iterator
The result of a status operation.
recursive_directory_iterator()
Construct an 'end' iterator.
AnnotatingParser & P
const SmallVectorImpl< AnnotatedLine * >::const_iterator End
ID
Defines the set of possible language-specific address spaces.
Definition: AddressSpaces.h:27
SourceManager & SM
llvm::sys::fs::UniqueID getUniqueID() const
SourceManager & SourceMgr
Definition: Format.cpp:1205
Represents an open file.
static bool pathHasTraversal(StringRef Path)
The result type of a method or function.
#define false
Definition: stdbool.h:33
Kind
recursive_directory_iterator & increment(std::error_code &EC)
Equivalent to operator++, with an error code.
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:53
ast_type_traits::DynTypedNode Node
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
Definition: ASTMatchers.h:1639
std::unique_ptr< DiagnosticConsumer > create(StringRef OutputFile, DiagnosticOptions *Diags, bool MergeChildRecords=false)
Returns a DiagnosticConsumer that serializes diagnostics to a bitcode file.
Defines the virtual file system interface vfs::FileSystem.
llvm::sys::fs::UniqueID getNextVirtualUniqueID()
Get a globally unique ID for a virtual file or directory.
static bool classof(const OMPClause *T)
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:40
An input iterator over the entries in a virtual path, similar to llvm::sys::fs::directory_iterator.
FormatToken * Current
static Decl::Kind getKind(const Decl *D)
Definition: DeclBase.cpp:739
An interface for virtual file systems to provide an iterator over the (non-recursive) contents of a d...
#define true
Definition: stdbool.h:32
unsigned Indent
The current line's indent.