Add a full fs.FS implementation to runfiles (#3969)

* WIP: Add a full `fs.FS` implementation to `runfiles`

* Extract out common implementation

* Minimal feature parity for manifest implementation

* Implement manifest except for dir statting plus test

* Test info

* Add basic trie implementation

* Fully fix test

* Simplify

* Vendor testfs

* FIx tests

* Cleanup

* Simplify

* Root symlinks

* Simplify further

* Add comments and refactor

* Adopt test to Bzlmod

* Also materialize canonical names

* Adjust comment

* Attempt to throw test failures

* Attemp to fix WIndows #2

* Fix test #3

* Address review comments

* Fix test

* Address review comments

* Rename links for better test coverage

* Rename manifest dir entries

* Add `String()` implementations and faithfully fake runfiles dir

* Resolve one layer of symlinks in directory impl

* Resolve all symlinks

.bazelci/presubmit.yml

@@ -29,6 +29,8 @@ tasks:
       - "-//tests/core/nogo/bzlmod/..."
       # Nogo includes/excludes doesn't work before bazel 6
       - "-//tests/core/nogo/includes_excludes:includes_exclude_test"
+      # _repo_mapping is missing
+      - "-//tests/runfiles:runfiles_test"
   ubuntu2004:
     # enable some unflipped incompatible flags on this platform to ensure we don't regress.
     shell_commands:
@@ -89,7 +91,7 @@ tasks:
       - "@go_default_sdk//..."
     test_targets:
       - "//..."
-  macos:
+  macos_legacy:
     shell_commands:
       - tests/core/cgo/generate_imported_dylib.sh
     build_flags:
@@ -130,6 +132,8 @@ tasks:
       - "--"
       - "//..."
       - "-//tests/core/stdlib:buildid_test"
+      # Source directories in runfiles are not supported with RBE.
+      - "-//tests/runfiles:runfiles_test"
   windows:
     build_flags:
       - '--action_env=PATH=C:\tools\msys64\usr\bin;C:\tools\msys64\bin;C:\tools\msys64\mingw64\bin;C:\python3\Scripts\;C:\python3;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0;C:\Windows\System32\OpenSSH;C:\ProgramData\GooGet;C:\Program Files\Google\Compute Engine\metadata_scripts;C:\Program Files (x86)\Google\Cloud SDK\google-cloud-sdk\bin;C:\Program Files\Google\Compute Engine\sysprep;C:\ProgramData\chocolatey\bin;C:\Program Files\Git\cmd;C:\tools\msys64\usr\bin;c:\openjdk\bin;C:\Program Files (x86)\Windows Kits\8.1\Windows Performance Toolkit\;C:\Program Files\CMake\bin;c:\ninja;c:\bazel;c:\buildkite'

.bazelrc

@@ -1,3 +1,5 @@
+startup --host_jvm_args=-DBAZEL_TRACK_SOURCE_DIRECTORIES=1
+
 common --enable_platform_specific_config
 # TODO: Temporarily disable while rules_go migrates to Bzlmod for its dev build.
 # https://github.com/bazelbuild/bazel/issues/18958

go/runfiles/directory.go

@@ -14,7 +14,12 @@
 
 package runfiles
 
-import "path/filepath"
+import (
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+)
 
 // Directory specifies the location of the runfiles directory.  You can pass
 // this as an option to New.  If unset or empty, use the value of the
@@ -37,3 +42,40 @@ func (d Directory) new(sourceRepo SourceRepo) (*Runfiles, error) {
 func (d Directory) path(s string) (string, error) {
 	return filepath.Join(string(d), filepath.FromSlash(s)), nil
 }
+
+func (d Directory) open(name string) (fs.File, error) {
+	dirFS := os.DirFS(string(d))
+	f, err := dirFS.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	return &resolvedFile{f.(*os.File), func(child string) (fs.FileInfo, error) {
+		return fs.Stat(dirFS, path.Join(name, child))
+	}}, nil
+}
+
+type resolvedFile struct {
+	fs.ReadDirFile
+	lstatChildAfterReadlink func(string) (fs.FileInfo, error)
+}
+
+func (f *resolvedFile) ReadDir(n int) ([]fs.DirEntry, error) {
+	entries, err := f.ReadDirFile.ReadDir(n)
+	if err != nil {
+		return nil, err
+	}
+	for i, entry := range entries {
+		// Bazel runfiles directories consist of symlinks to the real files, which may themselves
+		// be directories. We want fs.WalkDir to descend into these directories as it does with the
+		// manifest implementation. We do this by replacing the information about an entry that is
+		// a symlink by the info of the resolved file.
+		if entry.Type()&fs.ModeSymlink != 0 {
+			info, err := f.lstatChildAfterReadlink(entry.Name())
+			if err != nil {
+				return nil, err
+			}
+			entries[i] = renamedDirEntry{fs.FileInfoToDirEntry(info), entry.Name()}
+		}
+	}
+	return entries, nil
+}

go/runfiles/fs.go

@@ -12,63 +12,159 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build go1.16
-// +build go1.16
-
 package runfiles
 
 import (
-	"errors"
 	"io"
 	"io/fs"
-	"os"
+	"runtime"
+	"sort"
+	"strings"
 	"time"
 )
 
-// Open implements fs.FS.Open.
+// Open implements fs.FS for a Runfiles instance.
+//
+// Rlocation-style paths are supported with both apparent and canonical repo
+// names. The root directory of the filesystem (".") additionally lists the
+// apparent repo names that are visible to the current source repo
+// (with --enable_bzlmod).
 func (r *Runfiles) Open(name string) (fs.File, error) {
-	if !fs.ValidPath(name) {
+	// Required by testfs.TestFS.
+	if !fs.ValidPath(name) || (runtime.GOOS == "windows" && strings.ContainsRune(name, '\\')) {
 		return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid}
 	}
-	p, err := r.Rlocation(name)
-	if errors.Is(err, ErrEmpty) {
-		return emptyFile(name), nil
+	if name == "." {
+		return &rootDirFile{".", r, nil}, nil
+	}
+	repo, inRepoPath, hasInRepoPath := strings.Cut(name, "/")
+	key := repoMappingKey{r.sourceRepo, repo}
+	targetRepoDirectory, exists := r.repoMapping[key]
+	if !exists {
+		// Either name uses a canonical repo name or refers to a root symlink.
+		// In both cases, we can just open the file directly.
+		return r.impl.open(name)
 	}
+	// Construct the path with the target repo name replaced by the canonical
+	// name.
+	mappedPath := targetRepoDirectory
+	if hasInRepoPath {
+		mappedPath += "/" + inRepoPath
+	}
+	f, err := r.impl.open(mappedPath)
 	if err != nil {
-		return nil, pathError("open", name, err)
+		return nil, err
+	}
+	// The requested path is a child of a repo directory, return the unmodified
+	// file the implementation returned.
+	if hasInRepoPath {
+		return f, nil
 	}
-	return os.Open(p)
+	// Return a special file for a repo dir that knows its apparent name.
+	return &renamedFile{f, repo}, nil
 }
 
-// Stat implements fs.StatFS.Stat.
-func (r *Runfiles) Stat(name string) (fs.FileInfo, error) {
-	if !fs.ValidPath(name) {
-		return nil, &fs.PathError{Op: "stat", Path: name, Err: fs.ErrInvalid}
+type rootDirFile struct {
+	dirFile
+	rf      *Runfiles
+	entries []fs.DirEntry
+}
+
+func (r *rootDirFile) ReadDir(n int) ([]fs.DirEntry, error) {
+	if err := r.initEntries(); err != nil {
+		return nil, err
 	}
-	p, err := r.Rlocation(name)
-	if errors.Is(err, ErrEmpty) {
-		return emptyFileInfo(name), nil
+	if n > 0 && len(r.entries) == 0 {
+		return nil, io.EOF
 	}
+	if n <= 0 || n > len(r.entries) {
+		n = len(r.entries)
+	}
+	entries := r.entries[:n]
+	r.entries = r.entries[n:]
+	return entries, nil
+}
+
+func (r *rootDirFile) initEntries() error {
+	if r.entries != nil {
+		return nil
+	}
+	// The entries of the root dir should be the apparent names of the repos
+	// visible to the main repo (plus root symlinks). We thus need to read
+	// the real entries and then transform and filter them.
+	canonicalToApparentName := make(map[string]string)
+	for k, v := range r.rf.repoMapping {
+		if k.sourceRepo == r.rf.sourceRepo {
+			canonicalToApparentName[v] = k.targetRepoApparentName
+		}
+	}
+	rootFile, err := r.rf.impl.open(".")
 	if err != nil {
-		return nil, pathError("stat", name, err)
+		return err
 	}
-	return os.Stat(p)
+	realDirFile := rootFile.(fs.ReadDirFile)
+	realEntries, err := realDirFile.ReadDir(0)
+	if err != nil {
+		return err
+	}
+	for _, e := range realEntries {
+		r.entries = append(r.entries, e)
+		if apparent, ok := canonicalToApparentName[e.Name()]; ok && e.IsDir() && apparent != e.Name() {
+			// A repo directory that is visible to the current source repo is additionally
+			// materialized under its apparent name. We do not use a symlink as
+			// fs.WalkDir doesn't descend into symlinks.
+			r.entries = append(r.entries, renamedDirEntry{e, apparent})
+		}
+	}
+	sort.Slice(r.entries, func(i, j int) bool {
+		return r.entries[i].Name() < r.entries[j].Name()
+	})
+	return nil
 }
 
-// ReadFile implements fs.ReadFileFS.ReadFile.
-func (r *Runfiles) ReadFile(name string) ([]byte, error) {
-	if !fs.ValidPath(name) {
-		return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid}
+type renamedFile struct {
+	fs.File
+	name string
+}
+
+func (r renamedFile) Stat() (fs.FileInfo, error) {
+	info, err := r.File.Stat()
+	if err != nil {
+		return nil, err
 	}
-	p, err := r.Rlocation(name)
-	if errors.Is(err, ErrEmpty) {
-		return nil, nil
+	return renamedFileInfo{info, r.name}, nil
+}
+
+func (r renamedFile) ReadDir(n int) ([]fs.DirEntry, error) {
+	readDirFile, ok := r.File.(fs.ReadDirFile)
+	if !ok {
+		return nil, &fs.PathError{Op: "readdir", Path: r.name, Err: fs.ErrInvalid}
 	}
+	return readDirFile.ReadDir(n)
+}
+
+type renamedDirEntry struct {
+	fs.DirEntry
+	name string
+}
+
+func (r renamedDirEntry) Name() string { return r.name }
+func (r renamedDirEntry) Info() (fs.FileInfo, error) {
+	info, err := r.DirEntry.Info()
 	if err != nil {
-		return nil, pathError("open", name, err)
+		return nil, err
 	}
-	return os.ReadFile(p)
+	return renamedFileInfo{info, r.name}, nil
 }
+func (r renamedDirEntry) String() string { return fs.FormatDirEntry(r) }
+
+type renamedFileInfo struct {
+	fs.FileInfo
+	name string
+}
+
+func (r renamedFileInfo) Name() string { return r.name }
+func (r renamedFileInfo) String() string { return fs.FormatFileInfo(r) }
 
 type emptyFile string
 
@@ -84,16 +180,4 @@ func (emptyFileInfo) Mode() fs.FileMode  { return 0444 }
 func (emptyFileInfo) ModTime() time.Time { return time.Time{} }
 func (emptyFileInfo) IsDir() bool        { return false }
 func (emptyFileInfo) Sys() interface{}   { return nil }
-
-func pathError(op, name string, err error) error {
-	if err == nil {
-		return nil
-	}
-	var rerr Error
-	if errors.As(err, &rerr) {
-		// Unwrap the error because we don’t need the failing name
-		// twice.
-		return &fs.PathError{Op: op, Path: rerr.Name, Err: rerr.Err}
-	}
-	return &fs.PathError{Op: op, Path: name, Err: err}
-}
+func (i emptyFileInfo) String() string { return fs.FormatFileInfo(i) }