From 905040ef92c222625374d992b584cf67464def3b Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 11 Jan 2018 16:25:08 -0500 Subject: [PATCH 1/7] Add virtual file system settings and hook proc On index load, clear/set the skip worktree bits based on the virtual file system data. Use virtual file system data to update skip-worktree bit in unpack-trees. Use virtual file system data to exclude files and folders not explicitly requested. Signed-off-by: Ben Peart --- Documentation/config/core.txt | 8 + Documentation/githooks.txt | 20 ++ Makefile | 1 + cache.h | 1 + config.c | 23 ++- config.h | 1 + dir.c | 32 +++- environment.c | 1 + read-cache.c | 3 +- t/t1093-virtualfilesystem.sh | 349 ++++++++++++++++++++++++++++++++++ unpack-trees.c | 15 +- virtualfilesystem.c | 312 ++++++++++++++++++++++++++++++ virtualfilesystem.h | 25 +++ wt-status.c | 2 + 14 files changed, 787 insertions(+), 6 deletions(-) create mode 100755 t/t1093-virtualfilesystem.sh create mode 100644 virtualfilesystem.c create mode 100644 virtualfilesystem.h diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index e2f7f2160da9d8..4e59383d393449 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -79,6 +79,14 @@ core.fsmonitorHookVersion:: something that can be used to determine what files have changed without race conditions. +core.virtualFilesystem:: + If set, the value of this variable is used as a command which + will identify all files and directories that are present in + the working directory. Git will only track and update files + listed in the virtual file system. Using the virtual file system + will supersede the sparse-checkout settings which will be ignored. + See the "virtual file system" section of linkgit:githooks[5]. + core.trustctime:: If false, the ctime differences between the index and the working tree are ignored; useful when the inode change time diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt index 1f3b57d04db6ca..4e37610f8dfbeb 100644 --- a/Documentation/githooks.txt +++ b/Documentation/githooks.txt @@ -689,6 +689,26 @@ and "0" meaning they were not. Only one parameter should be set to "1" when the hook runs. The hook running passing "1", "1" should not be possible. +virtualFilesystem +~~~~~~~~~~~~~~~~~~ + +"Virtual File System" allows populating the working directory sparsely. +The projection data is typically automatically generated by an external +process. Git will limit what files it checks for changes as well as which +directories are checked for untracked files based on the path names given. +Git will also only update those files listed in the projection. + +The hook is invoked when the configuration option core.virtualFilesystem +is set. It takes one argument, a version (currently 1). + +The hook should output to stdout the list of all files in the working +directory that git should track. The paths are relative to the root +of the working directory and are separated by a single NUL. Full paths +('dir1/a.txt') as well as directories are supported (ie 'dir1/'). + +The exit status determines whether git will use the data from the +hook. On error, git will abort the command with an error message. + GIT --- Part of the linkgit:git[1] suite diff --git a/Makefile b/Makefile index ba60db167edee3..50bbf06bd255cb 100644 --- a/Makefile +++ b/Makefile @@ -1028,6 +1028,7 @@ LIB_OBJS += utf8.o LIB_OBJS += varint.o LIB_OBJS += version.o LIB_OBJS += versioncmp.o +LIB_OBJS += virtualfilesystem.o LIB_OBJS += walker.o LIB_OBJS += wildmatch.o LIB_OBJS += worktree.o diff --git a/cache.h b/cache.h index 8419c4c41d6cf1..115379ce7ef438 100644 --- a/cache.h +++ b/cache.h @@ -957,6 +957,7 @@ extern char *git_replace_ref_base; extern int fsync_object_files; extern int core_preload_index; +extern const char *core_virtualfilesystem; extern int core_gvfs; extern int precomposed_unicode; extern int protect_hfs; diff --git a/config.c b/config.c index da475a3e842012..b3d618eb094101 100644 --- a/config.c +++ b/config.c @@ -1549,7 +1549,11 @@ static int git_default_core_config(const char *var, const char *value, void *cb) } if (!strcmp(var, "core.sparsecheckout")) { - core_apply_sparse_checkout = git_config_bool(var, value); + /* virtual file system relies on the sparse checkout logic so force it on */ + if (core_virtualfilesystem) + core_apply_sparse_checkout = 1; + else + core_apply_sparse_checkout = git_config_bool(var, value); return 0; } @@ -2540,6 +2544,23 @@ int git_config_get_fsmonitor(void) return 0; } +int git_config_get_virtualfilesystem(void) +{ + if (git_config_get_pathname("core.virtualfilesystem", &core_virtualfilesystem)) + core_virtualfilesystem = getenv("GIT_VIRTUALFILESYSTEM_TEST"); + + if (core_virtualfilesystem && !*core_virtualfilesystem) + core_virtualfilesystem = NULL; + + /* virtual file system relies on the sparse checkout logic so force it on */ + if (core_virtualfilesystem) { + core_apply_sparse_checkout = 1; + return 1; + } + + return 0; +} + int git_config_get_index_threads(int *dest) { int is_bool, val; diff --git a/config.h b/config.h index 19a9adbaa9a3ed..20c78c256c5fa5 100644 --- a/config.h +++ b/config.h @@ -608,6 +608,7 @@ int git_config_get_untracked_cache(void); int git_config_get_split_index(void); int git_config_get_max_percent_split_change(void); int git_config_get_fsmonitor(void); +int git_config_get_virtualfilesystem(void); /* This dies if the configured or default date is in the future */ int git_config_get_expiry(const char *key, const char **output); diff --git a/dir.c b/dir.c index a0a212b5e494e3..88e203d72225d7 100644 --- a/dir.c +++ b/dir.c @@ -6,6 +6,7 @@ * Junio Hamano, 2005-2006 */ #include "cache.h" +#include "virtualfilesystem.h" #include "config.h" #include "dir.h" #include "object-store.h" @@ -1402,6 +1403,17 @@ enum pattern_match_result path_matches_pattern_list( int result = NOT_MATCHED; const char *slash_pos; + /* + * The virtual file system data is used to prevent git from traversing + * any part of the tree that is not in the virtual file system. Return + * 1 to exclude the entry if it is not found in the virtual file system, + * else fall through to the regular excludes logic as it may further exclude. + */ + if (*dtype == DT_UNKNOWN) + *dtype = resolve_dtype(DT_UNKNOWN, istate, pathname, pathlen); + if (is_excluded_from_virtualfilesystem(pathname, pathlen, *dtype) > 0) + return 1; + if (!pl->use_cone_patterns) { pattern = last_matching_pattern_from_list(pathname, pathlen, basename, dtype, pl, istate); @@ -1660,8 +1672,20 @@ struct path_pattern *last_matching_pattern(struct dir_struct *dir, int is_excluded(struct dir_struct *dir, struct index_state *istate, const char *pathname, int *dtype_p) { - struct path_pattern *pattern = - last_matching_pattern(dir, istate, pathname, dtype_p); + struct path_pattern *pattern; + + /* + * The virtual file system data is used to prevent git from traversing + * any part of the tree that is not in the virtual file system. Return + * 1 to exclude the entry if it is not found in the virtual file system, + * else fall through to the regular excludes logic as it may further exclude. + */ + if (*dtype_p == DT_UNKNOWN) + *dtype_p = resolve_dtype(DT_UNKNOWN, istate, pathname, strlen(pathname)); + if (is_excluded_from_virtualfilesystem(pathname, strlen(pathname), *dtype_p) > 0) + return 1; + + pattern = last_matching_pattern(dir, istate, pathname, dtype_p); if (pattern) return pattern->flags & PATTERN_FLAG_NEGATIVE ? 0 : 1; return 0; @@ -2211,6 +2235,8 @@ static enum path_treatment treat_path(struct dir_struct *dir, ignore_case); if (dtype != DT_DIR && has_path_in_index) return path_none; + if (is_excluded_from_virtualfilesystem(path->buf, path->len, dtype) > 0) + return path_excluded; /* * When we are looking at a directory P in the working tree, @@ -2415,6 +2441,8 @@ static void add_path_to_appropriate_result_list(struct dir_struct *dir, /* add the path to the appropriate result list */ switch (state) { case path_excluded: + if (is_excluded_from_virtualfilesystem(path->buf, path->len, DT_DIR) > 0) + break; if (dir->flags & DIR_SHOW_IGNORED) dir_add_name(dir, istate, path->buf, path->len); else if ((dir->flags & DIR_SHOW_IGNORED_TOO) || diff --git a/environment.c b/environment.c index f5e8068ea89e77..1902cc9b3cca3d 100644 --- a/environment.c +++ b/environment.c @@ -71,6 +71,7 @@ int grafts_replace_parents = 1; int core_apply_sparse_checkout; int core_sparse_checkout_cone; int core_gvfs; +const char *core_virtualfilesystem; int merge_log_config = -1; int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ unsigned long pack_size_limit_cfg; diff --git a/read-cache.c b/read-cache.c index 2c5990de65e58e..9d4d0999a3b6a5 100644 --- a/read-cache.c +++ b/read-cache.c @@ -5,6 +5,7 @@ */ #include "cache.h" #include "gvfs.h" +#include "virtualfilesystem.h" #include "config.h" #include "diff.h" #include "diffcore.h" @@ -26,7 +27,6 @@ #include "fsmonitor.h" #include "thread-utils.h" #include "progress.h" -#include "gvfs.h" /* Mask for the name length in ce_flags in the on-disk index */ @@ -1938,6 +1938,7 @@ static void post_read_index_from(struct index_state *istate) tweak_untracked_cache(istate); tweak_split_index(istate); tweak_fsmonitor(istate); + apply_virtualfilesystem(istate); } static size_t estimate_cache_size_from_compressed(unsigned int entries) diff --git a/t/t1093-virtualfilesystem.sh b/t/t1093-virtualfilesystem.sh new file mode 100755 index 00000000000000..88ab4a6a6084df --- /dev/null +++ b/t/t1093-virtualfilesystem.sh @@ -0,0 +1,349 @@ +#!/bin/sh + +test_description='virtual file system tests' + +. ./test-lib.sh + +clean_repo () { + rm .git/index && + git -c core.virtualfilesystem= reset --hard HEAD && + git -c core.virtualfilesystem= clean -fd && + touch untracked.txt && + touch dir1/untracked.txt && + touch dir2/untracked.txt +} + +test_expect_success 'setup' ' + mkdir -p .git/hooks/ && + cat > .gitignore <<-\EOF && + .gitignore + expect* + actual* + EOF + mkdir -p dir1 && + touch dir1/file1.txt && + touch dir1/file2.txt && + mkdir -p dir2 && + touch dir2/file1.txt && + touch dir2/file2.txt && + git add . && + git commit -m "initial" && + git config --local core.virtualfilesystem .git/hooks/virtualfilesystem +' + +test_expect_success 'test hook parameters and version' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + if test "$#" -ne 1 + then + echo "$0: Exactly 1 argument expected" >&2 + exit 2 + fi + + if test "$1" != 1 + then + echo "$0: Unsupported hook version." >&2 + exit 1 + fi + EOF + git status && + write_script .git/hooks/virtualfilesystem <<-\EOF && + exit 3 + EOF + test_must_fail git status +' + +test_expect_success 'verify status is clean' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir2/file1.txt\0" + EOF + rm -f .git/index && + git checkout -f && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir2/file1.txt\0" + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + EOF + git status > actual && + cat > expected <<-\EOF && + On branch master + nothing to commit, working tree clean + EOF + test_i18ncmp expected actual +' + +test_expect_success 'verify skip-worktree bit is set for absolute path' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + EOF + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/file1.txt + S dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify skip-worktree bit is cleared for absolute path' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file2.txt\0" + EOF + git ls-files -v > actual && + cat > expected <<-\EOF && + S dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify folder wild cards' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + EOF + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify folders not included are ignored' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git add . && + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify including one file doesnt include the rest' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + printf "dir1/dir2/a\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git add . && + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/dir2/a + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify files not listed are ignored by git clean -f -x' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "untracked.txt\0" + printf "dir1/\0" + EOF + mkdir -p dir3 && + touch dir3/untracked.txt && + git clean -f -x && + test ! -f untracked.txt && + test -d dir1 && + test -f dir1/file1.txt && + test -f dir1/file2.txt && + test ! -f dir1/untracked.txt && + test -f dir2/file1.txt && + test -f dir2/file2.txt && + test -f dir2/untracked.txt && + test -d dir3 && + test -f dir3/untracked.txt +' + +test_expect_success 'verify files not listed are ignored by git clean -f -d -x' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "untracked.txt\0" + printf "dir1/\0" + printf "dir3/\0" + EOF + mkdir -p dir3 && + touch dir3/untracked.txt && + git clean -f -d -x && + test ! -f untracked.txt && + test -d dir1 && + test -f dir1/file1.txt && + test -f dir1/file2.txt && + test ! -f dir1/untracked.txt && + test -f dir2/file1.txt && + test -f dir2/file2.txt && + test -f dir2/untracked.txt && + test ! -d dir3 && + test ! -f dir3/untracked.txt +' + +test_expect_success 'verify folder entries include all files' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git status -su > actual && + cat > expected <<-\EOF && + ?? dir1/a + ?? dir1/b + ?? dir1/untracked.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify case insensitivity of virtual file system entries' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/a\0" + printf "Dir1/Dir2/a\0" + printf "DIR2/\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git -c core.ignorecase=false status -su > actual && + cat > expected <<-\EOF && + ?? dir1/a + EOF + test_cmp expected actual && + git -c core.ignorecase=true status -su > actual && + cat > expected <<-\EOF && + ?? dir1/a + ?? dir1/dir2/a + ?? dir2/untracked.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file created' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file3.txt\0" + EOF + touch dir1/file3.txt && + git add . && + git ls-files -v > actual && + cat > expected <<-\EOF && + S dir1/file1.txt + S dir1/file2.txt + H dir1/file3.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file renamed' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + printf "dir1/file3.txt\0" + EOF + mv dir1/file1.txt dir1/file3.txt && + git status -su > actual && + cat > expected <<-\EOF && + D dir1/file1.txt + ?? dir1/file3.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file deleted' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + EOF + rm dir1/file1.txt && + git status -su > actual && + cat > expected <<-\EOF && + D dir1/file1.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file overwritten' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + EOF + echo "overwritten" > dir1/file1.txt && + git status -su > actual && + cat > expected <<-\EOF && + M dir1/file1.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on folder created' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/dir1/\0" + EOF + mkdir -p dir1/dir1 && + git status -su > actual && + cat > expected <<-\EOF && + EOF + test_cmp expected actual && + git clean -fd && + test ! -d "/dir1/dir1" +' + +test_expect_success 'on folder renamed' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir3/\0" + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + printf "dir3/file1.txt\0" + printf "dir3/file2.txt\0" + EOF + mv dir1 dir3 && + git status -su > actual && + cat > expected <<-\EOF && + D dir1/file1.txt + D dir1/file2.txt + ?? dir3/file1.txt + ?? dir3/file2.txt + ?? dir3/untracked.txt + EOF + test_cmp expected actual +' + +test_done diff --git a/unpack-trees.c b/unpack-trees.c index aec5d95dfaee9a..c61eaa3fdfc1aa 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -1,5 +1,6 @@ #include "cache.h" #include "gvfs.h" +#include "virtualfilesystem.h" #include "strvec.h" #include "repository.h" #include "config.h" @@ -17,7 +18,6 @@ #include "fsmonitor.h" #include "object-store.h" #include "promisor-remote.h" -#include "gvfs.h" /* * Error messages expected by scripts out of plumbing commands such as @@ -1429,6 +1429,14 @@ static int clear_ce_flags_1(struct index_state *istate, continue; } + /* if it's not in the virtual file system, exit early */ + if (core_virtualfilesystem) { + if (is_included_in_virtualfilesystem(ce->name, ce->ce_namelen) > 0) + ce->ce_flags &= ~clear_mask; + cache++; + continue; + } + if (prefix->len && strncmp(ce->name, prefix->buf, prefix->len)) break; @@ -1589,7 +1597,10 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options if (!o->skip_sparse_checkout && !o->pl) { memset(&pl, 0, sizeof(pl)); free_pattern_list = 1; - populate_from_existing_patterns(o, &pl); + if (core_virtualfilesystem) + o->pl = &pl; + else + populate_from_existing_patterns(o, &pl); } memset(&o->result, 0, sizeof(o->result)); diff --git a/virtualfilesystem.c b/virtualfilesystem.c new file mode 100644 index 00000000000000..0fca1efdb64ad9 --- /dev/null +++ b/virtualfilesystem.c @@ -0,0 +1,312 @@ +#include "cache.h" +#include "config.h" +#include "dir.h" +#include "hashmap.h" +#include "run-command.h" +#include "virtualfilesystem.h" + +#define HOOK_INTERFACE_VERSION (1) + +static struct strbuf virtual_filesystem_data = STRBUF_INIT; +static struct hashmap virtual_filesystem_hashmap; +static struct hashmap parent_directory_hashmap; + +struct virtualfilesystem { + struct hashmap_entry ent; /* must be the first member! */ + const char *pattern; + int patternlen; +}; + +static unsigned int(*vfshash)(const void *buf, size_t len); +static int(*vfscmp)(const char *a, const char *b, size_t len); + +static int vfs_hashmap_cmp(const void *unused_cmp_data, + const struct hashmap_entry *he1, + const struct hashmap_entry *he2, + const void *key) +{ + const struct virtualfilesystem *vfs1 = + container_of(he1, const struct virtualfilesystem, ent); + const struct virtualfilesystem *vfs2 = + container_of(he2, const struct virtualfilesystem, ent); + + return vfscmp(vfs1->pattern, vfs2->pattern, vfs1->patternlen); +} + +static void get_virtual_filesystem_data(struct strbuf *vfs_data) +{ + struct child_process cp = CHILD_PROCESS_INIT; + char ver[64]; + const char *argv[3]; + int err; + + strbuf_init(vfs_data, 0); + + snprintf(ver, sizeof(ver), "%d", HOOK_INTERFACE_VERSION); + argv[0] = core_virtualfilesystem; + argv[1] = ver; + argv[2] = NULL; + cp.argv = argv; + cp.use_shell = 1; + + err = capture_command(&cp, vfs_data, 1024); + if (err) + die("unable to load virtual file system"); +} + +static int check_includes_hashmap(struct hashmap *map, const char *pattern, int patternlen) +{ + struct strbuf sb = STRBUF_INIT; + struct virtualfilesystem vfs; + char *slash; + + /* Check straight mapping */ + strbuf_reset(&sb); + strbuf_add(&sb, pattern, patternlen); + vfs.pattern = sb.buf; + vfs.patternlen = sb.len; + hashmap_entry_init(&vfs.ent, vfshash(vfs.pattern, vfs.patternlen)); + if (hashmap_get_entry(map, &vfs, ent, NULL)) { + strbuf_release(&sb); + return 1; + } + + /* + * Check to see if it matches a directory or any path + * underneath it. In other words, 'a/b/foo.txt' will match + * '/', 'a/', and 'a/b/'. + */ + slash = strchr(sb.buf, '/'); + while (slash) { + vfs.pattern = sb.buf; + vfs.patternlen = slash - sb.buf + 1; + hashmap_entry_init(&vfs.ent, vfshash(vfs.pattern, vfs.patternlen)); + if (hashmap_get_entry(map, &vfs, ent, NULL)) { + strbuf_release(&sb); + return 1; + } + slash = strchr(slash + 1, '/'); + } + + strbuf_release(&sb); + return 0; +} + +static void includes_hashmap_add(struct hashmap *map, const char *pattern, const int patternlen) +{ + struct virtualfilesystem *vfs; + + vfs = xmalloc(sizeof(struct virtualfilesystem)); + vfs->pattern = pattern; + vfs->patternlen = patternlen; + hashmap_entry_init(&vfs->ent, vfshash(vfs->pattern, vfs->patternlen)); + hashmap_add(map, &vfs->ent); +} + +static void initialize_includes_hashmap(struct hashmap *map, struct strbuf *vfs_data) +{ + char *buf, *entry; + size_t len; + int i; + + /* + * Build a hashmap of the virtual file system data we can use to look + * for cache entry matches quickly + */ + vfshash = ignore_case ? memihash : memhash; + vfscmp = ignore_case ? strncasecmp : strncmp; + hashmap_init(map, vfs_hashmap_cmp, NULL, 0); + + entry = buf = vfs_data->buf; + len = vfs_data->len; + for (i = 0; i < len; i++) { + if (buf[i] == '\0') { + includes_hashmap_add(map, entry, buf + i - entry); + entry = buf + i + 1; + } + } +} + +/* + * Return 1 if the requested item is found in the virtual file system, + * 0 for not found and -1 for undecided. + */ +int is_included_in_virtualfilesystem(const char *pathname, int pathlen) +{ + if (!core_virtualfilesystem) + return -1; + + if (!virtual_filesystem_hashmap.tablesize && virtual_filesystem_data.len) + initialize_includes_hashmap(&virtual_filesystem_hashmap, &virtual_filesystem_data); + if (!virtual_filesystem_hashmap.tablesize) + return -1; + + return check_includes_hashmap(&virtual_filesystem_hashmap, pathname, pathlen); +} + +static void parent_directory_hashmap_add(struct hashmap *map, const char *pattern, const int patternlen) +{ + char *slash; + struct virtualfilesystem *vfs; + + /* + * Add any directories leading up to the file as the excludes logic + * needs to match directories leading up to the files as well. Detect + * and prevent unnecessary duplicate entries which will be common. + */ + if (patternlen > 1) { + slash = strchr(pattern + 1, '/'); + while (slash) { + vfs = xmalloc(sizeof(struct virtualfilesystem)); + vfs->pattern = pattern; + vfs->patternlen = slash - pattern + 1; + hashmap_entry_init(&vfs->ent, vfshash(vfs->pattern, vfs->patternlen)); + if (hashmap_get_entry(map, vfs, ent, NULL)) + free(vfs); + else + hashmap_add(map, &vfs->ent); + slash = strchr(slash + 1, '/'); + } + } +} + +static void initialize_parent_directory_hashmap(struct hashmap *map, struct strbuf *vfs_data) +{ + char *buf, *entry; + size_t len; + int i; + + /* + * Build a hashmap of the parent directories contained in the virtual + * file system data we can use to look for matches quickly + */ + vfshash = ignore_case ? memihash : memhash; + vfscmp = ignore_case ? strncasecmp : strncmp; + hashmap_init(map, vfs_hashmap_cmp, NULL, 0); + + entry = buf = vfs_data->buf; + len = vfs_data->len; + for (i = 0; i < len; i++) { + if (buf[i] == '\0') { + parent_directory_hashmap_add(map, entry, buf + i - entry); + entry = buf + i + 1; + } + } +} + +static int check_directory_hashmap(struct hashmap *map, const char *pathname, int pathlen) +{ + struct strbuf sb = STRBUF_INIT; + struct virtualfilesystem vfs; + + /* Check for directory */ + strbuf_reset(&sb); + strbuf_add(&sb, pathname, pathlen); + strbuf_addch(&sb, '/'); + vfs.pattern = sb.buf; + vfs.patternlen = sb.len; + hashmap_entry_init(&vfs.ent, vfshash(vfs.pattern, vfs.patternlen)); + if (hashmap_get_entry(map, &vfs, ent, NULL)) { + strbuf_release(&sb); + return 0; + } + + strbuf_release(&sb); + return 1; +} + +/* + * Return 1 for exclude, 0 for include and -1 for undecided. + */ +int is_excluded_from_virtualfilesystem(const char *pathname, int pathlen, int dtype) +{ + if (!core_virtualfilesystem) + return -1; + + if (dtype != DT_REG && dtype != DT_DIR && dtype != DT_LNK) + die(_("is_excluded_from_virtualfilesystem passed unhandled dtype")); + + if (dtype == DT_REG) { + int ret = is_included_in_virtualfilesystem(pathname, pathlen); + if (ret > 0) + return 0; + if (ret == 0) + return 1; + return ret; + } + + if (dtype == DT_DIR || dtype == DT_LNK) { + if (!parent_directory_hashmap.tablesize && virtual_filesystem_data.len) + initialize_parent_directory_hashmap(&parent_directory_hashmap, &virtual_filesystem_data); + if (!parent_directory_hashmap.tablesize) + return -1; + + return check_directory_hashmap(&parent_directory_hashmap, pathname, pathlen); + } + + return -1; +} + +/* + * Update the CE_SKIP_WORKTREE bits based on the virtual file system. + */ +void apply_virtualfilesystem(struct index_state *istate) +{ + char *buf, *entry; + int i; + + if (!git_config_get_virtualfilesystem()) + return; + + if (!virtual_filesystem_data.len) + get_virtual_filesystem_data(&virtual_filesystem_data); + + /* set CE_SKIP_WORKTREE bit on all entries */ + for (i = 0; i < istate->cache_nr; i++) + istate->cache[i]->ce_flags |= CE_SKIP_WORKTREE; + + /* clear CE_SKIP_WORKTREE bit for everything in the virtual file system */ + entry = buf = virtual_filesystem_data.buf; + for (i = 0; i < virtual_filesystem_data.len; i++) { + if (buf[i] == '\0') { + int pos, len; + + len = buf + i - entry; + + /* look for a directory wild card (ie "dir1/") */ + if (buf[i - 1] == '/') { + if (ignore_case) + adjust_dirname_case(istate, entry); + pos = index_name_pos(istate, entry, len - 1); + if (pos < 0) { + pos = -pos - 1; + while (pos < istate->cache_nr && !fspathncmp(istate->cache[pos]->name, entry, len)) { + istate->cache[pos]->ce_flags &= ~CE_SKIP_WORKTREE; + pos++; + } + } + } else { + if (ignore_case) { + struct cache_entry *ce = index_file_exists(istate, entry, len, ignore_case); + if (ce) + ce->ce_flags &= ~CE_SKIP_WORKTREE; + } else { + int pos = index_name_pos(istate, entry, len); + if (pos >= 0) + istate->cache[pos]->ce_flags &= ~CE_SKIP_WORKTREE; + } + } + + entry += len + 1; + } + } +} + +/* + * Free the virtual file system data structures. + */ +void free_virtualfilesystem(void) { + hashmap_clear_and_free(&virtual_filesystem_hashmap, struct virtualfilesystem, ent); + hashmap_clear_and_free(&parent_directory_hashmap, struct virtualfilesystem, ent); + strbuf_release(&virtual_filesystem_data); +} diff --git a/virtualfilesystem.h b/virtualfilesystem.h new file mode 100644 index 00000000000000..5e8c5b096df09a --- /dev/null +++ b/virtualfilesystem.h @@ -0,0 +1,25 @@ +#ifndef VIRTUALFILESYSTEM_H +#define VIRTUALFILESYSTEM_H + +/* + * Update the CE_SKIP_WORKTREE bits based on the virtual file system. + */ +void apply_virtualfilesystem(struct index_state *istate); + +/* + * Return 1 if the requested item is found in the virtual file system, + * 0 for not found and -1 for undecided. + */ +int is_included_in_virtualfilesystem(const char *pathname, int pathlen); + +/* + * Return 1 for exclude, 0 for include and -1 for undecided. + */ +int is_excluded_from_virtualfilesystem(const char *pathname, int pathlen, int dtype); + +/* + * Free the virtual file system data structures. + */ +void free_virtualfilesystem(void); + +#endif diff --git a/wt-status.c b/wt-status.c index 5b124eceff4e4d..c22d0536199abd 100644 --- a/wt-status.c +++ b/wt-status.c @@ -1496,6 +1496,8 @@ static void show_sparse_checkout_in_use(struct wt_status *s, { if (s->state.sparse_checkout_percentage == SPARSE_CHECKOUT_DISABLED) return; + if (core_virtualfilesystem) + return; status_printf_ln(s, color, _("You are in a sparse checkout with %d%% of tracked files present."), From ec4e07cea40bb4bc1e2485a6775ca8845141614d Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 7 Jun 2018 13:49:01 -0400 Subject: [PATCH 2/7] Update the virtualfilesystem support We now specify that it needs to be run from the root of the git work tree. This enables the hook to be found even if the current working directory is not the root of the repo (like when running 'git diff' with Beyond Compare configured as the diff tool. Also simpify how argv[] parameter is created. Signed-off-by: Ben Peart --- virtualfilesystem.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/virtualfilesystem.c b/virtualfilesystem.c index 0fca1efdb64ad9..12328876f8e6a5 100644 --- a/virtualfilesystem.c +++ b/virtualfilesystem.c @@ -36,18 +36,14 @@ static int vfs_hashmap_cmp(const void *unused_cmp_data, static void get_virtual_filesystem_data(struct strbuf *vfs_data) { struct child_process cp = CHILD_PROCESS_INIT; - char ver[64]; - const char *argv[3]; int err; strbuf_init(vfs_data, 0); - snprintf(ver, sizeof(ver), "%d", HOOK_INTERFACE_VERSION); - argv[0] = core_virtualfilesystem; - argv[1] = ver; - argv[2] = NULL; - cp.argv = argv; + strvec_push(&cp.args, core_virtualfilesystem); + strvec_pushf(&cp.args, "%d", HOOK_INTERFACE_VERSION); cp.use_shell = 1; + cp.dir = get_git_work_tree(); err = capture_command(&cp, vfs_data, 1024); if (err) From 16ba4cb85ebad2c365730fd5bd9c6f9a172a4f63 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Wed, 1 Aug 2018 13:26:22 -0400 Subject: [PATCH 3/7] virtualfilesystem: don't run the virtual file system hook if the index has been redirected Fixes #13 Some git commands spawn helpers and redirect the index to a different location. These include "difftool -d" and the sequencer (i.e. `git rebase -i`, `git cherry-pick` and `git revert`) and others. In those instances we don't want to update their temporary index with our virtualization data. Helped-by: Johannes Schindelin Signed-off-by: Ben Peart --- config.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/config.c b/config.c index b3d618eb094101..16264a680d784d 100644 --- a/config.c +++ b/config.c @@ -2552,10 +2552,24 @@ int git_config_get_virtualfilesystem(void) if (core_virtualfilesystem && !*core_virtualfilesystem) core_virtualfilesystem = NULL; - /* virtual file system relies on the sparse checkout logic so force it on */ if (core_virtualfilesystem) { - core_apply_sparse_checkout = 1; - return 1; + /* + * Some git commands spawn helpers and redirect the index to a different + * location. These include "difftool -d" and the sequencer + * (i.e. `git rebase -i`, `git cherry-pick` and `git revert`) and others. + * In those instances we don't want to update their temporary index with + * our virtualization data. + */ + char *default_index_file = xstrfmt("%s/%s", the_repository->gitdir, "index"); + int should_run_hook = !strcmp(default_index_file, the_repository->index_file); + + free(default_index_file); + if (should_run_hook) { + /* virtual file system relies on the sparse checkout logic so force it on */ + core_apply_sparse_checkout = 1; + return 1; + } + core_virtualfilesystem = NULL; } return 0; From 6a2cecb2ab92a789c02b24756d2cdb489da5f3e3 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Tue, 25 Sep 2018 16:28:16 -0400 Subject: [PATCH 4/7] virtualfilesystem: fix bug with symlinks being ignored The virtual file system code incorrectly treated symlinks as directories instead of regular files. This meant symlinks were not included even if they are listed in the list of files returned by the core.virtualFilesystem hook proc. Fixes #25 Signed-off-by: Ben Peart --- virtualfilesystem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/virtualfilesystem.c b/virtualfilesystem.c index 12328876f8e6a5..e829752e8952e2 100644 --- a/virtualfilesystem.c +++ b/virtualfilesystem.c @@ -222,7 +222,7 @@ int is_excluded_from_virtualfilesystem(const char *pathname, int pathlen, int dt if (dtype != DT_REG && dtype != DT_DIR && dtype != DT_LNK) die(_("is_excluded_from_virtualfilesystem passed unhandled dtype")); - if (dtype == DT_REG) { + if (dtype == DT_REG || dtype == DT_LNK) { int ret = is_included_in_virtualfilesystem(pathname, pathlen); if (ret > 0) return 0; @@ -231,7 +231,7 @@ int is_excluded_from_virtualfilesystem(const char *pathname, int pathlen, int dt return ret; } - if (dtype == DT_DIR || dtype == DT_LNK) { + if (dtype == DT_DIR) { if (!parent_directory_hashmap.tablesize && virtual_filesystem_data.len) initialize_parent_directory_hashmap(&parent_directory_hashmap, &virtual_filesystem_data); if (!parent_directory_hashmap.tablesize) From 6773a1769b9b62592a6fc7bfc6aaebb6fdd64377 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 9 Oct 2018 10:19:14 -0600 Subject: [PATCH 5/7] virtualfilesystem: check if directory is included Add check to see if a directory is included in the virtualfilesystem before checking the directory hashmap. This allows a directory entry like foo/ to find all untracked files in subdirectories. --- t/t1093-virtualfilesystem.sh | 2 ++ virtualfilesystem.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/t/t1093-virtualfilesystem.sh b/t/t1093-virtualfilesystem.sh index 88ab4a6a6084df..280746b7b94167 100755 --- a/t/t1093-virtualfilesystem.sh +++ b/t/t1093-virtualfilesystem.sh @@ -221,6 +221,8 @@ test_expect_success 'verify folder entries include all files' ' cat > expected <<-\EOF && ?? dir1/a ?? dir1/b + ?? dir1/dir2/a + ?? dir1/dir2/b ?? dir1/untracked.txt EOF test_cmp expected actual diff --git a/virtualfilesystem.c b/virtualfilesystem.c index e829752e8952e2..6163dd803060a8 100644 --- a/virtualfilesystem.c +++ b/virtualfilesystem.c @@ -232,6 +232,10 @@ int is_excluded_from_virtualfilesystem(const char *pathname, int pathlen, int dt } if (dtype == DT_DIR) { + int ret = is_included_in_virtualfilesystem(pathname, pathlen); + if (ret > 0) + return 0; + if (!parent_directory_hashmap.tablesize && virtual_filesystem_data.len) initialize_parent_directory_hashmap(&parent_directory_hashmap, &virtual_filesystem_data); if (!parent_directory_hashmap.tablesize) From d11a803d5d4f09562258554859efda52497a92b9 Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Tue, 20 Nov 2018 11:53:53 -0500 Subject: [PATCH 6/7] vfs: fix case where directories not handled correctly The vfs does not correctly handle the case when there is a file that begins with the same prefix as a directory. For example, the following setup would encounter this issue: A directory contains a file named `dir1.sln` and a directory named `dir1/`. The directory `dir1` contains other files. The directory `dir1` is in the virtual file system list The contents of `dir1` should be in the virtual file system, but it is not. The contents of this directory do not have the skip worktree bit cleared as expected. The problem is in the `apply_virtualfilesystem(...)` function where it does not include the trailing slash of the directory name when looking up the position in the index to start clearing the skip worktree bit. This fix is it include the trailing slash when finding the first index entry from `index_name_pos(...)`. --- t/t1093-virtualfilesystem.sh | 19 +++++++++++++++++++ virtualfilesystem.c | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/t/t1093-virtualfilesystem.sh b/t/t1093-virtualfilesystem.sh index 280746b7b94167..ddb54cbc0ffdb8 100755 --- a/t/t1093-virtualfilesystem.sh +++ b/t/t1093-virtualfilesystem.sh @@ -348,4 +348,23 @@ test_expect_success 'on folder renamed' ' test_cmp expected actual ' +test_expect_success 'folder with same prefix as file' ' + clean_repo && + touch dir1.sln && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + printf "dir1.sln\0" + EOF + git add dir1.sln && + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1.sln + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + test_done diff --git a/virtualfilesystem.c b/virtualfilesystem.c index 6163dd803060a8..ebb89678dac782 100644 --- a/virtualfilesystem.c +++ b/virtualfilesystem.c @@ -277,7 +277,7 @@ void apply_virtualfilesystem(struct index_state *istate) if (buf[i - 1] == '/') { if (ignore_case) adjust_dirname_case(istate, entry); - pos = index_name_pos(istate, entry, len - 1); + pos = index_name_pos(istate, entry, len); if (pos < 0) { pos = -pos - 1; while (pos < istate->cache_nr && !fspathncmp(istate->cache[pos]->name, entry, len)) { From ae754f016a3a4f077c5ff2f3053c2cf0b8df1cd6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 28 May 2019 21:48:08 +0200 Subject: [PATCH 7/7] backwards-compatibility: support the post-indexchanged hook When our patches to support that hook were upstreamed, the hook's name was eliciting some reviewer suggestions, and it was renamed to `post-index-change`. These patches (with the new name) made it into v2.22.0. However, VFSforGit users may very well have checkouts with that hook installed under the original name. To support this, let's just introduce a hack where we look a bit more closely when we just failed to find the `post-index-change` hook, and allow any `post-indexchanged` hook to run instead (if it exists). --- run-command.c | 11 +++++++++++ t/t7113-post-index-change-hook.sh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/run-command.c b/run-command.c index 59fb7913e75d08..7bd0cf74710fe2 100644 --- a/run-command.c +++ b/run-command.c @@ -1411,6 +1411,17 @@ int run_hook_strvec(const char *const *env, const char *name, const char *p; p = find_hook(name); + /* + * Backwards compatibility hack in VFS for Git: when originally + * introduced (and used!), it was called `post-indexchanged`, but this + * name was changed during the review on the Git mailing list. + * + * Therefore, when the `post-index-change` hook is not found, let's + * look for a hook with the old name (which would be found in case of + * already-existing checkouts). + */ + if (!p && !strcmp(name, "post-index-change")) + p = find_hook("post-indexchanged"); if (!p) return 0; diff --git a/t/t7113-post-index-change-hook.sh b/t/t7113-post-index-change-hook.sh index 688fa995c9164b..a1d192978de1ba 100755 --- a/t/t7113-post-index-change-hook.sh +++ b/t/t7113-post-index-change-hook.sh @@ -15,6 +15,36 @@ test_expect_success 'setup' ' git commit -m "initial" ' +test_expect_success 'post-indexchanged' ' + mkdir -p .git/hooks && + test_when_finished "rm -f .git/hooks/post-indexchanged marker" && + write_script .git/hooks/post-indexchanged <<-\EOF && + : >marker + EOF + + : make sure -changed is called if -change does not exist && + test_when_finished "echo testing >dir1/file2.txt && git status" && + echo changed >dir1/file2.txt && + : force index to be dirty && + test-tool chmtime -60 .git/index && + git status && + test_path_is_file marker && + + test_when_finished "rm -f .git/hooks/post-index-change marker2" && + write_script .git/hooks/post-index-change <<-\EOF && + : >marker2 + EOF + + : make sure -changed is not called if -change exists && + rm -f marker marker2 && + echo testing >dir1/file2.txt && + : force index to be dirty && + test-tool chmtime -60 .git/index && + git status && + test_path_is_missing marker && + test_path_is_file marker2 +' + test_expect_success 'test status, add, commit, others trigger hook without flags set' ' mkdir -p .git/hooks && write_script .git/hooks/post-index-change <<-\EOF &&