Skip to content

Commit

Permalink
Merge branch 'sparse-index-stuff'
Browse files Browse the repository at this point in the history
This is random stuff that probably all got upstream in the meantime.
  • Loading branch information
dscho committed Aug 16, 2023
2 parents c2d0cfc + 0547d97 commit 34ecb05
Show file tree
Hide file tree
Showing 16 changed files with 232 additions and 17 deletions.
6 changes: 6 additions & 0 deletions Documentation/config/index.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
index.deleteSparseDirectories::
When enabled, the cone mode sparse-checkout feature will delete
directories that are outside of the sparse-checkout cone, unless
such a directory contains an untracked, non-ignored file. Defaults
to true.

index.recordEndOfIndexEntries::
Specifies whether the index file should include an "End Of Index
Entry" section. This reduces index load time on multiprocessor
Expand Down
20 changes: 15 additions & 5 deletions builtin/add.c
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
*/
#define USE_THE_INDEX_VARIABLE
#include "builtin.h"
#include "environment.h"
#include "advice.h"
#include "config.h"
#include "lockfile.h"
Expand Down Expand Up @@ -48,6 +49,7 @@ static int chmod_pathspec(struct pathspec *pathspec, char flip, int show_only)
int err;

if (!include_sparse &&
!core_virtualfilesystem &&
(ce_skip_worktree(ce) ||
!path_in_sparse_checkout(ce->name, &the_index)))
continue;
Expand Down Expand Up @@ -128,8 +130,9 @@ static int refresh(int verbose, const struct pathspec *pathspec)
if (!seen[i]) {
const char *path = pathspec->items[i].original;

if (matches_skip_worktree(pathspec, i, &skip_worktree_seen) ||
!path_in_sparse_checkout(path, &the_index)) {
if (!core_virtualfilesystem &&
(matches_skip_worktree(pathspec, i, &skip_worktree_seen) ||
!path_in_sparse_checkout(path, &the_index))) {
string_list_append(&only_match_skip_worktree,
pathspec->items[i].original);
} else {
Expand All @@ -139,7 +142,11 @@ static int refresh(int verbose, const struct pathspec *pathspec)
}
}

if (only_match_skip_worktree.nr) {
/*
* When using a virtual filesystem, we might re-add a path
* that is currently virtual and we want that to succeed.
*/
if (!core_virtualfilesystem && only_match_skip_worktree.nr) {
advise_on_updating_sparse_paths(&only_match_skip_worktree);
ret = 1;
}
Expand Down Expand Up @@ -517,7 +524,11 @@ int cmd_add(int argc, const char **argv, const char *prefix)
if (seen[i])
continue;

if (!include_sparse &&
/*
* When using a virtual filesystem, we might re-add a path
* that is currently virtual and we want that to succeed.
*/
if (!include_sparse && !core_virtualfilesystem &&
matches_skip_worktree(&pathspec, i, &skip_worktree_seen)) {
string_list_append(&only_match_skip_worktree,
pathspec.items[i].original);
Expand All @@ -541,7 +552,6 @@ int cmd_add(int argc, const char **argv, const char *prefix)
}
}


if (only_match_skip_worktree.nr) {
advise_on_updating_sparse_paths(&only_match_skip_worktree);
exit_status = 1;
Expand Down
21 changes: 21 additions & 0 deletions builtin/gc.c
Original file line number Diff line number Diff line change
Expand Up @@ -1309,6 +1309,8 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts)
char *lock_path = xstrfmt("%s/maintenance", r->objects->odb->path);

if (hold_lock_file_for_update(&lk, lock_path, LOCK_NO_DEREF) < 0) {
struct stat st;
struct strbuf lock_dot_lock = STRBUF_INIT;
/*
* Another maintenance command is running.
*
Expand All @@ -1319,6 +1321,25 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts)
if (!opts->auto_flag && !opts->quiet)
warning(_("lock file '%s' exists, skipping maintenance"),
lock_path);

/*
* Check timestamp on .lock file to see if we should
* delete it to recover from a fail state.
*/
strbuf_addstr(&lock_dot_lock, lock_path);
strbuf_addstr(&lock_dot_lock, ".lock");
if (lstat(lock_dot_lock.buf, &st))
warning_errno(_("unable to stat '%s'"), lock_dot_lock.buf);
else {
if (st.st_mtime < time(NULL) - (6 * 60 * 60)) {
if (unlink(lock_dot_lock.buf))
warning_errno(_("unable to delete stale lock file"));
else
warning(_("deleted stale lock file"));
}
}

strbuf_release(&lock_dot_lock);
free(lock_path);
return 0;
}
Expand Down
50 changes: 48 additions & 2 deletions builtin/reset.c
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
#include "add-interactive.h"
#include "strbuf.h"
#include "quote.h"
#include "dir.h"
#include "entry.h"

#define REFRESH_INDEX_DELAY_WARNING_IN_MS (2 * 1000)

Expand Down Expand Up @@ -156,9 +158,47 @@ static void update_index_from_diff(struct diff_queue_struct *q,

for (i = 0; i < q->nr; i++) {
int pos;
int respect_skip_worktree = 1;
struct diff_filespec *one = q->queue[i]->one;
struct diff_filespec *two = q->queue[i]->two;
int is_in_reset_tree = one->mode && !is_null_oid(&one->oid);
int is_missing = !(one->mode && !is_null_oid(&one->oid));
int was_missing = !two->mode && is_null_oid(&two->oid);
struct cache_entry *ce;
struct cache_entry *ceBefore;
struct checkout state = CHECKOUT_INIT;

/*
* When using the virtual filesystem feature, the cache entries that are
* added here will not have the skip-worktree bit set.
*
* Without this code there is data that is lost because the files that
* would normally be in the working directory are not there and show as
* deleted for the next status or in the case of added files just disappear.
* We need to create the previous version of the files in the working
* directory so that they will have the right content and the next
* status call will show modified or untracked files correctly.
*/
if (core_virtualfilesystem && !file_exists(two->path))
{
pos = index_name_pos(&the_index, two->path, strlen(two->path));
if ((pos >= 0 && ce_skip_worktree(the_index.cache[pos])) &&
(is_missing || !was_missing))
{
state.force = 1;
state.refresh_cache = 1;
state.istate = &the_index;
ceBefore = make_cache_entry(&the_index, two->mode,
&two->oid, two->path,
0, 0);
if (!ceBefore)
die(_("make_cache_entry failed for path '%s'"),
two->path);

checkout_entry(ceBefore, &state, NULL, NULL);
respect_skip_worktree = 0;
}
}

if (!is_in_reset_tree && !intent_to_add) {
remove_file_from_index(&the_index, one->path);
Expand All @@ -177,8 +217,14 @@ static void update_index_from_diff(struct diff_queue_struct *q,
* to properly construct the reset sparse directory.
*/
pos = index_name_pos(&the_index, one->path, strlen(one->path));
if ((pos >= 0 && ce_skip_worktree(the_index.cache[pos])) ||
(pos < 0 && !path_in_sparse_checkout(one->path, &the_index)))

/*
* Do not add the SKIP_WORKTREE bit back if we populated the
* file on purpose in a virtual filesystem scenario.
*/
if (respect_skip_worktree &&
((pos >= 0 && ce_skip_worktree(the_index.cache[pos])) ||
(pos < 0 && !path_in_sparse_checkout(one->path, &the_index))))
ce->ce_flags |= CE_SKIP_WORKTREE;

if (!ce)
Expand Down
9 changes: 7 additions & 2 deletions builtin/rm.c
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
*/
#define USE_THE_INDEX_VARIABLE
#include "builtin.h"
#include "environment.h"
#include "advice.h"
#include "config.h"
#include "lockfile.h"
Expand Down Expand Up @@ -312,7 +313,7 @@ int cmd_rm(int argc, const char **argv, const char *prefix)
for (i = 0; i < the_index.cache_nr; i++) {
const struct cache_entry *ce = the_index.cache[i];

if (!include_sparse &&
if (!include_sparse && !core_virtualfilesystem &&
(ce_skip_worktree(ce) ||
!path_in_sparse_checkout(ce->name, &the_index)))
continue;
Expand Down Expand Up @@ -349,7 +350,11 @@ int cmd_rm(int argc, const char **argv, const char *prefix)
*original ? original : ".");
}

if (only_match_skip_worktree.nr) {
/*
* When using a virtual filesystem, we might re-add a path
* that is currently virtual and we want that to succeed.
*/
if (!core_virtualfilesystem && only_match_skip_worktree.nr) {
advise_on_updating_sparse_paths(&only_match_skip_worktree);
ret = 1;
}
Expand Down
9 changes: 8 additions & 1 deletion builtin/sparse-checkout.c
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ static int sparse_checkout_list(int argc, const char **argv, const char *prefix)

static void clean_tracked_sparse_directories(struct repository *r)
{
int i, was_full = 0;
int i, value, was_full = 0;
struct strbuf path = STRBUF_INIT;
size_t pathlen;
struct string_list_item *item;
Expand All @@ -127,6 +127,13 @@ static void clean_tracked_sparse_directories(struct repository *r)
!r->index->sparse_checkout_patterns->use_cone_patterns)
return;

/*
* Users can disable this behavior.
*/
if (!repo_config_get_bool(r, "index.deletesparsedirectories", &value) &&
!value)
return;

/*
* Use the sparse index as a data structure to assist finding
* directories that are safe to delete. This conversion to a
Expand Down
7 changes: 7 additions & 0 deletions diff.c
Original file line number Diff line number Diff line change
Expand Up @@ -3994,6 +3994,13 @@ static int reuse_worktree_file(struct index_state *istate,
if (!FAST_WORKING_DIRECTORY && !want_file && has_object_pack(oid))
return 0;

/*
* If this path does not match our sparse-checkout definition,
* then the file will not be in the working directory.
*/
if (!path_in_sparse_checkout(name, istate))
return 0;

/*
* Similarly, if we'd have to convert the file contents anyway, that
* makes the optimization not worthwhile.
Expand Down
7 changes: 7 additions & 0 deletions dir.c
Original file line number Diff line number Diff line change
Expand Up @@ -1533,6 +1533,13 @@ static int path_in_sparse_checkout_1(const char *path,
enum pattern_match_result match = UNDECIDED;
const char *end, *slash;

/*
* When using a virtual filesystem, there aren't really patterns
* to follow, but be extra careful to skip this check.
*/
if (core_virtualfilesystem)
return 1;

/*
* We default to accepting a path if the path is empty, there are no
* patterns, or the patterns are of the wrong type.
Expand Down
2 changes: 1 addition & 1 deletion read-cache.c
Original file line number Diff line number Diff line change
Expand Up @@ -3943,7 +3943,7 @@ static void update_callback(struct diff_queue_struct *q,
struct diff_filepair *p = q->queue[i];
const char *path = p->one->path;

if (!data->include_sparse &&
if (!data->include_sparse && !core_virtualfilesystem &&
!path_in_sparse_checkout(path, data->index))
continue;

Expand Down
2 changes: 1 addition & 1 deletion repo-settings.c
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ void prepare_repo_settings(struct repository *r)
/* Boolean config or default, does not cascade (simple) */
repo_cfg_bool(r, "pack.usesparse", &r->settings.pack_use_sparse, 1);
repo_cfg_bool(r, "core.multipackindex", &r->settings.core_multi_pack_index, 1);
repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 0);
repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 1);
repo_cfg_bool(r, "index.skiphash", &r->settings.index_skip_hash, r->settings.index_skip_hash);
repo_cfg_bool(r, "pack.readreverseindex", &r->settings.pack_read_reverse_index, 1);
repo_cfg_bool(r, "pack.usebitmapboundarytraversal",
Expand Down
2 changes: 1 addition & 1 deletion sequencer.c
Original file line number Diff line number Diff line change
Expand Up @@ -699,7 +699,7 @@ static int do_recursive_merge(struct repository *r,
o.branch2 = next ? next_label : "(empty tree)";
if (is_rebase_i(opts))
o.buffer_output = 2;
o.show_rename_progress = 1;
o.show_rename_progress = isatty(2);

head_tree = parse_tree_indirect(head);
next_tree = next ? repo_get_commit_tree(r, next) : empty_tree(r);
Expand Down
4 changes: 2 additions & 2 deletions t/perf/p2000-sparse-operations.sh
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ test_expect_success 'setup repo and indexes' '
git -c core.sparseCheckoutCone=true clone --branch=wide --sparse . full-v3 &&
(
cd full-v3 &&
git sparse-checkout init --cone &&
git sparse-checkout init --cone --no-sparse-index &&
git sparse-checkout set $SPARSE_CONE &&
git config index.version 3 &&
git update-index --index-version=3 &&
Expand All @@ -65,7 +65,7 @@ test_expect_success 'setup repo and indexes' '
git -c core.sparseCheckoutCone=true clone --branch=wide --sparse . full-v4 &&
(
cd full-v4 &&
git sparse-checkout init --cone &&
git sparse-checkout init --cone --no-sparse-index &&
git sparse-checkout set $SPARSE_CONE &&
git config index.version 4 &&
git update-index --index-version=4 &&
Expand Down
4 changes: 4 additions & 0 deletions t/t1091-sparse-checkout-builtin.sh
Original file line number Diff line number Diff line change
Expand Up @@ -782,6 +782,10 @@ test_expect_success 'cone mode clears ignored subdirectories' '
git -C repo status --porcelain=v2 >out &&
test_must_be_empty out &&
git -C repo -c index.deleteSparseDirectories=false sparse-checkout reapply &&
test_path_is_dir repo/folder1 &&
test_path_is_dir repo/deep/deeper2 &&
git -C repo sparse-checkout reapply &&
test_path_is_missing repo/folder1 &&
test_path_is_missing repo/deep/deeper2 &&
Expand Down
Loading

0 comments on commit 34ecb05

Please sign in to comment.