diff --git a/builtin/add.c b/builtin/add.c
index 7386e44dbaea25..2f7fd4913d3f61 100644
--- a/builtin/add.c
+++ b/builtin/add.c
@@ -208,7 +208,11 @@ static int refresh(int verbose, const struct pathspec *pathspec)
 		}
 	}
 
-	if (only_match_skip_worktree.nr) {
+	/*
+	 * When using a virtual filesystem, we might re-add a path
+	 * that is currently virtual and we want that to succeed.
+	 */
+	if (!core_virtualfilesystem && only_match_skip_worktree.nr) {
 		advise_on_updating_sparse_paths(&only_match_skip_worktree);
 		ret = 1;
 	}
@@ -629,7 +633,12 @@ int cmd_add(int argc, const char **argv, const char *prefix)
 			if (seen[i])
 				continue;
 
-			if (matches_skip_worktree(&pathspec, i, &skip_worktree_seen)) {
+			/*
+			 * When using a virtual filesystem, we might re-add a path
+			 * that is currently virtual and we want that to succeed.
+			 */
+			if (!core_virtualfilesystem &&
+			    matches_skip_worktree(&pathspec, i, &skip_worktree_seen)) {
 				string_list_append(&only_match_skip_worktree,
 						   pathspec.items[i].original);
 				continue;
@@ -652,7 +661,6 @@ int cmd_add(int argc, const char **argv, const char *prefix)
 			}
 		}
 
-
 		if (only_match_skip_worktree.nr) {
 			advise_on_updating_sparse_paths(&only_match_skip_worktree);
 			exit_status = 1;
diff --git a/builtin/rm.c b/builtin/rm.c
index 8a24c715e02bab..e6e736536e20cd 100644
--- a/builtin/rm.c
+++ b/builtin/rm.c
@@ -298,7 +298,7 @@ int cmd_rm(int argc, const char **argv, const char *prefix)
 	ensure_full_index(&the_index);
 	for (i = 0; i < active_nr; i++) {
 		const struct cache_entry *ce = active_cache[i];
-		if (ce_skip_worktree(ce))
+		if (!core_virtualfilesystem && ce_skip_worktree(ce))
 			continue;
 		if (!ce_path_match(&the_index, ce, &pathspec, seen))
 			continue;
@@ -332,7 +332,11 @@ int cmd_rm(int argc, const char **argv, const char *prefix)
 				    *original ? original : ".");
 		}
 
-		if (only_match_skip_worktree.nr) {
+		/*
+		 * When using a virtual filesystem, we might re-add a path
+		 * that is currently virtual and we want that to succeed.
+		 */
+		if (!core_virtualfilesystem && only_match_skip_worktree.nr) {
 			advise_on_updating_sparse_paths(&only_match_skip_worktree);
 			ret = 1;
 		}