From a989423650f6387decf1d1e05e45ec343290ea6c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Mar 2024 21:11:37 +0100 Subject: [PATCH 01/11] upgrade `gix` to v0.60 It contains the feature required to get a directory traversal. --- .github/renovate.json5 | 1 - Cargo.lock | 234 +++++++++++++++++++---------------------- Cargo.toml | 4 +- 3 files changed, 108 insertions(+), 131 deletions(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 81f710c148f..401956f16d9 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -72,7 +72,6 @@ 'cargo', ], matchPackageNames: [ - 'gix-features-for-configuration-only', 'gix', ], groupName: 'gix', diff --git a/Cargo.lock b/Cargo.lock index ea8e4354359..afd6866d066 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -226,15 +226,6 @@ dependencies = [ "serde", ] -[[package]] -name = "btoi" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd6407f73a9b8b6162d8a2ef999fe6afd7cc15902ebf42c5cd296addf17e0ad" -dependencies = [ - "num-traits", -] - [[package]] name = "bumpalo" version = "3.14.0" @@ -301,7 +292,6 @@ dependencies = [ "git2", "git2-curl", "gix", - "gix-features", "glob", "hex", "hmac", @@ -955,9 +945,6 @@ name = "faster-hex" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2a2b11eda1d40935b26cf18f6833c526845ae8c41e58d09af6adeb6f0269183" -dependencies = [ - "serde", -] [[package]] name = "fastrand" @@ -1087,9 +1074,9 @@ dependencies = [ [[package]] name = "gix" -version = "0.58.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31887c304d9a935f3e5494fb5d6a0106c34e965168ec0db9b457424eedd0c741" +checksum = "e4e0e59a44bf00de058ee98d6ecf3c9ed8f8842c1da642258ae4120d41ded8f7" dependencies = [ "gix-actor", "gix-attributes", @@ -1141,23 +1128,23 @@ dependencies = [ [[package]] name = "gix-actor" -version = "0.30.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a7bb9fad6125c81372987c06469601d37e1a2d421511adb69971b9083517a8a" +checksum = "45c3a3bde455ad2ee8ba8a195745241ce0b770a8a26faae59fcf409d01b28c46" dependencies = [ "bstr", - "btoi", "gix-date", + "gix-utils", "itoa 1.0.10", "thiserror", - "winnow 0.5.28", + "winnow", ] [[package]] name = "gix-attributes" -version = "0.22.0" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214ee3792e504ee1ce206b36dcafa4f328ca313d1e2ac0b41433d68ef4e14260" +checksum = "eefb48f42eac136a4a0023f49a54ec31be1c7a9589ed762c45dcb9b953f7ecc8" dependencies = [ "bstr", "gix-glob", @@ -1172,27 +1159,27 @@ dependencies = [ [[package]] name = "gix-bitmap" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b6cd0f246180034ddafac9b00a112f19178135b21eb031b3f79355891f7325" +checksum = "a371db66cbd4e13f0ed9dc4c0fea712d7276805fccc877f77e96374d317e87ae" dependencies = [ "thiserror", ] [[package]] name = "gix-chunk" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "003ec6deacf68076a0c157271a127e0bb2c031c1a41f7168cbe5d248d9b85c78" +checksum = "45c8751169961ba7640b513c3b24af61aa962c967aaf04116734975cd5af0c52" dependencies = [ "thiserror", ] [[package]] name = "gix-command" -version = "0.3.3" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1ffc7db3fb50b7dae6ecd937a3527cb725f444614df2ad8988d81806f13f09" +checksum = "f90009020dc4b3de47beed28e1334706e0a330ddd17f5cfeb097df3b15a54b77" dependencies = [ "bstr", "gix-path", @@ -1202,9 +1189,9 @@ dependencies = [ [[package]] name = "gix-commitgraph" -version = "0.24.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82dbd7fb959862e3df2583331f0ad032ac93533e8a52f1b0694bc517f5d292bc" +checksum = "f7b102311085da4af18823413b5176d7c500fb2272eaf391cfa8635d8bcb12c4" dependencies = [ "bstr", "gix-chunk", @@ -1216,9 +1203,9 @@ dependencies = [ [[package]] name = "gix-config" -version = "0.34.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62bf2073b6ce3921ffa6d8326f645f30eec5fc4a8e8a4bc0fcb721a2f3f69dc" +checksum = "62129c75e4b6229fe15fb9838cdc00c655e87105b651e4edd7c183fc5288b5d1" dependencies = [ "bstr", "gix-config-value", @@ -1232,14 +1219,14 @@ dependencies = [ "smallvec", "thiserror", "unicode-bom", - "winnow 0.5.28", + "winnow", ] [[package]] name = "gix-config-value" -version = "0.14.4" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e7bfb37a46ed0b8468db37a6d8a0a61d56bdbe4603ae492cb322e5f3958" +checksum = "fbd06203b1a9b33a78c88252a625031b094d9e1b647260070c25b09910c0a804" dependencies = [ "bitflags 2.4.1", "bstr", @@ -1250,9 +1237,9 @@ dependencies = [ [[package]] name = "gix-credentials" -version = "0.24.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206ede3fe433abba3c8b0174179d5bbac65ae3f0d9187e2ea96c0494db6a139f" +checksum = "5c70146183bd3c7119329a3c7392d1aa0e0adbe48d727f4df31828fe6d8fdaa1" dependencies = [ "bstr", "gix-command", @@ -1267,9 +1254,9 @@ dependencies = [ [[package]] name = "gix-date" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7f3dfb72bebe3449b5e642be64e3c6ccbe9821c8b8f19f487cf5bfbbf4067e" +checksum = "180b130a4a41870edfbd36ce4169c7090bca70e195da783dea088dd973daa59c" dependencies = [ "bstr", "itoa 1.0.10", @@ -1279,9 +1266,9 @@ dependencies = [ [[package]] name = "gix-diff" -version = "0.40.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbdcb5e49c4b9729dd1c361040ae5c3cd7c497b2260b18c954f62db3a63e98cf" +checksum = "78e605593c2ef74980a534ade0909c7dc57cca72baa30cbb67d2dda621f99ac4" dependencies = [ "bstr", "gix-hash", @@ -1291,9 +1278,9 @@ dependencies = [ [[package]] name = "gix-discover" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4669218f3ec0cbbf8f16857b32200890f8ca585f36f5817242e4115fe4551af" +checksum = "64bab49087ed3710caf77e473dc0efc54ca33d8ccc6441359725f121211482b1" dependencies = [ "bstr", "dunce", @@ -1307,9 +1294,9 @@ dependencies = [ [[package]] name = "gix-features" -version = "0.38.0" +version = "0.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "184f7f7d4e45db0e2a362aeaf12c06c5e84817d0ef91d08e8e90170dad9f0b07" +checksum = "db4254037d20a247a0367aa79333750146a369719f0c6617fec4f5752cc62b37" dependencies = [ "bytes", "crc32fast", @@ -1329,9 +1316,9 @@ dependencies = [ [[package]] name = "gix-filter" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9240862840fb740d209422937195e129e4ed3da49af212383260134bea8f6c1a" +checksum = "bd71bf3e64d8fb5d5635d4166ca5a36fe56b292ffff06eab1d93ea47fd5beb89" dependencies = [ "bstr", "encoding_rs", @@ -1350,9 +1337,9 @@ dependencies = [ [[package]] name = "gix-fs" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4436e883d5769f9fb18677b8712b49228357815f9e4104174a6fc2d8461a437b" +checksum = "634b8a743b0aae03c1a74ee0ea24e8c5136895efac64ce52b3ea106e1c6f0613" dependencies = [ "gix-features", "gix-utils", @@ -1360,9 +1347,9 @@ dependencies = [ [[package]] name = "gix-glob" -version = "0.16.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4965a1d06d0ab84a29d4a67697a97352ab14ae1da821084e5afb1fd6d8191ca0" +checksum = "682bdc43cb3c00dbedfcc366de2a849b582efd8d886215dbad2ea662ec156bb5" dependencies = [ "bitflags 2.4.1", "bstr", @@ -1372,9 +1359,9 @@ dependencies = [ [[package]] name = "gix-hash" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0ed89cdc1dce26685c80271c4287077901de3c3dd90234d5fa47c22b2268653" +checksum = "f93d7df7366121b5018f947a04d37f034717e113dcf9ccd85c34b58e57a74d5e" dependencies = [ "faster-hex", "thiserror", @@ -1382,9 +1369,9 @@ dependencies = [ [[package]] name = "gix-hashtable" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe47d8c0887f82355e2e9e16b6cecaa4d5e5346a7a474ca78ff94de1db35a5b" +checksum = "7ddf80e16f3c19ac06ce415a38b8591993d3f73aede049cb561becb5b3a8e242" dependencies = [ "gix-hash", "hashbrown", @@ -1393,9 +1380,9 @@ dependencies = [ [[package]] name = "gix-ignore" -version = "0.11.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7069aaca4a05784c4cb44e392f0eaf627c6e57e05d3100c0e2386a37a682f0" +checksum = "640dbeb4f5829f9fc14d31f654a34a0350e43a24e32d551ad130d99bf01f63f1" dependencies = [ "bstr", "gix-glob", @@ -1406,14 +1393,14 @@ dependencies = [ [[package]] name = "gix-index" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7152181ba8f0a3addc5075dd612cea31fc3e252b29c8be8c45f4892bf87426" +checksum = "549621f13d9ccf325a7de45506a3266af0d08f915181c5687abb5e8669bfd2e6" dependencies = [ "bitflags 2.4.1", "bstr", - "btoi", "filetime", + "fnv", "gix-bitmap", "gix-features", "gix-fs", @@ -1421,6 +1408,8 @@ dependencies = [ "gix-lock", "gix-object", "gix-traverse", + "gix-utils", + "hashbrown", "itoa 1.0.10", "libc", "memmap2", @@ -1431,9 +1420,9 @@ dependencies = [ [[package]] name = "gix-lock" -version = "13.0.0" +version = "13.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "651e46174dc5e7d18b7b809d31937b6de3681b1debd78618c99162cc30fcf3e1" +checksum = "e7c359f81f01b8352063319bcb39789b7ea0887b406406381106e38c4a34d049" dependencies = [ "gix-tempfile", "gix-utils", @@ -1442,9 +1431,9 @@ dependencies = [ [[package]] name = "gix-macros" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75e7ab728059f595f6ddc1ad8771b8d6a231971ae493d9d5948ecad366ee8bb" +checksum = "1dff438f14e67e7713ab9332f5fd18c8f20eb7eb249494f6c2bf170522224032" dependencies = [ "proc-macro2", "quote", @@ -1453,9 +1442,9 @@ dependencies = [ [[package]] name = "gix-negotiate" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a163adb84149e522e991cbe27250a6e01de56f98cd05b174614ce3f8a4e8b140" +checksum = "54ba98f8c8c06870dfc167d192ca38a38261867b836cb89ac80bc9176dba975e" dependencies = [ "bitflags 2.4.1", "gix-commitgraph", @@ -1469,28 +1458,28 @@ dependencies = [ [[package]] name = "gix-object" -version = "0.41.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693ce9d30741506cb082ef2d8b797415b48e032cce0ab23eff894c19a7e4777b" +checksum = "3d4f8efae72030df1c4a81d02dbe2348e748d9b9a11e108ed6efbd846326e051" dependencies = [ "bstr", - "btoi", "gix-actor", "gix-date", "gix-features", "gix-hash", + "gix-utils", "gix-validate", "itoa 1.0.10", "smallvec", "thiserror", - "winnow 0.5.28", + "winnow", ] [[package]] name = "gix-odb" -version = "0.57.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba2fa9e81f2461b78b4d81a807867667326c84cdab48e0aed7b73a593aa1be4" +checksum = "81b55378c719693380f66d9dd21ce46721eed2981d8789fc698ec1ada6fa176e" dependencies = [ "arc-swap", "gix-date", @@ -1508,9 +1497,9 @@ dependencies = [ [[package]] name = "gix-pack" -version = "0.47.0" +version = "0.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da5f3e78c96b76c4e6fe5e8e06b76221e4a0ee9a255aa935ed1fdf68988dfd8" +checksum = "6391aeaa030ad64aba346a9f5c69bb1c4e5c6fb4411705b03b40b49d8614ec30" dependencies = [ "clru", "gix-chunk", @@ -1528,9 +1517,9 @@ dependencies = [ [[package]] name = "gix-packetline" -version = "0.17.3" +version = "0.17.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ff45eef7747bde4986429a3e813478d50c2688b8f239e57bd3aa81065b285f" +checksum = "9ea5cd2b8ecbab2f3a2133686bf241dfc947a744347cfac8806c4ae5769ab931" dependencies = [ "bstr", "faster-hex", @@ -1552,9 +1541,9 @@ dependencies = [ [[package]] name = "gix-path" -version = "0.10.5" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e9ad649bf5e109562d6acba657ca428661ec08e77eaf3a755d8fa55485be9c" +checksum = "23623cf0f475691a6d943f898c4d0b89f5c1a2a64d0f92bce0e0322ee6528783" dependencies = [ "bstr", "gix-trace", @@ -1565,9 +1554,9 @@ dependencies = [ [[package]] name = "gix-pathspec" -version = "0.6.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cbd49750edb26b0a691e5246fc635fa554d344da825cd20fa9ee0da9c1b761f" +checksum = "1a96ed0e71ce9084a471fddfa74e842576a7cbf02fe8bd50388017ac461aed97" dependencies = [ "bitflags 2.4.1", "bstr", @@ -1580,9 +1569,9 @@ dependencies = [ [[package]] name = "gix-prompt" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd89d058258e53e0fd6c57f13ee16c5673a83066a68e11f88626fc8cfda5f6" +checksum = "f5325eb17ce7b5e5d25dec5c2315d642a09d55b9888b3bf46b7d72e1621a55d8" dependencies = [ "gix-command", "gix-config-value", @@ -1593,38 +1582,38 @@ dependencies = [ [[package]] name = "gix-protocol" -version = "0.44.0" +version = "0.44.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84af465436787ff423a1b4d5bd0c1979200e7165ed04324fa03ba2235485eebc" +checksum = "a905cd00946ed8ed6f4f2281f98a889c5b3d38361cd94b8d5a5771d25ab33b99" dependencies = [ "bstr", - "btoi", "gix-credentials", "gix-date", "gix-features", "gix-hash", "gix-transport", + "gix-utils", "maybe-async", "thiserror", - "winnow 0.5.28", + "winnow", ] [[package]] name = "gix-quote" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7dc10303d73a960d10fb82f81188b036ac3e6b11b5795b20b1a60b51d1321f" +checksum = "cbff4f9b9ea3fa7a25a70ee62f545143abef624ac6aa5884344e70c8b0a1d9ff" dependencies = [ "bstr", - "btoi", + "gix-utils", "thiserror", ] [[package]] name = "gix-ref" -version = "0.41.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5818958994ad7879fa566f5441ebcc48f0926aa027b28948e6fbf6578894dc31" +checksum = "fd4aba68b925101cb45d6df328979af0681364579db889098a0de75b36c77b65" dependencies = [ "gix-actor", "gix-date", @@ -1639,14 +1628,14 @@ dependencies = [ "gix-validate", "memmap2", "thiserror", - "winnow 0.5.28", + "winnow", ] [[package]] name = "gix-refspec" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613aa4d93034c5791d13bdc635e530f4ddab1412ddfb4a8215f76213177b61c7" +checksum = "dde848865834a54fe4d9b4573f15d0e9a68eaf3d061b42d3ed52b4b8acf880b2" dependencies = [ "bstr", "gix-hash", @@ -1658,9 +1647,9 @@ dependencies = [ [[package]] name = "gix-revision" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "288f6549d7666db74dc3f169a9a333694fc28ecd2f5aa7b2c979c89eb556751a" +checksum = "9e34196e1969bd5d36e2fbc4467d893999132219d503e23474a8ad2b221cb1e8" dependencies = [ "bstr", "gix-date", @@ -1674,9 +1663,9 @@ dependencies = [ [[package]] name = "gix-revwalk" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9b4d91dfc5c14fee61a28c65113ded720403b65a0f46169c0460f731a5d03c" +checksum = "e0a7d393ae814eeaae41a333c0ff684b243121cc61ccdc5bbe9897094588047d" dependencies = [ "gix-commitgraph", "gix-date", @@ -1689,9 +1678,9 @@ dependencies = [ [[package]] name = "gix-sec" -version = "0.10.4" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8d9bf462feaf05f2121cba7399dbc6c34d88a9cad58fc1e95027791d6a3c6d2" +checksum = "fddc27984a643b20dd03e97790555804f98cf07404e0e552c0ad8133266a79a1" dependencies = [ "bitflags 2.4.1", "gix-path", @@ -1701,9 +1690,9 @@ dependencies = [ [[package]] name = "gix-submodule" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73182f6c1f5ed1ed94ba16581ac62593d5e29cd1c028b2af618f836283b8f8d4" +checksum = "4fb7ea05666362472fecd44c1fc35fe48a5b9b841b431cc4f85b95e6f20c23ec" dependencies = [ "bstr", "gix-config", @@ -1716,9 +1705,9 @@ dependencies = [ [[package]] name = "gix-tempfile" -version = "13.0.0" +version = "13.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d337955b7af00fb87120d053d87cdfb422a80b9ff7a3aa4057a99c79422dc30" +checksum = "a761d76594f4443b675e85928e4902dec333273836bd386906f01e7e346a0d11" dependencies = [ "gix-fs", "libc", @@ -1729,15 +1718,15 @@ dependencies = [ [[package]] name = "gix-trace" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b202d766a7fefc596e2cc6a89cda8ad8ad733aed82da635ac120691112a9b1" +checksum = "9b838b2db8f62c9447d483a4c28d251b67fee32741a82cb4d35e9eb4e9fdc5ab" [[package]] name = "gix-transport" -version = "0.41.0" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58aba2869cc38937bc834b068c93e09e2ab1119bac626f0464d100c1438b799a" +checksum = "cf8e5f72ec9cad9ee44714b9a4ec7427b540a2418b62111f5e3a715bebe1ed9d" dependencies = [ "base64", "bstr", @@ -1754,9 +1743,9 @@ dependencies = [ [[package]] name = "gix-traverse" -version = "0.37.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfc30c5b5e4e838683b59e1b0574ce6bc1c35916df9709aaab32bb7751daf08b" +checksum = "95aef84bc777025403a09788b1e4815c06a19332e9e5d87a955e1ed7da9bf0cf" dependencies = [ "gix-commitgraph", "gix-date", @@ -1770,9 +1759,9 @@ dependencies = [ [[package]] name = "gix-url" -version = "0.27.0" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26f1981ecc700f4fd73ae62b9ca2da7c8816c8fd267f0185e3f8c21e967984ac" +checksum = "8f0b24f3ecc79a5a53539de9c2e99425d0ef23feacdcf3faac983aa9a2f26849" dependencies = [ "bstr", "gix-features", @@ -1784,9 +1773,9 @@ dependencies = [ [[package]] name = "gix-utils" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e839f3d0798b296411263da6bee780a176ef8008a5dfc31287f7eda9266ab8" +checksum = "0066432d4c277f9877f091279a597ea5331f68ca410efc874f0bdfb1cd348f92" dependencies = [ "fastrand", "unicode-normalization", @@ -1794,9 +1783,9 @@ dependencies = [ [[package]] name = "gix-validate" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac7cc36f496bd5d96cdca0f9289bb684480725d40db60f48194aa7723b883854" +checksum = "e39fc6e06044985eac19dd34d474909e517307582e462b2eb4c8fa51b6241545" dependencies = [ "bstr", "thiserror", @@ -1804,9 +1793,9 @@ dependencies = [ [[package]] name = "gix-worktree" -version = "0.30.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca36bb3dc54038c66507dc75c4d8edbee2d6d5cc45227b4eb508ad13dd60a006" +checksum = "fe78e03af9eec168eb187e05463a981c57f0a915f64b1788685a776bd2ef969c" dependencies = [ "bstr", "gix-attributes", @@ -3426,7 +3415,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.5", + "winnow", ] [[package]] @@ -3947,15 +3936,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" -[[package]] -name = "winnow" -version = "0.5.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c830786f7720c2fd27a1a0e27a709dbd3c4d009b56d098fc742d4f4eab91fe2" -dependencies = [ - "memchr", -] - [[package]] name = "winnow" version = "0.6.5" diff --git a/Cargo.toml b/Cargo.toml index b5b508c7b7f..1558a326b03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,8 +46,7 @@ filetime = "0.2.23" flate2 = { version = "1.0.28", default-features = false, features = ["zlib"] } git2 = "0.18.2" git2-curl = "0.19.0" -gix = { version = "0.58.0", default-features = false, features = ["blocking-http-transport-curl", "progress-tree", "revision"] } -gix-features-for-configuration-only = { version = "0.38.0", package = "gix-features", features = [ "parallel" ] } +gix = { version = "0.61.0", default-features = false, features = ["blocking-http-transport-curl", "progress-tree", "revision", "parallel"] } glob = "0.3.1" handlebars = { version = "5.1.0", features = ["dir_source"] } hex = "0.4.3" @@ -161,7 +160,6 @@ flate2.workspace = true git2.workspace = true git2-curl.workspace = true gix.workspace = true -gix-features-for-configuration-only.workspace = true glob.workspace = true hex.workspace = true hmac.workspace = true From 9115545053ae9913daf95f668e508e11b4e628c0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 16 Mar 2024 09:50:08 +0100 Subject: [PATCH 02/11] implement `list_files_gix` to be used when `gitoxide` is enabled. --- Cargo.lock | 22 +++ Cargo.toml | 2 +- src/cargo/sources/path.rs | 290 ++++++++++++++++++++++++++++++++++++- tests/testsuite/package.rs | 25 +++- 4 files changed, 335 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index afd6866d066..0de2de07e24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1086,6 +1086,7 @@ dependencies = [ "gix-credentials", "gix-date", "gix-diff", + "gix-dir", "gix-discover", "gix-features", "gix-filter", @@ -1276,6 +1277,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "gix-dir" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3413ccd29130900c17574678aee640e4847909acae9febf6424dc77b782c6d32" +dependencies = [ + "bstr", + "gix-discover", + "gix-fs", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils", + "gix-worktree", + "thiserror", +] + [[package]] name = "gix-discover" version = "0.31.0" @@ -1777,6 +1798,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0066432d4c277f9877f091279a597ea5331f68ca410efc874f0bdfb1cd348f92" dependencies = [ + "bstr", "fastrand", "unicode-normalization", ] diff --git a/Cargo.toml b/Cargo.toml index 1558a326b03..53230f099b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ filetime = "0.2.23" flate2 = { version = "1.0.28", default-features = false, features = ["zlib"] } git2 = "0.18.2" git2-curl = "0.19.0" -gix = { version = "0.61.0", default-features = false, features = ["blocking-http-transport-curl", "progress-tree", "revision", "parallel"] } +gix = { version = "0.61.0", default-features = false, features = ["blocking-http-transport-curl", "progress-tree", "revision", "parallel", "dirwalk"] } glob = "0.3.1" handlebars = { version = "5.1.0", features = ["dir_source"] } hex = "0.4.3" diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index 415298eb7c7..b3f7ec52307 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -13,6 +13,7 @@ use crate::util::{internal, CargoResult, GlobalContext}; use anyhow::Context as _; use cargo_util::paths; use filetime::FileTime; +use gix::bstr::{ByteSlice, ByteVec}; use ignore::gitignore::GitignoreBuilder; use tracing::{trace, warn}; use walkdir::WalkDir; @@ -139,7 +140,16 @@ impl<'gctx> PathSource<'gctx> { let root = pkg.root(); let no_include_option = pkg.manifest().include().is_empty(); let git_repo = if no_include_option { - self.discover_git_repo(root)? + if self + .gctx + .cli_unstable() + .gitoxide + .map_or(false, |features| features.fetch) + { + self.discover_gix_repo(root)?.map(Git2OrGixRepository::Gix) + } else { + self.discover_git_repo(root)?.map(Git2OrGixRepository::Git2) + } } else { None }; @@ -197,7 +207,10 @@ impl<'gctx> PathSource<'gctx> { // Attempt Git-prepopulate only if no `include` (see rust-lang/cargo#4135). if no_include_option { if let Some(repo) = git_repo { - return self.list_files_git(pkg, &repo, &filter); + return match repo { + Git2OrGixRepository::Git2(repo) => self.list_files_git(pkg, &repo, &filter), + Git2OrGixRepository::Gix(repo) => self.list_files_gix(pkg, &repo, &filter), + }; } } self.list_files_walk(pkg, &filter) @@ -246,6 +259,52 @@ impl<'gctx> PathSource<'gctx> { Ok(None) } + /// Returns [`Some(gix::Repository)`](gix::Repository) if there is a sibling `Cargo.toml` and `.git` + /// directory; otherwise, the caller should fall back on full file list. + fn discover_gix_repo(&self, root: &Path) -> CargoResult> { + let repo = match gix::discover(root) { + Ok(repo) => repo, + Err(e) => { + tracing::debug!( + "could not discover git repo at or above {}: {}", + root.display(), + e + ); + return Ok(None); + } + }; + let index = repo + .index_or_empty() + .with_context(|| format!("failed to open git index at {}", repo.path().display()))?; + let repo_root = repo.work_dir().ok_or_else(|| { + anyhow::format_err!( + "did not expect repo at {} to be bare", + repo.path().display() + ) + })?; + let repo_relative_path = match paths::strip_prefix_canonical(root, repo_root) { + Ok(p) => p, + Err(e) => { + tracing::warn!( + "cannot determine if path `{:?}` is in git repo `{:?}`: {:?}", + root, + repo_root, + e + ); + return Ok(None); + } + }; + let manifest_path = gix::path::join_bstr_unix_pathsep( + gix::path::into_bstr(repo_relative_path), + "Cargo.toml", + ); + if index.entry_index_by_path(&manifest_path).is_ok() { + return Ok(Some(repo)); + } + // Package Cargo.toml is not in git, don't use git to guide our selection. + Ok(None) + } + /// Lists files relevant to building this package inside this source by /// consulting both Git index (tracked) or status (untracked) under /// a given Git repository. @@ -411,6 +470,228 @@ impl<'gctx> PathSource<'gctx> { } } + /// Lists files relevant to building this package inside this source by + /// traversing the git working tree, while avoiding ignored files. + /// + /// This looks into Git submodules as well, resolving them to individual files. + /// Symlinks to directories will also be resolved, which may include `.git` folders + /// in entirety if the link points to a submodule. + fn list_files_gix( + &self, + pkg: &Package, + repo: &gix::Repository, + filter: &dyn Fn(&Path, bool) -> bool, + ) -> CargoResult> { + use gix::dir::entry::Status; + warn!("list_files_gix {}", pkg.package_id()); + let options = repo + .dirwalk_options()? + .emit_untracked(gix::dir::walk::EmissionMode::Matching) + .emit_ignored(None) + .emit_tracked(true) + .recurse_repositories(false) + .symlinks_to_directories_are_ignored_like_directories(true) + .emit_empty_directories(false); + let index = repo.index_or_empty()?; + let root = repo + .work_dir() + .ok_or_else(|| anyhow::format_err!("can't list files on a bare repository"))?; + assert!( + root.is_absolute(), + "BUG: paths used internally are absolute, and the repo inherits that" + ); + + let pkg_path = pkg.root(); + let mut target_prefix; + let repo_relative_pkg_path = pkg_path.strip_prefix(root).ok().unwrap_or(Path::new("")); + + let pathspec = { + let mut include = gix::path::to_unix_separators_on_windows(gix::path::into_bstr( + repo_relative_pkg_path, + )); + + target_prefix = include.clone(); + target_prefix.to_mut().push_str(if include.is_empty() { + "target/" + } else { + "/target/" + }); + + // The `target` directory is never included if it is in the package root. + let exclude = { + let mut pattern = include.clone(); + pattern + .to_mut() + .insert_str(0, if include.is_empty() { ":!" } else { ":!/" }); + pattern.to_mut().push_str(b"/target/"); + pattern + }; + include.to_mut().insert_str(0, ":/"); + + vec![include, exclude] + }; + + let mut delegate = Delegate::new(self, pkg, pkg_path, repo, root, filter)?; + repo.dirwalk( + &index, + pathspec, + &Default::default(), + options, + &mut delegate, + )?; + + let mut files = delegate.into_result()?; + // Append all normal files that might be tracked in `target/`. + files.extend( + index + .prefixed_entries(target_prefix.as_ref()) + .unwrap_or_default() + .iter() + .filter(|entry| { + entry.stage() == 0 + && entry + .mode + .to_tree_entry_mode() + .map(|mode| mode.kind()) + .map_or(false, |kind| { + use gix::object::tree::EntryKind::*; + matches!(kind, Blob | BlobExecutable | Link) + }) + }) + .map(|entry| root.join(gix::path::from_bstr(entry.path(&index)))), + ); + return Ok(files); + + struct Delegate<'a, 'gctx> { + root: &'a Path, + pkg: &'a Package, + pkg_path: &'a Path, + parent: &'a PathSource<'gctx>, + paths: Vec, + submodules_by_rela_path: Vec<(gix::bstr::BString, gix::Submodule<'a>)>, + subpackages_found: Vec, + filter: &'a dyn Fn(&Path, bool) -> bool, + err: Option, + } + + impl<'a, 'gctx> gix::dir::walk::Delegate for Delegate<'a, 'gctx> { + fn emit( + &mut self, + entry: gix::dir::EntryRef<'_>, + _collapsed_directory_status: Option, + ) -> gix::dir::walk::Action { + match self.handle_entry(entry) { + Ok(()) => gix::dir::walk::Action::Continue, + Err(e) => { + self.err = Some(e.into()); + gix::dir::walk::Action::Cancel + } + } + } + } + + impl<'a, 'gctx> Delegate<'a, 'gctx> { + fn new( + parent: &'a PathSource<'gctx>, + pkg: &'a Package, + pkg_path: &'a Path, + repo: &'a gix::Repository, + root: &'a Path, + filter: &'a dyn Fn(&Path, bool) -> bool, + ) -> CargoResult { + let submodules_by_rela_path: Vec<_> = repo + .submodules()? + .into_iter() + .flatten() + .map(|sm| { + sm.path() + .map(|path| path.into_owned()) + .map(|path| (path, sm)) + }) + .collect::>()?; + Ok(Self { + root, + pkg, + pkg_path, + parent, + filter, + submodules_by_rela_path, + paths: vec![], + subpackages_found: vec![], + err: None, + }) + } + + fn into_result(self) -> CargoResult> { + match self.err { + None => { + return Ok(self.paths); + } + Some(e) => return Err(e), + } + } + + fn handle_entry(&mut self, entry: gix::dir::EntryRef<'_>) -> CargoResult<()> { + if entry.status == Status::Untracked && entry.rela_path.as_ref() == "Cargo.lock" { + return Ok(()); + } + let file_path = self + .root + .join(gix::path::from_bstr(entry.rela_path.as_ref())); + if file_path.file_name().and_then(|name| name.to_str()) == Some("Cargo.toml") { + // Keep track of all sub-packages found and also strip out all + // matches we've found so far. Note, though, that if we find + // our own `Cargo.toml`, we keep going. + let path = file_path.parent().unwrap(); + if path != self.pkg_path { + warn!("subpackage found: {}", path.display()); + self.paths.retain(|p| !p.starts_with(path)); + self.subpackages_found.push(path.to_path_buf()); + return Ok(()); + } + } + + // If this file is part of any other sub-package we've found so far, + // skip it. + if self + .subpackages_found + .iter() + .any(|p| file_path.starts_with(p)) + { + return Ok(()); + } + + let is_dir = entry.disk_kind.map_or(false, |kind| kind.is_dir()); + if entry.disk_kind == Some(gix::dir::entry::Kind::Repository) { + match self + .submodules_by_rela_path + .binary_search_by(|(sm_path, _)| { + sm_path.as_bstr().cmp(entry.rela_path.as_ref()) + }) + .map(|idx| self.submodules_by_rela_path[idx].1.open()) + { + Ok(Ok(Some(sm_repo))) => { + let files = + self.parent + .list_files_gix(self.pkg, &sm_repo, self.filter)?; + self.paths.extend(files); + } + _ => { + self.parent + .walk(&file_path, &mut self.paths, false, self.filter)?; + } + } + } else if (self.filter)(&file_path, is_dir) { + assert!(!is_dir); + // We found a file! + warn!(" found {}", file_path.display()); + self.paths.push(file_path); + } + Ok(()) + } + } + } + /// Lists files relevant to building this package inside this source by /// walking the filesystem from the package root path. /// @@ -541,6 +822,11 @@ impl<'gctx> PathSource<'gctx> { } } +enum Git2OrGixRepository { + Git2(git2::Repository), + Gix(gix::Repository), +} + impl<'gctx> Debug for PathSource<'gctx> { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { write!(f, "the paths source") diff --git a/tests/testsuite/package.rs b/tests/testsuite/package.rs index d5d189e062b..35db8fa3c99 100644 --- a/tests/testsuite/package.rs +++ b/tests/testsuite/package.rs @@ -3394,7 +3394,7 @@ src/main.rs ) .run(); - // if target is committed, it should be include. + // if target is committed, it should be included. git::add(&repo); git::commit(&repo); p.cargo("package -l") @@ -3410,6 +3410,29 @@ derp/not_target/foo.txt derp/target/foo.txt src/main.rs target/foo.txt +", + ) + .run(); + + // Untracked files shouldn't be included, if they are also ignored. + _ = fs::write(repo.workdir().unwrap().join(".gitignore"), "target/").unwrap(); + git::add(&repo); + git::commit(&repo); + _ = fs::write(p.build_dir().join("untracked.txt"), "").unwrap(); + p.cargo("package -l") + .with_stdout( + "\ +.cargo_vcs_info.json +.gitignore +Cargo.lock +Cargo.toml +Cargo.toml.orig +data/not_target +data/target +derp/not_target/foo.txt +derp/target/foo.txt +src/main.rs +target/foo.txt ", ) .run(); From a710d459af3adad3976c9ab01925776cac7ef9ad Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Mar 2024 14:56:52 +0100 Subject: [PATCH 03/11] assure repositories are always walked into to avoid `.git` folders. With a traditional walk, `.git` will be picked up, and so will be ignored directories. This commit also doesn't give submodules special treatment - instead it just tries to open directories as repositories, or walks them if that fails. --- src/cargo/sources/path.rs | 63 +++++++++++++++----------------------- tests/testsuite/package.rs | 1 + 2 files changed, 25 insertions(+), 39 deletions(-) diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index b3f7ec52307..57cecbddcaa 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -13,7 +13,7 @@ use crate::util::{internal, CargoResult, GlobalContext}; use anyhow::Context as _; use cargo_util::paths; use filetime::FileTime; -use gix::bstr::{ByteSlice, ByteVec}; +use gix::bstr::ByteVec; use ignore::gitignore::GitignoreBuilder; use tracing::{trace, warn}; use walkdir::WalkDir; @@ -295,7 +295,7 @@ impl<'gctx> PathSource<'gctx> { } }; let manifest_path = gix::path::join_bstr_unix_pathsep( - gix::path::into_bstr(repo_relative_path), + gix::path::to_unix_separators_on_windows(gix::path::into_bstr(repo_relative_path)), "Cargo.toml", ); if index.entry_index_by_path(&manifest_path).is_ok() { @@ -424,15 +424,8 @@ impl<'gctx> PathSource<'gctx> { // symlink points to a directory. let is_dir = is_dir.unwrap_or_else(|| file_path.is_dir()); if is_dir { - warn!(" found submodule {}", file_path.display()); - let rel = file_path.strip_prefix(root)?; - let rel = rel.to_str().ok_or_else(|| { - anyhow::format_err!("invalid utf-8 filename: {}", rel.display()) - })?; - // Git submodules are currently only named through `/` path - // separators, explicitly not `\` which windows uses. Who knew? - let rel = rel.replace(r"\", "/"); - match repo.find_submodule(&rel).and_then(|s| s.open()) { + warn!(" found directory {}", file_path.display()); + match git2::Repository::open(&file_path) { Ok(repo) => { let files = self.list_files_git(pkg, &repo, filter)?; ret.extend(files.into_iter()); @@ -473,9 +466,9 @@ impl<'gctx> PathSource<'gctx> { /// Lists files relevant to building this package inside this source by /// traversing the git working tree, while avoiding ignored files. /// - /// This looks into Git submodules as well, resolving them to individual files. - /// Symlinks to directories will also be resolved, which may include `.git` folders - /// in entirety if the link points to a submodule. + /// This looks into Git sub-repositories as well, resolving them to individual files. + /// Symlinks to directories will also be resolved, but walked as repositories if they + /// point to one to avoid picking up `.git` directories. fn list_files_gix( &self, pkg: &Package, @@ -531,7 +524,7 @@ impl<'gctx> PathSource<'gctx> { vec![include, exclude] }; - let mut delegate = Delegate::new(self, pkg, pkg_path, repo, root, filter)?; + let mut delegate = Delegate::new(self, pkg, pkg_path, root, filter)?; repo.dirwalk( &index, pathspec, @@ -568,7 +561,6 @@ impl<'gctx> PathSource<'gctx> { pkg_path: &'a Path, parent: &'a PathSource<'gctx>, paths: Vec, - submodules_by_rela_path: Vec<(gix::bstr::BString, gix::Submodule<'a>)>, subpackages_found: Vec, filter: &'a dyn Fn(&Path, bool) -> bool, err: Option, @@ -595,27 +587,15 @@ impl<'gctx> PathSource<'gctx> { parent: &'a PathSource<'gctx>, pkg: &'a Package, pkg_path: &'a Path, - repo: &'a gix::Repository, root: &'a Path, filter: &'a dyn Fn(&Path, bool) -> bool, ) -> CargoResult { - let submodules_by_rela_path: Vec<_> = repo - .submodules()? - .into_iter() - .flatten() - .map(|sm| { - sm.path() - .map(|path| path.into_owned()) - .map(|path| (path, sm)) - }) - .collect::>()?; Ok(Self { root, pkg, pkg_path, parent, filter, - submodules_by_rela_path, paths: vec![], subpackages_found: vec![], err: None, @@ -661,19 +641,24 @@ impl<'gctx> PathSource<'gctx> { return Ok(()); } - let is_dir = entry.disk_kind.map_or(false, |kind| kind.is_dir()); - if entry.disk_kind == Some(gix::dir::entry::Kind::Repository) { - match self - .submodules_by_rela_path - .binary_search_by(|(sm_path, _)| { - sm_path.as_bstr().cmp(entry.rela_path.as_ref()) - }) - .map(|idx| self.submodules_by_rela_path[idx].1.open()) - { - Ok(Ok(Some(sm_repo))) => { + let is_dir = entry.disk_kind.map_or(false, |kind| { + if kind == gix::dir::entry::Kind::Symlink { + // Symlinks must be checked to see if they point to a directory + // we should traverse. + file_path.is_dir() + } else { + kind.is_dir() + } + }); + if is_dir { + // This could be a submodule, or a sub-repository. In any case, we prefer to walk + // it with git-support to leverage ignored files and to avoid pulling in entire + // .git repositories. + match gix::open_opts(&file_path, gix::open::Options::isolated()) { + Ok(sub_repo) => { let files = self.parent - .list_files_gix(self.pkg, &sm_repo, self.filter)?; + .list_files_gix(self.pkg, &sub_repo, self.filter)?; self.paths.extend(files); } _ => { diff --git a/tests/testsuite/package.rs b/tests/testsuite/package.rs index 35db8fa3c99..2fa51dc533b 100644 --- a/tests/testsuite/package.rs +++ b/tests/testsuite/package.rs @@ -689,6 +689,7 @@ fn package_symlink_to_submodule() { project .cargo("package --no-verify -v") .with_stderr_contains("[ARCHIVING] submodule/Makefile") + .with_stderr_does_not_contain("[ARCHIVING] submodule-link/.git/config") .run(); } From 5312586492bbea2c304c59feb247bf295e06a377 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Mar 2024 16:06:13 +0100 Subject: [PATCH 04/11] Assure the extra-logic in dirwalks applies to index files as well This is the case for the git2 implementation naturally, but as `gitoxide` doesn't yet have a dirwalk iterator, it's much less intuitive here. --- src/cargo/sources/path.rs | 163 +++++++++++++++++++++----------------- 1 file changed, 91 insertions(+), 72 deletions(-) diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index 57cecbddcaa..f7b00e9271c 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -533,26 +533,30 @@ impl<'gctx> PathSource<'gctx> { &mut delegate, )?; - let mut files = delegate.into_result()?; + let (mut files, mut subpackages_found) = delegate.into_result()?; // Append all normal files that might be tracked in `target/`. - files.extend( - index - .prefixed_entries(target_prefix.as_ref()) - .unwrap_or_default() - .iter() - .filter(|entry| { - entry.stage() == 0 - && entry - .mode - .to_tree_entry_mode() - .map(|mode| mode.kind()) - .map_or(false, |kind| { - use gix::object::tree::EntryKind::*; - matches!(kind, Blob | BlobExecutable | Link) - }) - }) - .map(|entry| root.join(gix::path::from_bstr(entry.path(&index)))), - ); + for entry in index + .prefixed_entries(target_prefix.as_ref()) + .unwrap_or_default() + .iter() + // probably not needed as conflicts prevent this to run, but let's be explicit. + .filter(|entry| entry.stage() == 0) + { + handle_path( + entry.path(&index), + // Do not trust what's recorded in the index, enforce checking the disk. + // This traversal is not part of a `status()`, and tracking things in `target/` + // is rare. + None, + root, + pkg, + pkg_path, + self, + &mut files, + &mut subpackages_found, + filter, + )? + } return Ok(files); struct Delegate<'a, 'gctx> { @@ -602,10 +606,10 @@ impl<'gctx> PathSource<'gctx> { }) } - fn into_result(self) -> CargoResult> { + fn into_result(self) -> CargoResult<(Vec, Vec)> { match self.err { None => { - return Ok(self.paths); + return Ok((self.paths, self.subpackages_found)); } Some(e) => return Err(e), } @@ -615,65 +619,80 @@ impl<'gctx> PathSource<'gctx> { if entry.status == Status::Untracked && entry.rela_path.as_ref() == "Cargo.lock" { return Ok(()); } - let file_path = self - .root - .join(gix::path::from_bstr(entry.rela_path.as_ref())); - if file_path.file_name().and_then(|name| name.to_str()) == Some("Cargo.toml") { - // Keep track of all sub-packages found and also strip out all - // matches we've found so far. Note, though, that if we find - // our own `Cargo.toml`, we keep going. - let path = file_path.parent().unwrap(); - if path != self.pkg_path { - warn!("subpackage found: {}", path.display()); - self.paths.retain(|p| !p.starts_with(path)); - self.subpackages_found.push(path.to_path_buf()); - return Ok(()); - } - } + handle_path( + entry.rela_path.as_ref(), + entry.disk_kind, + self.root, + self.pkg, + self.pkg_path, + self.parent, + &mut self.paths, + &mut self.subpackages_found, + self.filter, + ) + } + } - // If this file is part of any other sub-package we've found so far, - // skip it. - if self - .subpackages_found - .iter() - .any(|p| file_path.starts_with(p)) - { + fn handle_path( + rela_path: &gix::bstr::BStr, + kind: Option, + root: &Path, + pkg: &Package, + pkg_path: &Path, + source: &PathSource<'_>, + paths: &mut Vec, + subpackages_found: &mut Vec, + filter: &dyn Fn(&Path, bool) -> bool, + ) -> CargoResult<()> { + let file_path = root.join(gix::path::from_bstr(rela_path)); + if file_path.file_name().and_then(|name| name.to_str()) == Some("Cargo.toml") { + // Keep track of all sub-packages found and also strip out all + // matches we've found so far. Note, though, that if we find + // our own `Cargo.toml`, we keep going. + let path = file_path.parent().unwrap(); + if path != pkg_path { + warn!("subpackage found: {}", path.display()); + paths.retain(|p| !p.starts_with(path)); + subpackages_found.push(path.to_path_buf()); return Ok(()); } + } - let is_dir = entry.disk_kind.map_or(false, |kind| { - if kind == gix::dir::entry::Kind::Symlink { - // Symlinks must be checked to see if they point to a directory - // we should traverse. - file_path.is_dir() - } else { - kind.is_dir() + // If this file is part of any other sub-package we've found so far, + // skip it. + if subpackages_found.iter().any(|p| file_path.starts_with(p)) { + return Ok(()); + } + + let is_dir = kind.map_or(false, |kind| { + if kind == gix::dir::entry::Kind::Symlink { + // Symlinks must be checked to see if they point to a directory + // we should traverse. + file_path.is_dir() + } else { + kind.is_dir() + } + }); + if is_dir { + // This could be a submodule, or a sub-repository. In any case, we prefer to walk + // it with git-support to leverage ignored files and to avoid pulling in entire + // .git repositories. + match gix::open_opts(&file_path, gix::open::Options::isolated()) { + Ok(sub_repo) => { + let files = source.list_files_gix(pkg, &sub_repo, filter)?; + paths.extend(files); } - }); - if is_dir { - // This could be a submodule, or a sub-repository. In any case, we prefer to walk - // it with git-support to leverage ignored files and to avoid pulling in entire - // .git repositories. - match gix::open_opts(&file_path, gix::open::Options::isolated()) { - Ok(sub_repo) => { - let files = - self.parent - .list_files_gix(self.pkg, &sub_repo, self.filter)?; - self.paths.extend(files); - } - _ => { - self.parent - .walk(&file_path, &mut self.paths, false, self.filter)?; - } + _ => { + source.walk(&file_path, paths, false, filter)?; } - } else if (self.filter)(&file_path, is_dir) { - assert!(!is_dir); - // We found a file! - warn!(" found {}", file_path.display()); - self.paths.push(file_path); } - Ok(()) + } else if (filter)(&file_path, is_dir) { + assert!(!is_dir); + // We found a file! + warn!(" found {}", file_path.display()); + paths.push(file_path); } + Ok(()) } } From 9bf91497db00b61e9e14685c65847ec8648fef17 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Mar 2024 23:14:36 +0100 Subject: [PATCH 05/11] Use the new dirwalk iterator for comparable ergonomics --- src/cargo/sources/path.rs | 177 ++++++++++---------------------------- 1 file changed, 46 insertions(+), 131 deletions(-) diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index f7b00e9271c..758630e8780 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -14,6 +14,7 @@ use anyhow::Context as _; use cargo_util::paths; use filetime::FileTime; use gix::bstr::ByteVec; +use gix::dir::entry::Status; use ignore::gitignore::GitignoreBuilder; use tracing::{trace, warn}; use walkdir::WalkDir; @@ -475,7 +476,6 @@ impl<'gctx> PathSource<'gctx> { repo: &gix::Repository, filter: &dyn Fn(&Path, bool) -> bool, ) -> CargoResult> { - use gix::dir::entry::Status; warn!("list_files_gix {}", pkg.package_id()); let options = repo .dirwalk_options()? @@ -521,129 +521,45 @@ impl<'gctx> PathSource<'gctx> { }; include.to_mut().insert_str(0, ":/"); - vec![include, exclude] + vec![include.into_owned(), exclude.into_owned()] }; - let mut delegate = Delegate::new(self, pkg, pkg_path, root, filter)?; - repo.dirwalk( - &index, - pathspec, - &Default::default(), - options, - &mut delegate, - )?; - - let (mut files, mut subpackages_found) = delegate.into_result()?; - // Append all normal files that might be tracked in `target/`. - for entry in index - .prefixed_entries(target_prefix.as_ref()) - .unwrap_or_default() - .iter() - // probably not needed as conflicts prevent this to run, but let's be explicit. - .filter(|entry| entry.stage() == 0) - { - handle_path( - entry.path(&index), - // Do not trust what's recorded in the index, enforce checking the disk. - // This traversal is not part of a `status()`, and tracking things in `target/` - // is rare. - None, - root, - pkg, - pkg_path, - self, - &mut files, - &mut subpackages_found, - filter, - )? - } - return Ok(files); - - struct Delegate<'a, 'gctx> { - root: &'a Path, - pkg: &'a Package, - pkg_path: &'a Path, - parent: &'a PathSource<'gctx>, - paths: Vec, - subpackages_found: Vec, - filter: &'a dyn Fn(&Path, bool) -> bool, - err: Option, - } - - impl<'a, 'gctx> gix::dir::walk::Delegate for Delegate<'a, 'gctx> { - fn emit( - &mut self, - entry: gix::dir::EntryRef<'_>, - _collapsed_directory_status: Option, - ) -> gix::dir::walk::Action { - match self.handle_entry(entry) { - Ok(()) => gix::dir::walk::Action::Continue, - Err(e) => { - self.err = Some(e.into()); - gix::dir::walk::Action::Cancel - } - } - } - } - - impl<'a, 'gctx> Delegate<'a, 'gctx> { - fn new( - parent: &'a PathSource<'gctx>, - pkg: &'a Package, - pkg_path: &'a Path, - root: &'a Path, - filter: &'a dyn Fn(&Path, bool) -> bool, - ) -> CargoResult { - Ok(Self { - root, - pkg, - pkg_path, - parent, - filter, - paths: vec![], - subpackages_found: vec![], - err: None, + let mut files = Vec::::new(); + let mut subpackages_found = Vec::new(); + for item in repo + .dirwalk_iter(index.clone(), pathspec, Default::default(), options)? + .filter(|res| { + // Don't include Cargo.lock if it is untracked. Packaging will + // generate a new one as needed. + res.as_ref().map_or(true, |item| { + !(item.entry.status == Status::Untracked + && item.entry.rela_path == "Cargo.lock") }) - } - - fn into_result(self) -> CargoResult<(Vec, Vec)> { - match self.err { - None => { - return Ok((self.paths, self.subpackages_found)); - } - Some(e) => return Err(e), - } - } - - fn handle_entry(&mut self, entry: gix::dir::EntryRef<'_>) -> CargoResult<()> { - if entry.status == Status::Untracked && entry.rela_path.as_ref() == "Cargo.lock" { - return Ok(()); - } - handle_path( - entry.rela_path.as_ref(), - entry.disk_kind, - self.root, - self.pkg, - self.pkg_path, - self.parent, - &mut self.paths, - &mut self.subpackages_found, - self.filter, - ) - } - } - - fn handle_path( - rela_path: &gix::bstr::BStr, - kind: Option, - root: &Path, - pkg: &Package, - pkg_path: &Path, - source: &PathSource<'_>, - paths: &mut Vec, - subpackages_found: &mut Vec, - filter: &dyn Fn(&Path, bool) -> bool, - ) -> CargoResult<()> { + }) + .map(|res| res.map(|item| (item.entry.rela_path, item.entry.disk_kind))) + .chain( + // Append entries that might be tracked in `/target/`. + index + .prefixed_entries(target_prefix.as_ref()) + .unwrap_or_default() + .iter() + .filter(|entry| { + // probably not needed as conflicts prevent this to run, but let's be explicit. + entry.stage() == 0 + }) + .map(|entry| { + ( + entry.path(&index).to_owned(), + // Do not trust what's recorded in the index, enforce checking the disk. + // This traversal is not part of a `status()`, and tracking things in `target/` + // is rare. + None, + ) + }) + .map(Ok), + ) + { + let (rela_path, kind) = item?; let file_path = root.join(gix::path::from_bstr(rela_path)); if file_path.file_name().and_then(|name| name.to_str()) == Some("Cargo.toml") { // Keep track of all sub-packages found and also strip out all @@ -652,16 +568,16 @@ impl<'gctx> PathSource<'gctx> { let path = file_path.parent().unwrap(); if path != pkg_path { warn!("subpackage found: {}", path.display()); - paths.retain(|p| !p.starts_with(path)); + files.retain(|p| !p.starts_with(path)); subpackages_found.push(path.to_path_buf()); - return Ok(()); + continue; } } // If this file is part of any other sub-package we've found so far, // skip it. if subpackages_found.iter().any(|p| file_path.starts_with(p)) { - return Ok(()); + continue; } let is_dir = kind.map_or(false, |kind| { @@ -679,21 +595,20 @@ impl<'gctx> PathSource<'gctx> { // .git repositories. match gix::open_opts(&file_path, gix::open::Options::isolated()) { Ok(sub_repo) => { - let files = source.list_files_gix(pkg, &sub_repo, filter)?; - paths.extend(files); + files.extend(self.list_files_gix(pkg, &sub_repo, filter)?); } - _ => { - source.walk(&file_path, paths, false, filter)?; + Err(_) => { + self.walk(&file_path, &mut files, false, filter)?; } } } else if (filter)(&file_path, is_dir) { assert!(!is_dir); - // We found a file! warn!(" found {}", file_path.display()); - paths.push(file_path); + files.push(file_path); } - Ok(()) } + + return Ok(files); } /// Lists files relevant to building this package inside this source by From 363d2da187d2ae7a9273a29cf612eadd11126939 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Mar 2024 15:10:08 +0100 Subject: [PATCH 06/11] address PR review (see details) * remove renovate group as it's not needed anymore * repository discovery will open with isolation --- .github/renovate.json5 | 9 --------- src/cargo/sources/path.rs | 8 ++++++-- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 401956f16d9..963a28ac14a 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -67,15 +67,6 @@ internalChecksFilter: 'strict', groupName: 'msrv', }, - { - matchManagers: [ - 'cargo', - ], - matchPackageNames: [ - 'gix', - ], - groupName: 'gix', - }, // Goals: // - Rollup safe upgrades to reduce CI runner load // - Have lockfile and manifest in-sync (implicit rules) diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index 758630e8780..2ce54e624cc 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -263,8 +263,12 @@ impl<'gctx> PathSource<'gctx> { /// Returns [`Some(gix::Repository)`](gix::Repository) if there is a sibling `Cargo.toml` and `.git` /// directory; otherwise, the caller should fall back on full file list. fn discover_gix_repo(&self, root: &Path) -> CargoResult> { - let repo = match gix::discover(root) { - Ok(repo) => repo, + let mut mapping = gix::sec::trust::Mapping::default(); + mapping.full = gix::open::Options::isolated(); + mapping.reduced = gix::open::Options::isolated(); + let repo = match gix::ThreadSafeRepository::discover_opts(root, Default::default(), mapping) + { + Ok(repo) => repo.to_thread_local(), Err(e) => { tracing::debug!( "could not discover git repo at or above {}: {}", From acef08407bda364fe3f44ebeda671315f1821dff Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 20 Mar 2024 22:06:27 +0100 Subject: [PATCH 07/11] add custom feature toggle for `gitoxide` usage in `list_files()` --- src/cargo/core/features.rs | 8 +++++++- src/cargo/sources/path.rs | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/cargo/core/features.rs b/src/cargo/core/features.rs index 76ccdadb613..f85392352d6 100644 --- a/src/cargo/core/features.rs +++ b/src/cargo/core/features.rs @@ -909,6 +909,8 @@ fn parse_git(it: impl Iterator>) -> CargoResult Self { GitoxideFeatures { fetch: true, + list_files: true, checkout: true, internal_use_git2: false, } @@ -932,6 +935,7 @@ impl GitoxideFeatures { fn safe() -> Self { GitoxideFeatures { fetch: true, + list_files: true, checkout: true, internal_use_git2: false, } @@ -944,6 +948,7 @@ fn parse_gitoxide( let mut out = GitoxideFeatures::default(); let GitoxideFeatures { fetch, + list_files, checkout, internal_use_git2, } = &mut out; @@ -952,9 +957,10 @@ fn parse_gitoxide( match e.as_ref() { "fetch" => *fetch = true, "checkout" => *checkout = true, + "list-files" => *list_files = true, "internal-use-git2" => *internal_use_git2 = true, _ => { - bail!("unstable 'gitoxide' only takes `fetch` and 'checkout' as valid input, for shallow fetches see `-Zgit=shallow-index,shallow-deps`") + bail!("unstable 'gitoxide' only takes `fetch`, `list-files` and 'checkout' as valid input, for shallow fetches see `-Zgit=shallow-index,shallow-deps`") } } } diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index 2ce54e624cc..1b3008ce975 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -145,7 +145,7 @@ impl<'gctx> PathSource<'gctx> { .gctx .cli_unstable() .gitoxide - .map_or(false, |features| features.fetch) + .map_or(false, |features| features.list_files) { self.discover_gix_repo(root)?.map(Git2OrGixRepository::Gix) } else { From 8375cf40eec17c47daa5f3ce6276f3decf097c95 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 21 Mar 2024 07:05:46 +0100 Subject: [PATCH 08/11] assure user and system configuration for .gitignore/exclude files is picked up --- src/cargo/sources/path.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index 1b3008ce975..de80dd7a69f 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -263,11 +263,7 @@ impl<'gctx> PathSource<'gctx> { /// Returns [`Some(gix::Repository)`](gix::Repository) if there is a sibling `Cargo.toml` and `.git` /// directory; otherwise, the caller should fall back on full file list. fn discover_gix_repo(&self, root: &Path) -> CargoResult> { - let mut mapping = gix::sec::trust::Mapping::default(); - mapping.full = gix::open::Options::isolated(); - mapping.reduced = gix::open::Options::isolated(); - let repo = match gix::ThreadSafeRepository::discover_opts(root, Default::default(), mapping) - { + let repo = match gix::ThreadSafeRepository::discover(root) { Ok(repo) => repo.to_thread_local(), Err(e) => { tracing::debug!( @@ -597,7 +593,7 @@ impl<'gctx> PathSource<'gctx> { // This could be a submodule, or a sub-repository. In any case, we prefer to walk // it with git-support to leverage ignored files and to avoid pulling in entire // .git repositories. - match gix::open_opts(&file_path, gix::open::Options::isolated()) { + match gix::open(&file_path) { Ok(sub_repo) => { files.extend(self.list_files_gix(pkg, &sub_repo, filter)?); } From ac68aa1c7cd9d979a57d310f61950958b2634cb1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 22 Mar 2024 07:18:37 +0100 Subject: [PATCH 09/11] Don't call .ok() before `unwrap_or()` It's not necessary, and adds noise. --- src/cargo/sources/path.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index de80dd7a69f..a04a779da25 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -260,8 +260,9 @@ impl<'gctx> PathSource<'gctx> { Ok(None) } - /// Returns [`Some(gix::Repository)`](gix::Repository) if there is a sibling `Cargo.toml` and `.git` - /// directory; otherwise, the caller should fall back on full file list. + /// Returns [`Some(gix::Repository)`](gix::Repository) if the discovered repository + /// (searched upwards from `root`) contains a tracked `/Cargo.toml`. + /// Otherwise, the caller should fall back on full file list. fn discover_gix_repo(&self, root: &Path) -> CargoResult> { let repo = match gix::ThreadSafeRepository::discover(root) { Ok(repo) => repo.to_thread_local(), @@ -496,7 +497,7 @@ impl<'gctx> PathSource<'gctx> { let pkg_path = pkg.root(); let mut target_prefix; - let repo_relative_pkg_path = pkg_path.strip_prefix(root).ok().unwrap_or(Path::new("")); + let repo_relative_pkg_path = pkg_path.strip_prefix(root).unwrap_or(Path::new("")); let pathspec = { let mut include = gix::path::to_unix_separators_on_windows(gix::path::into_bstr( From dbf1b6affdeaa3118776c324a64248d570a90e83 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 23 Mar 2024 08:41:27 +0100 Subject: [PATCH 10/11] Simplify pathspec generation in `list_files_gix` Co-authored-by: Arlo Siemsen --- src/cargo/sources/path.rs | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index a04a779da25..f6817cba24f 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -496,33 +496,23 @@ impl<'gctx> PathSource<'gctx> { ); let pkg_path = pkg.root(); - let mut target_prefix; let repo_relative_pkg_path = pkg_path.strip_prefix(root).unwrap_or(Path::new("")); + let target_prefix = gix::path::to_unix_separators_on_windows(gix::path::into_bstr( + repo_relative_pkg_path.join("target"), + )); + let package_prefix = + gix::path::to_unix_separators_on_windows(gix::path::into_bstr(repo_relative_pkg_path)); let pathspec = { - let mut include = gix::path::to_unix_separators_on_windows(gix::path::into_bstr( - repo_relative_pkg_path, - )); + // Include the package root. + let mut include = BString::from(":/"); + include.push_str(package_prefix.as_ref()); - target_prefix = include.clone(); - target_prefix.to_mut().push_str(if include.is_empty() { - "target/" - } else { - "/target/" - }); - - // The `target` directory is never included if it is in the package root. - let exclude = { - let mut pattern = include.clone(); - pattern - .to_mut() - .insert_str(0, if include.is_empty() { ":!" } else { ":!/" }); - pattern.to_mut().push_str(b"/target/"); - pattern - }; - include.to_mut().insert_str(0, ":/"); + // Exclude the target directory. + let mut exclude = BString::from(":!"); + exclude.push_str(target_prefix.as_ref()); - vec![include.into_owned(), exclude.into_owned()] + vec![include, exclude] }; let mut files = Vec::::new(); From 91781609a54e42fa7009b61f349c03585e6eefbe Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 23 Mar 2024 08:52:12 +0100 Subject: [PATCH 11/11] Assure top-level refspecs and tailing '/' for target prefix Top-level pathspecs are needed to assure they are not affected by the CWD. The trailing `/` in `'target` is needed to assure excluded items are in a folder, and that only entries in that folder are extracted from the index. --- src/cargo/sources/path.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index f6817cba24f..7d56b7194f9 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -13,7 +13,7 @@ use crate::util::{internal, CargoResult, GlobalContext}; use anyhow::Context as _; use cargo_util::paths; use filetime::FileTime; -use gix::bstr::ByteVec; +use gix::bstr::{BString, ByteVec}; use gix::dir::entry::Status; use ignore::gitignore::GitignoreBuilder; use tracing::{trace, warn}; @@ -498,7 +498,7 @@ impl<'gctx> PathSource<'gctx> { let pkg_path = pkg.root(); let repo_relative_pkg_path = pkg_path.strip_prefix(root).unwrap_or(Path::new("")); let target_prefix = gix::path::to_unix_separators_on_windows(gix::path::into_bstr( - repo_relative_pkg_path.join("target"), + repo_relative_pkg_path.join("target/"), )); let package_prefix = gix::path::to_unix_separators_on_windows(gix::path::into_bstr(repo_relative_pkg_path)); @@ -509,7 +509,7 @@ impl<'gctx> PathSource<'gctx> { include.push_str(package_prefix.as_ref()); // Exclude the target directory. - let mut exclude = BString::from(":!"); + let mut exclude = BString::from(":!/"); exclude.push_str(target_prefix.as_ref()); vec![include, exclude]