diff --git a/toolsrc/CMakeLists.txt b/toolsrc/CMakeLists.txt index ca23db41318394..1db8bc36a494c6 100644 --- a/toolsrc/CMakeLists.txt +++ b/toolsrc/CMakeLists.txt @@ -35,7 +35,7 @@ if(GCC OR CLANG) endif() file(GLOB_RECURSE VCPKGLIB_SOURCES src/vcpkg/*.cpp) -file(GLOB_RECURSE VCPKGTEST_SOURCES src/vcpkg-tests/*.cpp) +file(GLOB_RECURSE VCPKGTEST_SOURCES src/vcpkg-test/*.cpp) if (DEFINE_DISABLE_METRICS) set(DISABLE_METRICS_VALUE "1") @@ -52,7 +52,11 @@ add_executable(vcpkg-test EXCLUDE_FROM_ALL ${VCPKGTEST_SOURCES} ${VCPKGLIB_SOURC target_compile_definitions(vcpkg-test PRIVATE -DDISABLE_METRICS=${DISABLE_METRICS_VALUE}) target_include_directories(vcpkg-test PRIVATE include) -foreach(TEST_NAME arguments chrono dependencies paragraph plan specifier supports) +foreach(TEST_NAME + arguments chrono dependencies files + paragraph plan specifier statusparagraphs + strings supports update +) add_test(${TEST_NAME} vcpkg-test [${TEST_NAME}]) endforeach() @@ -91,3 +95,4 @@ endif() set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) target_link_libraries(vcpkg PRIVATE Threads::Threads) +target_link_libraries(vcpkg-test PRIVATE Threads::Threads) diff --git a/toolsrc/include/vcpkg-tests/catch.h b/toolsrc/include/vcpkg-test/catch.h similarity index 100% rename from toolsrc/include/vcpkg-tests/catch.h rename to toolsrc/include/vcpkg-test/catch.h diff --git a/toolsrc/include/vcpkg-tests/util.h b/toolsrc/include/vcpkg-test/util.h similarity index 79% rename from toolsrc/include/vcpkg-tests/util.h rename to toolsrc/include/vcpkg-test/util.h index fe4ee0eec9a5c5..fa650abc850286 100644 --- a/toolsrc/include/vcpkg-tests/util.h +++ b/toolsrc/include/vcpkg-test/util.h @@ -1,3 +1,4 @@ +#include #include #include @@ -30,4 +31,12 @@ T&& unwrap(vcpkg::Optional&& opt) return std::move(*opt.get()); } +extern const bool SYMLINKS_ALLOWED; + +extern const fs::path TEMPORARY_DIRECTORY; + +void create_symlink(const fs::path& file, const fs::path& target, std::error_code& ec); + +void create_directory_symlink(const fs::path& file, const fs::path& target, std::error_code& ec); + } diff --git a/toolsrc/include/vcpkg/base/files.h b/toolsrc/include/vcpkg/base/files.h index 3ea0d60366d3e8..a5e04db2513213 100644 --- a/toolsrc/include/vcpkg/base/files.h +++ b/toolsrc/include/vcpkg/base/files.h @@ -13,13 +13,60 @@ namespace fs using stdfs::file_status; using stdfs::file_type; using stdfs::path; + using stdfs::perms; using stdfs::u8path; - inline bool is_regular_file(file_status s) { return stdfs::is_regular_file(s); } - inline bool is_directory(file_status s) { return stdfs::is_directory(s); } - inline bool is_symlink(file_status s) { return stdfs::is_symlink(s); } + /* + std::experimental::filesystem's file_status and file_type are broken in + the presence of symlinks -- a symlink is treated as the object it points + to for `symlink_status` and `symlink_type` + */ + + using stdfs::status; + + // we want to poison ADL with these niebloids + + namespace detail + { + struct symlink_status_t + { + file_status operator()(const path& p, std::error_code& ec) const noexcept; + file_status operator()(const path& p, vcpkg::LineInfo li) const noexcept; + }; + struct is_symlink_t + { + inline bool operator()(file_status s) const { return stdfs::is_symlink(s); } + }; + struct is_regular_file_t + { + inline bool operator()(file_status s) const { return stdfs::is_regular_file(s); } + }; + struct is_directory_t + { + inline bool operator()(file_status s) const { return stdfs::is_directory(s); } + }; + } + + constexpr detail::symlink_status_t symlink_status{}; + constexpr detail::is_symlink_t is_symlink{}; + constexpr detail::is_regular_file_t is_regular_file{}; + constexpr detail::is_directory_t is_directory{}; } +/* + if someone attempts to use unqualified `symlink_status` or `is_symlink`, + they might get the ADL version, which is broken. + Therefore, put `symlink_status` in the global namespace, so that they get + our symlink_status. + + We also want to poison the ADL on is_regular_file and is_directory, because + we don't want people calling these functions on paths +*/ +using fs::is_directory; +using fs::is_regular_file; +using fs::is_symlink; +using fs::symlink_status; + namespace vcpkg::Files { struct Filesystem @@ -44,7 +91,9 @@ namespace vcpkg::Files std::error_code& ec) = 0; bool remove(const fs::path& path, LineInfo linfo); virtual bool remove(const fs::path& path, std::error_code& ec) = 0; - virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec) = 0; + + virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec, fs::path& failure_point) = 0; + std::uintmax_t remove_all(const fs::path& path, LineInfo li); virtual bool exists(const fs::path& path) const = 0; virtual bool is_directory(const fs::path& path) const = 0; virtual bool is_regular_file(const fs::path& path) const = 0; diff --git a/toolsrc/include/vcpkg/base/strings.h b/toolsrc/include/vcpkg/base/strings.h index d553d1d4177730..d7de9b0b23581f 100644 --- a/toolsrc/include/vcpkg/base/strings.h +++ b/toolsrc/include/vcpkg/base/strings.h @@ -184,4 +184,9 @@ namespace vcpkg::Strings const char* search(StringView haystack, StringView needle); bool contains(StringView haystack, StringView needle); + + // base 32 encoding, since base64 encoding requires lowercase letters, + // which are not distinct from uppercase letters on macOS or Windows filesystems. + // follows RFC 4648 + std::string b32_encode(std::uint64_t x) noexcept; } diff --git a/toolsrc/include/vcpkg/base/work_queue.h b/toolsrc/include/vcpkg/base/work_queue.h new file mode 100644 index 00000000000000..70142e110c311e --- /dev/null +++ b/toolsrc/include/vcpkg/base/work_queue.h @@ -0,0 +1,230 @@ +#pragma once + +#include +#include +#include + +namespace vcpkg +{ + template + struct WorkQueue; + + namespace detail + { + // for SFINAE purposes, keep out of the class + template + auto call_moved_action(Action& action, + const WorkQueue& work_queue, + ThreadLocalData& tld) -> decltype(static_cast(std::move(action)(tld, work_queue))) + { + std::move(action)(tld, work_queue); + } + + template + auto call_moved_action(Action& action, const WorkQueue&, ThreadLocalData& tld) + -> decltype(static_cast(std::move(action)(tld))) + { + std::move(action)(tld); + } + } + + template + struct WorkQueue + { + template + WorkQueue(std::uint16_t num_threads, LineInfo li, const F& tld_init) noexcept + { + m_line_info = li; + + set_unjoined_workers(num_threads); + m_threads.reserve(num_threads); + for (std::size_t i = 0; i < num_threads; ++i) + { + m_threads.push_back(std::thread(Worker{this, tld_init()})); + } + } + + WorkQueue(WorkQueue const&) = delete; + WorkQueue(WorkQueue&&) = delete; + + ~WorkQueue() + { + auto lck = std::unique_lock(m_mutex); + if (!is_joined(m_state)) + { + Checks::exit_with_message(m_line_info, "Failed to call join() on a WorkQueue that was destroyed"); + } + } + + // should only be called once; anything else is an error + void run(LineInfo li) + { + // this should _not_ be locked before `run()` is called; however, we + // want to terminate if someone screws up, rather than cause UB + auto lck = std::unique_lock(m_mutex); + + if (m_state != State::BeforeRun) + { + Checks::exit_with_message(li, "Attempted to run() twice"); + } + + m_state = State::Running; + } + + // runs all remaining tasks, and blocks on their finishing + // if this is called in an existing task, _will block forever_ + // DO NOT DO THAT + // thread-unsafe + void join(LineInfo li) + { + { + auto lck = std::unique_lock(m_mutex); + if (is_joined(m_state)) + { + Checks::exit_with_message(li, "Attempted to call join() more than once"); + } + else if (m_state == State::Terminated) + { + m_state = State::TerminatedJoined; + } + else + { + m_state = State::Joined; + } + } + + while (unjoined_workers()) + { + if (!running_workers()) + { + m_cv.notify_one(); + } + } + + // wait for all threads to join + for (auto& thrd : m_threads) + { + thrd.join(); + } + } + + // useful in the case of errors + // doesn't stop any existing running tasks + // returns immediately, so that one can call this in a task + void terminate() const + { + { + auto lck = std::unique_lock(m_mutex); + if (is_joined(m_state)) + { + m_state = State::TerminatedJoined; + } + else + { + m_state = State::Terminated; + } + } + m_cv.notify_all(); + } + + void enqueue_action(Action a) const + { + { + auto lck = std::unique_lock(m_mutex); + m_actions.push_back(std::move(a)); + + if (m_state == State::BeforeRun) return; + } + m_cv.notify_one(); + } + + private: + struct Worker + { + const WorkQueue* work_queue; + ThreadLocalData tld; + + void operator()() + { + // unlocked when waiting, or when in the action + // locked otherwise + auto lck = std::unique_lock(work_queue->m_mutex); + + work_queue->m_cv.wait(lck, [&] { return work_queue->m_state != State::BeforeRun; }); + + work_queue->increment_running_workers(); + for (;;) + { + const auto state = work_queue->m_state; + + if (is_terminated(state)) + { + break; + } + + if (work_queue->m_actions.empty()) + { + if (state == State::Running || work_queue->running_workers() > 1) + { + work_queue->decrement_running_workers(); + work_queue->m_cv.wait(lck); + work_queue->increment_running_workers(); + continue; + } + + // the queue is joining, and we are the only worker running + // no more work! + break; + } + + Action action = std::move(work_queue->m_actions.back()); + work_queue->m_actions.pop_back(); + + lck.unlock(); + work_queue->m_cv.notify_one(); + detail::call_moved_action(action, *work_queue, tld); + lck.lock(); + } + + work_queue->decrement_running_workers(); + work_queue->decrement_unjoined_workers(); + } + }; + + enum class State : std::int16_t + { + // can only exist upon construction + BeforeRun = -1, + + Running, + Joined, + Terminated, + TerminatedJoined, + }; + + static bool is_terminated(State st) { return st == State::Terminated || st == State::TerminatedJoined; } + + static bool is_joined(State st) { return st == State::Joined || st == State::TerminatedJoined; } + + mutable std::mutex m_mutex{}; + // these are all under m_mutex + mutable State m_state = State::BeforeRun; + mutable std::vector m_actions{}; + mutable std::condition_variable m_cv{}; + + mutable std::atomic m_workers; + // = unjoined_workers << 16 | running_workers + + void set_unjoined_workers(std::uint16_t threads) { m_workers = std::uint32_t(threads) << 16; } + void decrement_unjoined_workers() const { m_workers -= 1 << 16; } + + std::uint16_t unjoined_workers() const { return std::uint16_t(m_workers >> 16); } + + void increment_running_workers() const { ++m_workers; } + void decrement_running_workers() const { --m_workers; } + std::uint16_t running_workers() const { return std::uint16_t(m_workers); } + + std::vector m_threads{}; + LineInfo m_line_info; + }; +} diff --git a/toolsrc/src/vcpkg-tests/arguments.cpp b/toolsrc/src/vcpkg-test/arguments.cpp similarity index 99% rename from toolsrc/src/vcpkg-tests/arguments.cpp rename to toolsrc/src/vcpkg-test/arguments.cpp index 8c625be0f81321..3fe5fa420a1a4f 100644 --- a/toolsrc/src/vcpkg-tests/arguments.cpp +++ b/toolsrc/src/vcpkg-test/arguments.cpp @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/toolsrc/src/vcpkg-tests/catch.cpp b/toolsrc/src/vcpkg-test/catch.cpp similarity index 85% rename from toolsrc/src/vcpkg-tests/catch.cpp rename to toolsrc/src/vcpkg-test/catch.cpp index 701dcb39ae0d5c..8b5d1aa15b4875 100644 --- a/toolsrc/src/vcpkg-tests/catch.cpp +++ b/toolsrc/src/vcpkg-test/catch.cpp @@ -1,5 +1,5 @@ #define CATCH_CONFIG_RUNNER -#include +#include #include diff --git a/toolsrc/src/vcpkg-tests/chrono.cpp b/toolsrc/src/vcpkg-test/chrono.cpp similarity index 96% rename from toolsrc/src/vcpkg-tests/chrono.cpp rename to toolsrc/src/vcpkg-test/chrono.cpp index c164753f944927..306217ad07da6e 100644 --- a/toolsrc/src/vcpkg-tests/chrono.cpp +++ b/toolsrc/src/vcpkg-test/chrono.cpp @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/toolsrc/src/vcpkg-tests/dependencies.cpp b/toolsrc/src/vcpkg-test/dependencies.cpp similarity index 96% rename from toolsrc/src/vcpkg-tests/dependencies.cpp rename to toolsrc/src/vcpkg-test/dependencies.cpp index 0dee6f296ee230..5ed05cc078c5cc 100644 --- a/toolsrc/src/vcpkg-tests/dependencies.cpp +++ b/toolsrc/src/vcpkg-test/dependencies.cpp @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/toolsrc/src/vcpkg-test/files.cpp b/toolsrc/src/vcpkg-test/files.cpp new file mode 100644 index 00000000000000..9e14cec0c302d8 --- /dev/null +++ b/toolsrc/src/vcpkg-test/files.cpp @@ -0,0 +1,121 @@ +#include +#include + +#include +#include + +#include +#include + +#include + +using vcpkg::Test::SYMLINKS_ALLOWED; +using vcpkg::Test::TEMPORARY_DIRECTORY; + +namespace +{ + using uid = std::uniform_int_distribution; + + std::mt19937_64 get_urbg(std::uint64_t index) + { + // smallest prime > 2**63 - 1 + return std::mt19937_64{index + 9223372036854775837ULL}; + } + + std::string get_random_filename(std::mt19937_64& urbg) { return vcpkg::Strings::b32_encode(uid{}(urbg)); } + + void create_directory_tree(std::mt19937_64& urbg, + vcpkg::Files::Filesystem& fs, + std::uint64_t depth, + const fs::path& base) + { + std::random_device rd; + constexpr std::uint64_t max_depth = 5; + constexpr std::uint64_t width = 5; + + // we want ~70% of our "files" to be directories, and then a third + // each of the remaining ~30% to be regular files, directory symlinks, + // and regular symlinks + constexpr std::uint64_t directory_min_tag = 0; + constexpr std::uint64_t directory_max_tag = 6; + constexpr std::uint64_t regular_file_tag = 7; + constexpr std::uint64_t regular_symlink_tag = 8; + constexpr std::uint64_t directory_symlink_tag = 9; + + // if we're at the max depth, we only want to build non-directories + std::uint64_t file_type; + if (depth < max_depth) + { + file_type = uid{directory_min_tag, regular_symlink_tag}(urbg); + } + else + { + file_type = uid{regular_file_tag, regular_symlink_tag}(urbg); + } + + if (!SYMLINKS_ALLOWED && file_type > regular_file_tag) + { + file_type = regular_file_tag; + } + + std::error_code ec; + if (file_type <= directory_max_tag) + { + fs.create_directory(base, ec); + if (ec) { + INFO("File that failed: " << base); + REQUIRE_FALSE(ec); + } + + for (int i = 0; i < width; ++i) + { + create_directory_tree(urbg, fs, depth + 1, base / get_random_filename(urbg)); + } + } + else if (file_type == regular_file_tag) + { + // regular file + fs.write_contents(base, "", ec); + } + else if (file_type == regular_symlink_tag) + { + // regular symlink + fs.write_contents(base, "", ec); + REQUIRE_FALSE(ec); + auto base_link = base; + base_link.replace_filename(base.filename().u8string() + "-link"); + vcpkg::Test::create_symlink(base, base_link, ec); + } + else // type == directory_symlink_tag + { + // directory symlink + vcpkg::Test::create_directory_symlink(base / "..", base, ec); + } + + REQUIRE_FALSE(ec); + } +} + +TEST_CASE ("remove all", "[files]") +{ + auto urbg = get_urbg(0); + + fs::path temp_dir = TEMPORARY_DIRECTORY / get_random_filename(urbg); + + auto& fs = vcpkg::Files::get_real_filesystem(); + + std::error_code ec; + fs.create_directory(TEMPORARY_DIRECTORY, ec); + + REQUIRE_FALSE(ec); + + INFO("temp dir is: " << temp_dir); + + create_directory_tree(urbg, fs, 0, temp_dir); + + fs::path fp; + fs.remove_all(temp_dir, ec, fp); + REQUIRE_FALSE(ec); + + REQUIRE_FALSE(fs.exists(temp_dir)); +} diff --git a/toolsrc/src/vcpkg-tests/paragraph.cpp b/toolsrc/src/vcpkg-test/paragraph.cpp similarity index 99% rename from toolsrc/src/vcpkg-tests/paragraph.cpp rename to toolsrc/src/vcpkg-test/paragraph.cpp index 0fb85ec693e4e1..a95879cfa353cb 100644 --- a/toolsrc/src/vcpkg-tests/paragraph.cpp +++ b/toolsrc/src/vcpkg-test/paragraph.cpp @@ -1,5 +1,5 @@ -#include -#include +#include +#include #include diff --git a/toolsrc/src/vcpkg-tests/plan.cpp b/toolsrc/src/vcpkg-test/plan.cpp similarity index 99% rename from toolsrc/src/vcpkg-tests/plan.cpp rename to toolsrc/src/vcpkg-test/plan.cpp index 7ecab460b33cc0..049ef20662fe85 100644 --- a/toolsrc/src/vcpkg-tests/plan.cpp +++ b/toolsrc/src/vcpkg-test/plan.cpp @@ -1,5 +1,5 @@ -#include -#include +#include +#include #include #include diff --git a/toolsrc/src/vcpkg-tests/specifier.cpp b/toolsrc/src/vcpkg-test/specifier.cpp similarity index 99% rename from toolsrc/src/vcpkg-tests/specifier.cpp rename to toolsrc/src/vcpkg-test/specifier.cpp index 52ef044e798cfc..330a96d78a4236 100644 --- a/toolsrc/src/vcpkg-tests/specifier.cpp +++ b/toolsrc/src/vcpkg-test/specifier.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/toolsrc/src/vcpkg-tests/statusparagraphs.cpp b/toolsrc/src/vcpkg-test/statusparagraphs.cpp similarity index 97% rename from toolsrc/src/vcpkg-tests/statusparagraphs.cpp rename to toolsrc/src/vcpkg-test/statusparagraphs.cpp index df52ccb875b1ab..c0833e8ba0c331 100644 --- a/toolsrc/src/vcpkg-tests/statusparagraphs.cpp +++ b/toolsrc/src/vcpkg-test/statusparagraphs.cpp @@ -1,5 +1,5 @@ -#include -#include +#include +#include #include #include diff --git a/toolsrc/src/vcpkg-test/strings.cpp b/toolsrc/src/vcpkg-test/strings.cpp new file mode 100644 index 00000000000000..6b744eee65125c --- /dev/null +++ b/toolsrc/src/vcpkg-test/strings.cpp @@ -0,0 +1,33 @@ +#include + +#include + +#include +#include +#include + +TEST_CASE ("b32 encoding", "[strings]") +{ + using u64 = std::uint64_t; + + std::vector> map; + + map.emplace_back(0, "AAAAAAAAAAAAA"); + map.emplace_back(1, "BAAAAAAAAAAAA"); + + map.emplace_back(u64(1) << 32, "AAAAAAEAAAAAA"); + map.emplace_back((u64(1) << 32) + 1, "BAAAAAEAAAAAA"); + + map.emplace_back(0xE4D0'1065'D11E'0229, "JRA4RIXMQAUJO"); + map.emplace_back(0xA626'FE45'B135'07FF, "77BKTYWI6XJMK"); + map.emplace_back(0xEE36'D228'0C31'D405, "FAVDDGAFSWN4O"); + map.emplace_back(0x1405'64E7'FE7E'A88C, "MEK5H774ELBIB"); + map.emplace_back(0xFFFF'FFFF'FFFF'FFFF, "777777777777P"); + + std::string result; + for (const auto& pr : map) + { + result = vcpkg::Strings::b32_encode(pr.first); + REQUIRE(vcpkg::Strings::b32_encode(pr.first) == pr.second); + } +} diff --git a/toolsrc/src/vcpkg-tests/supports.cpp b/toolsrc/src/vcpkg-test/supports.cpp similarity index 98% rename from toolsrc/src/vcpkg-tests/supports.cpp rename to toolsrc/src/vcpkg-test/supports.cpp index c6c88bdbca51bd..8bd386da056ce5 100644 --- a/toolsrc/src/vcpkg-tests/supports.cpp +++ b/toolsrc/src/vcpkg-test/supports.cpp @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/toolsrc/src/vcpkg-tests/update.cpp b/toolsrc/src/vcpkg-test/update.cpp similarity index 98% rename from toolsrc/src/vcpkg-tests/update.cpp rename to toolsrc/src/vcpkg-test/update.cpp index 93a8f74a98027e..70b2f04c1ba6ba 100644 --- a/toolsrc/src/vcpkg-tests/update.cpp +++ b/toolsrc/src/vcpkg-test/update.cpp @@ -1,5 +1,5 @@ -#include -#include +#include +#include #include diff --git a/toolsrc/src/vcpkg-test/util.cpp b/toolsrc/src/vcpkg-test/util.cpp new file mode 100644 index 00000000000000..a80ab36a0f80c7 --- /dev/null +++ b/toolsrc/src/vcpkg-test/util.cpp @@ -0,0 +1,177 @@ +#include +#include + +#include +#include +#include + +// used to get the implementation specific compiler flags (i.e., __cpp_lib_filesystem) +#include + +#include +#include + +#if defined(_WIN32) +#include +#endif + +#define FILESYSTEM_SYMLINK_STD 0 +#define FILESYSTEM_SYMLINK_UNIX 1 +#define FILESYSTEM_SYMLINK_NONE 2 + +#if defined(__cpp_lib_filesystem) + +#define FILESYSTEM_SYMLINK FILESYSTEM_SYMLINK_STD +#include // required for filesystem::create_{directory_}symlink + +#elif !defined(_MSC_VER) + +#define FILESYSTEM_SYMLINK FILESYSTEM_SYMLINK_UNIX +#include + +#else + +#define FILESYSTEM_SYMLINK FILESYSTEM_SYMLINK_NONE + +#endif + +namespace vcpkg::Test +{ + std::unique_ptr make_status_pgh(const char* name, + const char* depends, + const char* default_features, + const char* triplet) + { + using Pgh = std::unordered_map; + return std::make_unique(Pgh{{"Package", name}, + {"Version", "1"}, + {"Architecture", triplet}, + {"Multi-Arch", "same"}, + {"Depends", depends}, + {"Default-Features", default_features}, + {"Status", "install ok installed"}}); + } + + std::unique_ptr make_status_feature_pgh(const char* name, + const char* feature, + const char* depends, + const char* triplet) + { + using Pgh = std::unordered_map; + return std::make_unique(Pgh{{"Package", name}, + {"Version", "1"}, + {"Feature", feature}, + {"Architecture", triplet}, + {"Multi-Arch", "same"}, + {"Depends", depends}, + {"Status", "install ok installed"}}); + } + + PackageSpec unsafe_pspec(std::string name, Triplet t) + { + auto m_ret = PackageSpec::from_name_and_triplet(name, t); + REQUIRE(m_ret.has_value()); + return m_ret.value_or_exit(VCPKG_LINE_INFO); + } + + static bool system_allows_symlinks() + { +#if defined(_WIN32) + if (!__cpp_lib_filesystem) + { + return false; + } + + HKEY key; + bool allow_symlinks = true; + + const auto status = RegOpenKeyExW( + HKEY_LOCAL_MACHINE, LR"(SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock)", 0, 0, &key); + + if (status == ERROR_FILE_NOT_FOUND) + { + allow_symlinks = false; + std::clog << "Symlinks are not allowed on this system\n"; + } + + if (status == ERROR_SUCCESS) RegCloseKey(key); + + return allow_symlinks; +#else + return true; +#endif + } + + static fs::path internal_temporary_directory() + { +#if defined(_WIN32) + wchar_t* tmp = static_cast(std::calloc(32'767, 2)); + + if (!GetEnvironmentVariableW(L"TEMP", tmp, 32'767)) + { + std::cerr << "No temporary directory found.\n"; + std::abort(); + } + + fs::path result = tmp; + std::free(tmp); + + return result / L"vcpkg-test"; +#else + return "/tmp/vcpkg-test"; +#endif + } + + const bool SYMLINKS_ALLOWED = system_allows_symlinks(); + const fs::path TEMPORARY_DIRECTORY = internal_temporary_directory(); + +#if FILESYSTEM_SYMLINK == FILSYSTEM_SYMLINK_NONE + constexpr inline char no_filesystem_message[] = + " doesn't exist; on windows, we don't attempt to use the win32 calls to create symlinks"; +#endif + + void create_symlink(const fs::path& target, const fs::path& file, std::error_code& ec) + { +#if FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_STD + if (SYMLINKS_ALLOWED) + { + std::filesystem::path targetp = target.native(); + std::filesystem::path filep = file.native(); + + std::filesystem::create_symlink(targetp, filep); + } + else + { + vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, "Symlinks are not allowed on this system"); + } +#elif FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_UNIX + if (symlink(target.c_str(), file.c_str()) != 0) + { + ec.assign(errno, std::system_category()); + } +#else + vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, no_filesystem_message); +#endif + } + + void create_directory_symlink(const fs::path& target, const fs::path& file, std::error_code& ec) + { +#if FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_STD + if (SYMLINKS_ALLOWED) + { + std::filesystem::path targetp = target.native(); + std::filesystem::path filep = file.native(); + + std::filesystem::create_symlink(targetp, filep); + } + else + { + vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, "Symlinks are not allowed on this system"); + } +#elif FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_UNIX + ::vcpkg::Test::create_symlink(target, file, ec); +#else + vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, no_filesystem_message); +#endif + } +} diff --git a/toolsrc/src/vcpkg-tests/util.cpp b/toolsrc/src/vcpkg-tests/util.cpp deleted file mode 100644 index 54102f1aaa79e7..00000000000000 --- a/toolsrc/src/vcpkg-tests/util.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include -#include - -#include - -#include - -namespace vcpkg::Test -{ - std::unique_ptr make_status_pgh(const char* name, - const char* depends, - const char* default_features, - const char* triplet) - { - using Pgh = std::unordered_map; - return std::make_unique(Pgh{{"Package", name}, - {"Version", "1"}, - {"Architecture", triplet}, - {"Multi-Arch", "same"}, - {"Depends", depends}, - {"Default-Features", default_features}, - {"Status", "install ok installed"}}); - } - - std::unique_ptr make_status_feature_pgh(const char* name, - const char* feature, - const char* depends, - const char* triplet) - { - using Pgh = std::unordered_map; - return std::make_unique(Pgh{{"Package", name}, - {"Version", "1"}, - {"Feature", feature}, - {"Architecture", triplet}, - {"Multi-Arch", "same"}, - {"Depends", depends}, - {"Status", "install ok installed"}}); - } - - PackageSpec unsafe_pspec(std::string name, Triplet t) - { - auto m_ret = PackageSpec::from_name_and_triplet(name, t); - REQUIRE(m_ret.has_value()); - return m_ret.value_or_exit(VCPKG_LINE_INFO); - } - -} diff --git a/toolsrc/src/vcpkg/archives.cpp b/toolsrc/src/vcpkg/archives.cpp index 69a916828d1599..d22e841de3b865 100644 --- a/toolsrc/src/vcpkg/archives.cpp +++ b/toolsrc/src/vcpkg/archives.cpp @@ -15,9 +15,10 @@ namespace vcpkg::Archives #endif ; + fs.remove_all(to_path, VCPKG_LINE_INFO); + fs.remove_all(to_path_partial, VCPKG_LINE_INFO); + // TODO: check this error code std::error_code ec; - fs.remove_all(to_path, ec); - fs.remove_all(to_path_partial, ec); fs.create_directories(to_path_partial, ec); const auto ext = archive.extension(); #if defined(_WIN32) diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index 5099795e9ea426..6c6945e44aaee2 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #if defined(__linux__) || defined(__APPLE__) #include @@ -20,6 +21,59 @@ #include #endif +namespace fs::detail +{ + file_status symlink_status_t::operator()(const path& p, std::error_code& ec) const noexcept + { +#if defined(_WIN32) + static_cast(ec); + + /* + do not find the permissions of the file -- it's unnecessary for the + things that vcpkg does. + if one were to add support for this in the future, one should look + into GetFileSecurityW + */ + perms permissions = perms::unknown; + + WIN32_FILE_ATTRIBUTE_DATA file_attributes; + file_type ft = file_type::unknown; + if (!GetFileAttributesExW(p.c_str(), GetFileExInfoStandard, &file_attributes)) + { + ft = file_type::not_found; + } + else if (file_attributes.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) + { + // check for reparse point -- if yes, then symlink + ft = file_type::symlink; + } + else if (file_attributes.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + { + ft = file_type::directory; + } + else + { + // otherwise, the file is a regular file + ft = file_type::regular; + } + + return file_status(ft, permissions); + +#else + return stdfs::symlink_status(p, ec); +#endif + } + + file_status symlink_status_t::operator()(const path& p, vcpkg::LineInfo li) const noexcept + { + std::error_code ec; + auto result = symlink_status(p, ec); + if (ec) vcpkg::Checks::exit_with_message(li, "error getting status of path %s: %s", p.string(), ec.message()); + + return result; + } +} + namespace vcpkg::Files { static const std::regex FILESYSTEM_INVALID_CHARACTERS_REGEX = std::regex(R"([\/:*?"<>|])"); @@ -63,6 +117,25 @@ namespace vcpkg::Files if (ec) Checks::exit_with_message(linfo, "error writing lines: %s: %s", path.u8string(), ec.message()); } + std::uintmax_t Filesystem::remove_all(const fs::path& path, LineInfo li) + { + std::error_code ec; + fs::path failure_point; + + const auto result = this->remove_all(path, ec, failure_point); + + if (ec) + { + Checks::exit_with_message(li, + "Failure to remove_all(%s) due to file %s: %s", + path.string(), + failure_point.string(), + ec.message()); + } + + return result; + } + struct RealFilesystem final : Filesystem { virtual Expected read_contents(const fs::path& file_path) const override @@ -87,7 +160,7 @@ namespace vcpkg::Files file_stream.read(&output[0], length); file_stream.close(); - return std::move(output); + return output; } virtual Expected> read_lines(const fs::path& file_path) const override { @@ -105,7 +178,7 @@ namespace vcpkg::Files } file_stream.close(); - return std::move(output); + return output; } virtual fs::path find_file_recursively_up(const fs::path& starting_dir, const std::string& filename) const override @@ -254,28 +327,163 @@ namespace vcpkg::Files #endif } virtual bool remove(const fs::path& path, std::error_code& ec) override { return fs::stdfs::remove(path, ec); } - virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec) override + virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec, fs::path& failure_point) override { - // Working around the currently buggy remove_all() - std::uintmax_t out = fs::stdfs::remove_all(path, ec); + /* + does not use the std::filesystem call since it is buggy, and can + have spurious errors before VS 2017 update 6, and on later versions + (as well as on macOS and Linux), this is just as fast and will have + fewer spurious errors due to locks. + */ + + /* + `remove` doesn't actually remove anything -- it simply moves the + files into a parent directory (which ends up being at `path`), + and then inserts `actually_remove{current_path}` into the work + queue. + */ + struct remove + { + struct tld + { + const fs::path& tmp_directory; + std::uint64_t index; + + std::atomic& files_deleted; - for (int i = 0; i < 5 && this->exists(path); i++) + std::mutex& ec_mutex; + std::error_code& ec; + fs::path& failure_point; + }; + + struct actually_remove; + using queue = WorkQueue; + + /* + if `current_path` is a directory, first `remove`s all + elements of the directory, then calls remove. + + else, just calls remove. + */ + struct actually_remove + { + fs::path current_path; + + void operator()(tld& info, const queue& queue) const + { + std::error_code ec; + const auto path_type = fs::symlink_status(current_path, ec).type(); + + if (check_ec(ec, info, queue, current_path)) return; + + if (path_type == fs::file_type::directory) + { + for (const auto& entry : fs::stdfs::directory_iterator(current_path)) + { + remove{}(entry, info, queue); + } + } + + if (fs::stdfs::remove(current_path, ec)) + { + info.files_deleted.fetch_add(1, std::memory_order_relaxed); + } + else + { + check_ec(ec, info, queue, current_path); + } + } + }; + + static bool check_ec(const std::error_code& ec, + tld& info, + const queue& queue, + const fs::path& failure_point) + { + if (ec) + { + queue.terminate(); + + auto lck = std::unique_lock(info.ec_mutex); + if (!info.ec) + { + info.ec = ec; + info.failure_point = failure_point; + } + + return true; + } + else + { + return false; + } + } + + void operator()(const fs::path& current_path, tld& info, const queue& queue) const + { + std::error_code ec; + + const auto tmp_name = Strings::b32_encode(info.index++); + const auto tmp_path = info.tmp_directory / tmp_name; + + fs::stdfs::rename(current_path, tmp_path, ec); + if (check_ec(ec, info, queue, current_path)) return; + + queue.enqueue_action(actually_remove{std::move(tmp_path)}); + } + }; + + const auto path_type = fs::symlink_status(path, ec).type(); + + std::atomic files_deleted{0}; + + if (path_type == fs::file_type::directory) { - using namespace std::chrono_literals; - std::this_thread::sleep_for(i * 100ms); - out += fs::stdfs::remove_all(path, ec); + std::uint64_t index = 0; + std::mutex ec_mutex; + + auto const tld_gen = [&] { + index += static_cast(1) << 32; + return remove::tld{path, index, files_deleted, ec_mutex, ec, failure_point}; + }; + + remove::queue queue{4, VCPKG_LINE_INFO, tld_gen}; + + // note: we don't actually start the queue running until the + // `join()`. This allows us to rename all the top-level files in + // peace, so that we don't get collisions. + auto main_tld = tld_gen(); + for (const auto& entry : fs::stdfs::directory_iterator(path)) + { + remove{}(entry, main_tld, queue); + } + + queue.join(VCPKG_LINE_INFO); } - if (this->exists(path)) + /* + we need to do backoff on the removal of the top level directory, + since we need to place all moved files into that top level + directory, and so we can only delete the directory after all the + lower levels have been deleted. + */ + for (int backoff = 0; backoff < 5; ++backoff) { - System::print2( - System::Color::warning, - "Some files in ", - path.u8string(), - " were unable to be removed. Close any editors operating in this directory and retry.\n"); + if (backoff) + { + using namespace std::chrono_literals; + auto backoff_time = 100ms * backoff; + std::this_thread::sleep_for(backoff_time); + } + + if (fs::stdfs::remove(path, ec)) + { + files_deleted.fetch_add(1, std::memory_order_relaxed); + break; + } } - return out; + return files_deleted; } virtual bool exists(const fs::path& path) const override { return fs::stdfs::exists(path); } virtual bool is_directory(const fs::path& path) const override { return fs::stdfs::is_directory(path); } @@ -307,11 +515,11 @@ namespace vcpkg::Files virtual fs::file_status status(const fs::path& path, std::error_code& ec) const override { - return fs::stdfs::status(path, ec); + return fs::status(path, ec); } virtual fs::file_status symlink_status(const fs::path& path, std::error_code& ec) const override { - return fs::stdfs::symlink_status(path, ec); + return fs::symlink_status(path, ec); } virtual void write_contents(const fs::path& file_path, const std::string& data, std::error_code& ec) override { diff --git a/toolsrc/src/vcpkg/base/strings.cpp b/toolsrc/src/vcpkg/base/strings.cpp index 54a74a7a16e0c2..46e78a36346783 100644 --- a/toolsrc/src/vcpkg/base/strings.cpp +++ b/toolsrc/src/vcpkg/base/strings.cpp @@ -288,3 +288,43 @@ bool Strings::contains(StringView haystack, StringView needle) { return Strings::search(haystack, needle) != haystack.end(); } + +namespace vcpkg::Strings +{ + namespace + { + template + std::string b32_encode_implementation(Integral x) + { + static_assert(std::is_integral::value, "b64url_encode must take an integer type"); + using Unsigned = std::make_unsigned_t; + auto value = static_cast(x); + + // 32 values, plus the implicit \0 + constexpr static char map[33] = "ABCDEFGHIJKLMNOP" + "QRSTUVWXYZ234567"; + + // log2(32) + constexpr static int shift = 5; + // 32 - 1 + constexpr static auto mask = 31; + + // ceiling(bitsize(Integral) / log2(32)) + constexpr static auto result_size = (sizeof(value) * 8 + shift - 1) / shift; + + std::string result; + result.reserve(result_size); + + for (std::size_t i = 0; i < result_size; ++i) + { + result.push_back(map[value & mask]); + value >>= shift; + } + + return result; + } + } + + std::string b32_encode(std::uint64_t x) noexcept { return b32_encode_implementation(x); } + +} diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index b99f300d3741c0..235adb819cf594 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -586,7 +586,8 @@ namespace vcpkg::Build if (fs.is_directory(file)) // Will only keep the logs { std::error_code ec; - fs.remove_all(file, ec); + fs::path failure_point; + fs.remove_all(file, ec, failure_point); } } } @@ -701,8 +702,8 @@ namespace vcpkg::Build auto& fs = paths.get_filesystem(); auto pkg_path = paths.package_dir(spec); + fs.remove_all(pkg_path, VCPKG_LINE_INFO); std::error_code ec; - fs.remove_all(pkg_path, ec); fs.create_directories(pkg_path, ec); auto files = fs.get_files_non_recursive(pkg_path); Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string()); @@ -886,7 +887,7 @@ namespace vcpkg::Build fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec); // clean up temporary directory - fs.remove_all(tmp_log_path, ec); + fs.remove_all(tmp_log_path, VCPKG_LINE_INFO); } } @@ -1053,7 +1054,7 @@ namespace vcpkg::Build { switch (maybe_option->second) { - case VcpkgTripletVar::TARGET_ARCHITECTURE : + case VcpkgTripletVar::TARGET_ARCHITECTURE : pre_build_info.target_architecture = variable_value; break; case VcpkgTripletVar::CMAKE_SYSTEM_NAME : diff --git a/toolsrc/src/vcpkg/commands.exportifw.cpp b/toolsrc/src/vcpkg/commands.exportifw.cpp index f0946110c28f1f..3d963a29761971 100644 --- a/toolsrc/src/vcpkg/commands.exportifw.cpp +++ b/toolsrc/src/vcpkg/commands.exportifw.cpp @@ -352,13 +352,15 @@ namespace vcpkg::Export::IFW System::print2("Generating repository ", repository_dir.generic_u8string(), "...\n"); std::error_code ec; + fs::path failure_point; Files::Filesystem& fs = paths.get_filesystem(); - fs.remove_all(repository_dir, ec); + fs.remove_all(repository_dir, ec, failure_point); Checks::check_exit(VCPKG_LINE_INFO, !ec, - "Could not remove outdated repository directory %s", - repository_dir.generic_u8string()); + "Could not remove outdated repository directory %s due to file %s", + repository_dir.generic_u8string(), + failure_point.string()); const auto cmd_line = Strings::format(R"("%s" --packages "%s" "%s" > nul)", repogen_exe.u8string(), @@ -414,16 +416,18 @@ namespace vcpkg::Export::IFW const VcpkgPaths& paths) { std::error_code ec; + fs::path failure_point; Files::Filesystem& fs = paths.get_filesystem(); // Prepare packages directory const fs::path ifw_packages_dir_path = get_packages_dir_path(export_id, ifw_options, paths); - fs.remove_all(ifw_packages_dir_path, ec); + fs.remove_all(ifw_packages_dir_path, ec, failure_point); Checks::check_exit(VCPKG_LINE_INFO, !ec, - "Could not remove outdated packages directory %s", - ifw_packages_dir_path.generic_u8string()); + "Could not remove outdated packages directory %s due to file %s", + ifw_packages_dir_path.generic_u8string(), + failure_point.string()); fs.create_directory(ifw_packages_dir_path, ec); Checks::check_exit( diff --git a/toolsrc/src/vcpkg/commands.portsdiff.cpp b/toolsrc/src/vcpkg/commands.portsdiff.cpp index b30c38f43a62dd..cddc274b8948a4 100644 --- a/toolsrc/src/vcpkg/commands.portsdiff.cpp +++ b/toolsrc/src/vcpkg/commands.portsdiff.cpp @@ -105,7 +105,7 @@ namespace vcpkg::Commands::PortsDiff std::map names_and_versions; for (auto&& port : all_ports) names_and_versions.emplace(port->core_paragraph->name, port->core_paragraph->version); - fs.remove_all(temp_checkout_path, ec); + fs.remove_all(temp_checkout_path, VCPKG_LINE_INFO); return names_and_versions; } diff --git a/toolsrc/src/vcpkg/export.cpp b/toolsrc/src/vcpkg/export.cpp index 88c1526c5cc6a3..f306bf4e69df17 100644 --- a/toolsrc/src/vcpkg/export.cpp +++ b/toolsrc/src/vcpkg/export.cpp @@ -400,8 +400,10 @@ namespace vcpkg::Export Files::Filesystem& fs = paths.get_filesystem(); const fs::path export_to_path = paths.root; const fs::path raw_exported_dir_path = export_to_path / export_id; + fs.remove_all(raw_exported_dir_path, VCPKG_LINE_INFO); + + // TODO: error handling std::error_code ec; - fs.remove_all(raw_exported_dir_path, ec); fs.create_directory(raw_exported_dir_path, ec); // execute the plan @@ -476,7 +478,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console if (!opts.raw) { - fs.remove_all(raw_exported_dir_path, ec); + fs.remove_all(raw_exported_dir_path, VCPKG_LINE_INFO); } } diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp index 425a4cdbc9e7bc..32af57b39df510 100644 --- a/toolsrc/src/vcpkg/install.cpp +++ b/toolsrc/src/vcpkg/install.cpp @@ -354,8 +354,7 @@ namespace vcpkg::Install { auto& fs = paths.get_filesystem(); const fs::path package_dir = paths.package_dir(action.spec); - std::error_code ec; - fs.remove_all(package_dir, ec); + fs.remove_all(package_dir, VCPKG_LINE_INFO); } if (action.build_options.clean_downloads == Build::CleanDownloads::YES) diff --git a/toolsrc/src/vcpkg/remove.cpp b/toolsrc/src/vcpkg/remove.cpp index a40b27bd745bf2..84ec6c981b9360 100644 --- a/toolsrc/src/vcpkg/remove.cpp +++ b/toolsrc/src/vcpkg/remove.cpp @@ -179,8 +179,7 @@ namespace vcpkg::Remove { System::printf("Purging package %s...\n", display_name); Files::Filesystem& fs = paths.get_filesystem(); - std::error_code ec; - fs.remove_all(paths.packages / action.spec.dir(), ec); + fs.remove_all(paths.packages / action.spec.dir(), VCPKG_LINE_INFO); System::printf(System::Color::success, "Purging package %s... done\n", display_name); } } diff --git a/toolsrc/vcpkg/vcpkg.vcxproj b/toolsrc/vcpkg/vcpkg.vcxproj index 8edea2244f2fa2..917cd3b9c86123 100644 --- a/toolsrc/vcpkg/vcpkg.vcxproj +++ b/toolsrc/vcpkg/vcpkg.vcxproj @@ -147,4 +147,4 @@ - \ No newline at end of file + diff --git a/toolsrc/vcpkglib/vcpkglib.vcxproj b/toolsrc/vcpkglib/vcpkglib.vcxproj index 2eff1abee20c17..d28bae8ec52119 100644 --- a/toolsrc/vcpkglib/vcpkglib.vcxproj +++ b/toolsrc/vcpkglib/vcpkglib.vcxproj @@ -274,4 +274,4 @@ - \ No newline at end of file + diff --git a/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj b/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj index e533d0e15d6062..14ec1e105dadb6 100644 --- a/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj +++ b/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj @@ -144,4 +144,4 @@ - \ No newline at end of file + diff --git a/toolsrc/vcpkgtest/vcpkgtest.vcxproj b/toolsrc/vcpkgtest/vcpkgtest.vcxproj index 929092dae356eb..530dfbc5dc35de 100644 --- a/toolsrc/vcpkgtest/vcpkgtest.vcxproj +++ b/toolsrc/vcpkgtest/vcpkgtest.vcxproj @@ -23,10 +23,12 @@ + + diff --git a/toolsrc/vcpkgtest/vcpkgtest.vcxproj.filters b/toolsrc/vcpkgtest/vcpkgtest.vcxproj.filters index fcf513fa94df52..d9808ca896a21d 100644 --- a/toolsrc/vcpkgtest/vcpkgtest.vcxproj.filters +++ b/toolsrc/vcpkgtest/vcpkgtest.vcxproj.filters @@ -48,6 +48,12 @@ Source Files + + Source Files + + + Source Files +