Add interfaces to popular Rust hash libraries

.appveyor.yml

@@ -48,6 +48,11 @@ init:
 #- if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2013" (call "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" amd64)
 # call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
 
+install:
+  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
+  - rustup-init.exe -y --default-toolchain stable --profile minimal
+  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
+
 build_script:
   - git submodule update --init
   #- if "%MINGW%"=="1" call build-msys2.bat
@@ -61,3 +66,10 @@ build_script:
 
 test_script:
   - Release\SMHasher --test=VerifyAll,Sanity,Speed,Cyclic,Zeroes,Seed
+
+# TODO remove debugging
+on_failure:
+  - rustc -vV
+  - ls rust-hashes/target/
+  - ls rust-hashes/target/release
+  - dumpbin /symbols /exports rust-hashes/target/release/rust_hashes.lib

.cirrus.yml

@@ -8,6 +8,9 @@ task:
   install_script:
     - pkg install -y cmake git
     - git submodule update --init
+    - >
+      fetch -q --no-tlsv1 -o- https://sh.rustup.rs |
+      sh -s -- -y --default-toolchain stable --profile minimal
   build_script:
     - cmake .
     - make VERBOSE=1

.github/workflows/ci.yml

@@ -21,6 +21,7 @@ jobs:
       with:
         fetch-depth: 1
         submodules: recursive
+    - uses: dtolnay/rust-toolchain@stable
     - run: cmake ${{ matrix.CMAKE_ARGS }} .
     - run: make -j4 VERBOSE=1
     - if: ${{ ! contains(matrix.CMAKE_ARGS, 'CMAKE_BUILD_TYPE') }}
@@ -34,10 +35,13 @@ jobs:
       with:
         fetch-depth: 1
         submodules: recursive
+    - uses: dtolnay/rust-toolchain@stable
+      with:
+        targets: aarch64-unknown-linux-gnu
     - run: sudo gem install apt-spy2 && sudo apt-spy2 fix
     - run: sudo apt-get update -y
     - run: sudo apt-get install -o Acquire::Retries=5 -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
-    - run: cmake -DCMAKE_TOOLCHAIN_FILE=CMakeARM64Cross.txt
+    - run: cmake -DCMAKE_TOOLCHAIN_FILE=CMakeARM64Cross.txt -DRUST_TARGET=aarch64-unknown-linux-gnu
     - run: make -j4 VERBOSE=1
   armv7:
     runs-on: ubuntu-latest
@@ -46,10 +50,13 @@ jobs:
       with:
         fetch-depth: 1
         submodules: recursive
+    - uses: dtolnay/rust-toolchain@stable
+      with:
+        targets: armv7-unknown-linux-gnueabihf
     - run: sudo gem install apt-spy2 && sudo apt-spy2 fix
     - run: sudo apt-get update -y
     - run: sudo apt-get install -o Acquire::Retries=5 -y gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf
-    - run: cmake -DCMAKE_TOOLCHAIN_FILE=CMakeARMCross.txt
+    - run: cmake -DCMAKE_TOOLCHAIN_FILE=CMakeARMCross.txt -DRUST_TARGET=armv7-unknown-linux-gnueabihf
     - run: make -j4 VERBOSE=1
   macOS:
     name: macOS
@@ -61,6 +68,7 @@ jobs:
       with:
         fetch-depth: 1
         submodules: true
+    - uses: dtolnay/rust-toolchain@stable
     - run: CXXFLAGS='-std=c++17' cmake .
     - run: make -j4 VERBOSE=1
     - run: make -j4 test ARGS=-V
@@ -71,6 +79,7 @@ jobs:
       with:
         fetch-depth: 1
         submodules: recursive
+    - uses: dtolnay/rust-toolchain@stable
     - run: cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_BUILD_TYPE=Release .
     - run: cmake --build . --config Release --verbose
     - run: Release\SMHasher.exe --test=VerifyAll,Sanity,Speed,Cyclic,Zeroes,Seed
@@ -96,6 +105,7 @@ jobs:
           cmake
           base-devel
           mingw-w64-x86_64-toolchain
+    - uses: dtolnay/rust-toolchain@stable
     - run: reg add "HKLM\Software\Microsoft\Windows\Windows Error Reporting" /f /v DontShowUI /d 1
     - uses: actions/checkout@v3
       with:
@@ -109,3 +119,27 @@ jobs:
     - shell: msys2 {0}
       run: make -j4 test ARGS=-V
       continue-on-error: true
+  rust_check:
+    name: check rust bindings
+    runs-on: ubuntu-latest
+    env:
+      RUST_GENERATE_BINDINGS: 1
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 1
+        submodules: recursive
+    - uses: dtolnay/rust-toolchain@nightly
+    - uses: Swatinem/rust-cache@v2
+    - run: |
+        cd rust-hashes
+        mv rust_hashes.h committed.h
+        cargo build # regenerate the bindings
+        diff -up committed.h rust_hashes.h
+        if [ $? -eq 0 ]; then
+          echo "Files match"
+        else
+          echo "Binding files changed or do not match"
+          echo "regenerate them with `RUST_GENERATE_BINDINGS=1 cargo +nightly build`"
+          exit 1
+        fi

.gitignore

@@ -37,4 +37,8 @@ prof.h
 /blake3/test.py
 /blake3/test_vectors.json
 
+# rust outputs
+target/
+rust-hashes/**.out
+
 *.swp

CMakeLists.txt

@@ -630,6 +630,77 @@ endif()
 # "${CMAKE_CURRENT_SOURCE_DIR}/fanom_hash.h") set(CMAKE_CXX_FLAGS
 # "${CMAKE_CXX_FLAGS} -DHAVE_FANOM_HASH") ENDIF()
 
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+  set(RUST_NOT_DEBUG false)
+else()
+  set(RUST_NOT_DEBUG true)
+endif()
+
+set(RUST_ENABLED true CACHE BOOL "enable rust support (requires cargo)")
+set(RUST_TARGET "" CACHE STRING "set target for cross compiling (blank for current)")
+set(RUST_SIMD ${RUST_NOT_DEBUG} CACHE BOOL
+    "enable native features (non-portable SIMD instructions)")
+
+if(RUST_ENABLED)
+  message(STATUS "Rust enabled")
+  set(RUST_TOOLCHAIN "") # use the default
+  set(CARGO_PATH "--manifest-path=${CMAKE_CURRENT_SOURCE_DIR}/rust-hashes/Cargo.toml")
+  set(RUST_LIB_PATH "${CMAKE_CURRENT_SOURCE_DIR}/rust-hashes/target")
+
+  if(NOT RUST_TARGET STREQUAL "") # set target arch if needed
+    message(STATUS "Cross compiling Rust to ${RUST_TARGET}")
+    set(CARGO_TARGET "--target=${RUST_TARGET}")
+    set(RUST_LIB_PATH "${RUST_LIB_PATH}/${RUST_TARGET}")
+  endif()
+
+  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(CARGO_PROFILE "--release")
+    set(RUST_LIB_PATH "${RUST_LIB_PATH}/release")
+  else()
+    set(RUST_LIB_PATH "${RUST_LIB_PATH}/debug")
+  endif()
+
+  if(CMAKE_BUILD_TYPE STREQUAL "asan")
+    set(RUSTFLAGS "${RUSTFLAGS} -Zsanitizer=address")
+    set(RUST_TOOLCHAIN "+nightly") # sanitizers are unstable and require nightly
+  endif()
+
+  if(RUST_CPU_NATIVE) # enable all CPU features
+    set(RUSTFLAGS "${RUSTFLAGS} -Ctarget-cpu=native")
+  endif()
+
+  if(CMAKE_POSITION_INDEPENDENT_CODE)
+    set(RUSTFLAGS "${RUSTFLAGS} -Crelocation-model=pic")
+  endif()
+
+  if(WIN32)
+    set(RUST_LIB_PATH "${RUST_LIB_PATH}/rust_hashes.lib")
+    # give us a reminder of what needs to be linked
+    set(RUSTFLAGS "${RUSTFLAGS} --print=native-static-libs")
+
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "i?[356x]86")
+      # these are not included by default on Windows
+      find_library(BCRYPT bcrypt)   
+      find_library(NTDLL ntdll)
+    endif()
+  else()
+    set(RUST_LIB_PATH "${RUST_LIB_PATH}/librust_hashes.a")
+  endif()
+
+  # enable the faster dependency protocol for < 1.70 (no runtime effect)
+  set(ENV{CARGO_REGISTRIES_CRATES_IO_PROTOCOL} "sparse")
+  set(ENV{RUSTFLAGS} "${RUSTFLAGS}")
+
+  message(STATUS "RUSTFLAGS: $ENV{RUSTFLAGS}")
+  add_custom_target(
+    librust_hashes
+    COMMAND cargo ${RUST_TOOLCHAIN} build --quiet ${CARGO_PATH}
+            ${CARGO_TARGET} ${CARGO_PROFILE}
+  )
+
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DRUST_ENABLED")
+endif()
+
 if((CMAKE_MAJOR_VERSION EQUAL 3 AND CMAKE_MINOR_VERSION GREATER_EQUAL 9)
    AND (CMAKE_BUILD_TYPE STREQUAL "Release"))
   cmake_policy(SET CMP0069 NEW)
@@ -735,6 +806,11 @@ target_link_libraries(SMHasher SMHasherSupport
                       ${HIGHWAY_LIB} ${BLAKE3_LIB} ${AHASH_C_LIB}
                       ${CMAKE_THREAD_LIBS_INIT})
 
+if (RUST_ENABLED)
+  add_dependencies(SMHasher librust_hashes)
+  target_link_libraries(SMHasher ${RUST_LIB_PATH} ${BCRYPT} ${NTDLL})
+endif()
+
 SET(exectargets ${exectargets} SMHasher)
 # add_executable( bittest bittest.cpp )
 #

main.cpp

@@ -6,6 +6,9 @@
 #include "AvalancheTest.h"
 #include "DifferentialTest.h"
 #include "HashMapTest.h"
+#ifdef RUST_ENABLED
+#include "rust-hashes/rust_hashes.h"
+#endif
 
 #if NCPU > 1 // disable with -DNCPU=0 or 1
 #include <thread>
@@ -757,6 +760,60 @@ HashInfo g_hashes[] =
 #endif
 { komihash_test,        64, 0x8157FF6D, "komihash",    "komihash 5.7", GOOD, {} },
 { polymur_test,         64, 0x4F894810, "polymur",     "github.com/orlp/polymur-hash v1", GOOD, {} },
+#ifdef RUST_ENABLED
+{ adler_rs,             32, 0x00000000, "Adler_rs",           "Adler-32 checksum implementation (used by `zlib`, crate `adler`)", GOOD, {} },
+{ ahash_rs,             64, 0x00000000, "aHash_rs",           "aHash keyed hash function (crate `ahash`)", GOOD, {} },
+{ ascon_rs,            256, 0x00000000, "Ascon_rs",           "Ascon hash (crate `ascon-hash`)", GOOD, {} },
+{ ascona_rs,           256, 0x00000000, "AsconA_rs",          "AsconA hash (crate `ascon-hash`)", GOOD, {} },
+{ blake2b512_rs,       512, 0x00000000, "BLAKE2b-512_rs",     "BLAKE2b-512 (crate `blake2`)", GOOD , {} },
+{ blake2s256_rs,       256, 0x00000000, "BLAKE2s-256_rs",     "BLAKE2s-256 (crate `blake2`)", GOOD, {} },
+{ blake3_rs,           256, 0x00000000, "BLAKE3_rs",          "BLAKE3 (crate `blake3`)", GOOD, {} },
+{ crc32fast_rs,         32, 0x00000000, "CRC32_rs",           "CRC32 (crate `crc32fast`)", GOOD, {} },
+{ fnvhash_rs,           64, 0x00000000, "FNV_rs",             "Fowler-Noll-Vo hash function (crate `fnv`)", GOOD, {} },
+{ fxhash32_rs,          32, 0x00000000, "FxHash32",           "FxHash (FireFox hash) with 32 bits (crate `fxhash`)", GOOD, {} },
+{ fxhash64_rs,          64, 0x00000000, "FxHash64",           "FxHash (FireFox hash) with 64 bits (crate `fxhash`)", GOOD, {} },
+{ hash_hasher_rs,       64, 0x00000000, "hash_hasher_rs",     "A hasher designed to work on already hashed or hash-like data (crate `hash_hasher`)", GOOD, {} },
+{ highway_rs,           64, 0x00000000, "Highway_rs",         "An implementation of Google's HighwayHash (crate `highway`)", GOOD, {} },
+{ md5_rs,              128, 0x00000000, "MD5_rs",             "MD5 algorithm (crate `md-5`)", GOOD, {} },
+{ ripemd128_rs,        128, 0x00000000, "RIPEMD-128_rs",      "RIPEMD-128 algorithm (crate `ripemd`)", GOOD, {} },
+{ ripemd160_rs,        160, 0x00000000, "RIPEMD-160_rs",      "RIPEMD-160 algorithm (crate `ripemd`)", GOOD, {} },
+{ ripemd256_rs,        256, 0x00000000, "RIPEMD-256_rs",      "RIPEMD-256 algorithm (crate `ripemd`)", GOOD, {} },
+{ ripemd320_rs,        320, 0x00000000, "RIPEMD-320_rs",      "RIPEMD-320 algorithm (crate `ripemd`)", GOOD, {} },
+{ sha1_rs,             160, 0x00000000, "SHA-1_rs",           "SHA-1 algorithm (crate `sha1`)", GOOD, {} },
+{ sha224_rs,           224, 0x00000000, "SHA-224_rs",         "SHA-224 algorithm (crate `sha2`)", GOOD, {} },
+{ sha256_rs,           256, 0x00000000, "SHA-256_rs",         "SHA-256 algorithm (crate `sha2`)", GOOD, {} },
+{ sha384_rs,           384, 0x00000000, "SHA-384_rs",         "SHA-384 algorithm (crate `sha2`)", GOOD, {} },
+{ sha512_rs,           512, 0x00000000, "SHA-512_rs",         "SHA-512 algorithm (crate `sha2`)", GOOD, {} },
+{ sha512_224_rs,       224, 0x00000000, "SHA-512/224_rs",     "SHA-512 algorithm with 224-bit output (crate `sha2`)", GOOD, {} },
+{ sha512_256_rs,       256, 0x00000000, "SHA-512/256_rs",     "SHA-512 algorithm with 256-bit output (crate `sha2`)", GOOD, {} },
+{ cshake128_rs,        128, 0x00000000, "CSHAKE128_rs",       "CSHAKE128 algorithm with variable-length output (128 bits for test, crate `sha3`)", GOOD, {} },
+{ cshake256_rs,        256, 0x00000000, "CSHAKE256_rs",       "CSHAKE256 algorithm with variable-length output (256 bits for test, crate `sha3`)", GOOD, {} },
+{ keccak224_rs,        224, 0x00000000, "Keccak-224_rs",      "Keccak-224 algorithm (crate `sha3`)", GOOD, {} },
+{ keccak256_rs,        256, 0x00000000, "Keccak-256_rs",      "Keccak-256 algorithm (crate `sha3`)", GOOD, {} },
+{ keccak256full_rs,   1600, 0x00000000, "Keccak-256_CryptoNight_rs","Keccak-256 hash algorithm, SHA-3 CryptoNight with 1600-bit output (crate `sha3`)", GOOD, {} },
+{ keccak384_rs,        384, 0x00000000, "Keccak-384_rs",      "Keccak-384 algorithm (crate `sha3`)", GOOD, {} },
+{ keccak512_rs,        512, 0x00000000, "Keccak-512_rs",      "Keccak-512 algorithm (crate `sha3`)", GOOD, {} },
+{ sha3_224_rs,         224, 0x00000000, "SHA-3-224_rs",       "SHA-3 algorithm with 224-bit output (crate `sha3`)", GOOD, {} },
+{ sha3_256_rs,         256, 0x00000000, "SHA-3-256_rs",       "SHA-3 algorithm with 256-bit output (crate `sha3`)", GOOD, {} },
+{ sha3_384_rs,         384, 0x00000000, "SHA-3-384_rs",       "SHA-3 algorithm with 384-bit output (crate `sha3`)", GOOD, {} },
+{ sha3_512_rs,         512, 0x00000000, "SHA-3-512_rs",       "SHA-3 algorithm with 512-bit output (crate `sha3`)", GOOD, {} },
+{ shake128_rs,         123, 0x00000000, "SHAKE128_rs",        "SHAKE128 algorithm with variable-length output (128 bits for test, crate `sha3`)", GOOD, {} },
+{ shake256_rs,         256, 0x00000000, "SHAKE256_rs",        "SHAKE256 algorithm with variable-length output (256 bits for test, crate `sha3`)", GOOD, {} },
+{ turboshake128_1f_rs, 128, 0x00000000, "TurboSHAKE128_rs",   "turboshake128 algorithm with variable-length output (128 bits for test, domain separation 0x1f, crate `sha3`)", GOOD, {} },
+{ turboshake256_1f_rs, 256, 0x00000000, "TurboSHAKE256_rs",   "turboshake256 algorithm with variable-length output (256 bits for test, domain separation 0x1f, crate `sha3`)", GOOD, {} },
+{ twox_xxhash32_rs,     32, 0x00000000, "xxHash32_twox_rs",   "32-bit xxHash implementation (crate `twox_hash`)", GOOD, {} },
+{ twox_xxhash64_rs,     64, 0x00000000, "xxHash64_twox_rs",   "64-bit xxHash implementation (crate `twox_hash`)", GOOD, {} },
+{ siphash13_rs,         64, 0x00000000, "SipHash_1-3_rs",     "SipHash with 1 round and 3 finalization rounds (crate `siphasher`)", GOOD, {} },
+{ siphash24_rs,         64, 0x00000000, "SipHash_2-4_rs",     "SipHash with 2 rounds and 4 finalization rounds (crate `siphasher`)", GOOD, {} },
+{ siphash128_13_rs,     64, 0x00000000, "SipHash128_1-3_rs",  "SipHash128 with 1 round and 3 finalization rounds (crate `siphasher`)", GOOD, {} },
+{ siphash128_24_rs,     64, 0x00000000, "SipHash128_2-4_rs",  "SipHash128 with 2 rounds and 4 finalization rounds (crate `siphasher`)", GOOD, {} },
+{ whirlpool_rs,        512, 0x00000000, "Whirlpool_rs",       "Whirlpool algorithm (crate `whirlpool`)", GOOD, {} },
+{ wyhash_rs,            64, 0x00000000, "WyHash_rs",          "WyHash algorithm (crate `wyhash`)", GOOD, {} },
+{ xxhash3_rs,           64, 0x00000000, "xxHash3_rs",         "64-bit xxHash implementation (crate `xxhash-rust`)", GOOD, {} },
+{ xxhash128_rs,        128, 0x00000000, "xxHash128_rs",       "128-bit xxHash implementation (crate `xxhash-rust`)", GOOD, {} },
+{ xxhash32_rs,          32, 0x00000000, "xxHash32_rs",        "32-bit xxHash implementation (crate `xxhash-rust`)", GOOD, {} },
+{ xxhash64_rs,          64, 0x00000000, "xxHash64_rs",        "64-bit xxHash implementation (crate `xxhash-rust`)", GOOD, {} },
+#endif
 };
 
 HashInfo * findHash ( const char * name )