Fix builds on clang 16+ (#96)

awslabs · Sep 13, 2024 · ce04ab0 · ce04ab0
1 parent e5c9b16
commit ce04ab0
Show file tree

Hide file tree

Showing 5 changed files with 96 additions and 22 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,19 +6,19 @@ on:
       - 'main'
 
 env:
-  BUILDER_VERSION: v0.9.63
+  BUILDER_VERSION: v0.9.66
   BUILDER_SOURCE: releases
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
   PACKAGE_NAME: aws-checksums
-  LINUX_BASE_IMAGE: ubuntu-18-x64
+  LINUX_BASE_IMAGE: ubuntu-22-x64
   RUN: ${{ github.run_id }}-${{ github.run_number }}
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_REGION: us-east-1
 
 jobs:
   linux-compat:
-    runs-on: ubuntu-22.04 # latest
+    runs-on: ubuntu-24.04 # latest
     strategy:
       fail-fast: false
       matrix:
@@ -38,8 +38,9 @@ jobs:
         aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
         ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }}
 
-  linux-compiler-compat:
-    runs-on: ubuntu-22.04 # latest
+
+  linux-compiler-compat-old-compilers:
+    runs-on: ubuntu-24.04 # latest
     strategy:
       matrix:
         compiler:
@@ -49,11 +50,33 @@ jobs:
           - clang-9
           - clang-10
           - clang-11
+          - clang-12
           - gcc-4.8
           - gcc-5
           - gcc-6
           - gcc-7
           - gcc-8
+    steps:
+      # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
+      - name: Build ${{ env.PACKAGE_NAME }}
+        run: |
+          aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
+          ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-ubuntu-18-x64 build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }}
+
+  linux-compiler-compat:
+    runs-on: ubuntu-24.04 # latest
+    strategy:
+      matrix:
+        compiler:
+          - clang-13
+          - clang-14
+          - clang-15
+          - clang-16
+          - clang-17
+          - clang-18
+          - gcc-11
+          - gcc-12
+          - gcc-13
     steps:
       # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
       - name: Build ${{ env.PACKAGE_NAME }}
@@ -62,7 +85,7 @@ jobs:
           ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }}
 
   clang-sanitizers:
-    runs-on: ubuntu-22.04 # latest
+    runs-on: ubuntu-24.04 # latest
     strategy:
       matrix:
         sanitizers: [",thread", ",address,undefined"]
@@ -71,10 +94,10 @@ jobs:
       - name: Build ${{ env.PACKAGE_NAME }}
         run: |
           aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
-          ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=clang-11 --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}"
+          ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=clang-16 --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}"
 
   linux-shared-libs:
-    runs-on: ubuntu-22.04 # latest
+    runs-on: ubuntu-24.04 # latest
     steps:
       # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
       - name: Build ${{ env.PACKAGE_NAME }}
@@ -83,7 +106,7 @@ jobs:
           ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DBUILD_SHARED_LIBS=ON
 
   linux-no-cpu-extensions:
-    runs-on: ubuntu-22.04 # latest
+    runs-on: ubuntu-24.04 # latest
     steps:
       # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
       - name: Build ${{ env.PACKAGE_NAME }}
@@ -178,7 +201,7 @@ jobs:
 
   cross_compile:
     name: Cross Compile ${{matrix.arch}}
-    runs-on: ubuntu-22.04 # latest
+    runs-on: ubuntu-24.04 # latest
     strategy:
       matrix:
         arch: [linux-armv6, linux-armv7, linux-arm64, android-armv7]
@@ -193,10 +216,56 @@ jobs:
   # Test downstream repos.
   # This should not be required because we can run into a chicken and egg problem if there is a change that needs some fix in a downstream repo.
   downstream:
-    runs-on: ubuntu-22.04 # latest
+    runs-on: ubuntu-24.04 # latest
     steps:
       # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
       - name: Build ${{ env.PACKAGE_NAME }}
         run: |
           aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
           ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build downstream -p ${{ env.PACKAGE_NAME }}
+
+  freebsd:
+    runs-on: ubuntu-24.04  # latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build ${{ env.PACKAGE_NAME }} + consumers
+        id: test
+        uses: cross-platform-actions/action@v0.24.0
+        with:
+          operating_system: freebsd
+          architecture: x86-64
+          version: '14.0'
+          cpu_count: 4
+          shell: bash
+          run: |
+            sudo pkg install -y python3 net/py-urllib3
+            python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
+            chmod a+x builder
+            ./builder build -p ${{ env.PACKAGE_NAME }}
+
+  openbsd:
+    runs-on: ubuntu-24.04 # latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # OpenBSD only supports the two most recent releases
+        version: ['7.4', '7.5']
+    steps:
+      # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Build ${{ env.PACKAGE_NAME }} + consumers
+        uses: cross-platform-actions/action@v0.24.0
+        with:
+          operating_system: openbsd
+          architecture: x86-64
+          version: ${{ matrix.version }}
+          cpu_count: 4
+          shell: bash
+          environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION
+          run: |
+            sudo pkg_add awscli py3-pip py3-urllib3
+            python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')"
+            chmod a+x builder
+            ./builder build -p ${{ env.PACKAGE_NAME }}
diff --git a/bin/benchmark/main.c b/bin/benchmark/main.c
@@ -55,6 +55,10 @@ static void s_runcrc64(struct aws_byte_cursor checksum_this) {
     (void)crc;
 }
 
+#define KB_TO_BYTES(kb) ((kb) * 1024)
+#define MB_TO_BYTES(mb) ((mb) * 1024 * 1024)
+#define GB_TO_BYTES(gb) ((gb) * 1024 * 1024 * 1024ULL)
+
 int main(void) {
 
     fprintf(stdout, "hw features for this run:\n");
@@ -97,7 +101,8 @@ int main(void) {
 
             // get buffer sizes large enough that all the simd code paths get hit hard, but
             // also measure the smaller buffer paths since they often can't be optimized as thoroughly.
-            size_t buffer_sizes[] = {8, 16, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536};
+            size_t buffer_sizes[] = {8, 16, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384,
+                KB_TO_BYTES(32), KB_TO_BYTES(64), KB_TO_BYTES(256), MB_TO_BYTES(1), MB_TO_BYTES(10), MB_TO_BYTES(100), GB_TO_BYTES(1)};
             size_t buffer_sizes_len = AWS_ARRAY_SIZE(buffer_sizes);
 
             // warm it up to factor out the cpuid checks:
@@ -115,9 +120,10 @@ int main(void) {
                 aws_high_res_clock_get_ticks(&end_time);
                 fprintf(
                     stdout,
-                    "buffer size %zu (bytes), latency: %" PRIu64 " ns\n",
+                    "buffer size %zu (bytes), latency: %" PRIu64 " ns throughput: %f GiB/s\n",
                     buffer_sizes[k],
-                    end_time - start_time);
+                    end_time - start_time,
+                    (buffer_sizes[k] * 1000000000.0 /* ns -> sec factor */ / GB_TO_BYTES(1)) / (end_time - start_time));
                 aws_byte_buf_clean_up(&x_bytes);
             }
             fprintf(stdout, "\n");

diff --git a/source/intel/intrin/crc32c_sse42_avx512.c b/source/intel/intrin/crc32c_sse42_avx512.c
@@ -17,8 +17,7 @@
 
 #    include <wmmintrin.h>
 
-AWS_ALIGNED_TYPEDEF(const uint64_t, zalign_8, 64);
-AWS_ALIGNED_TYPEDEF(const uint64_t, zalign_2, 16);
+AWS_ALIGNED_TYPEDEF(const uint64_t, aligned_512_u64[8], 64);
 
 // This macro uses casting to ensure the compiler actually uses the unaligned load instructions
 #    define load_zmm(ptr) _mm512_loadu_si512((const uint8_t *)(const void *)(ptr))
@@ -48,13 +47,13 @@ static uint32_t s_checksums_crc32c_avx512_impl(const uint8_t *input, int length,
      * k6 = ( x ^ ( 128 - 32 ) mod P(x) << 32 )' << 1
      */
 
-    static zalign_8 k1k2[8] = {
+    static aligned_512_u64 k1k2 = {
         0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86};
-    static zalign_8 k3k4[8] = {
+    static aligned_512_u64 k3k4 = {
         0x740eef02, 0x9e4addf8, 0x740eef02, 0x9e4addf8, 0x740eef02, 0x9e4addf8, 0x740eef02, 0x9e4addf8};
-    static zalign_8 k9k10[8] = {
+    static aligned_512_u64 k9k10 = {
         0x6992cea2, 0x0d3b6092, 0x6992cea2, 0x0d3b6092, 0x6992cea2, 0x0d3b6092, 0x6992cea2, 0x0d3b6092};
-    static zalign_8 k1k4[8] = {
+    static aligned_512_u64 k1k4 = {
         0x1c291d04, 0xddc0152b, 0x3da6d0cb, 0xba4fc28e, 0xf20c0dfe, 0x493c7d27, 0x00000000, 0x00000000};
 
     __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8;

diff --git a/tests/crc64_test.c b/tests/crc64_test.c
@@ -129,7 +129,7 @@ AWS_TEST_CASE(test_crc64nvme, s_test_crc64nvme)
 
 static int s_test_large_buffer_crc64(struct aws_allocator *allocator, void *ctx) {
     (void)ctx;
-#if SIZE_BITS == 32
+#if SIZE_BITS == 32 || defined(__OpenBSD__) /* openbsd fails to allocate big buffer */
     (void)allocator;
     return AWS_OP_SKIP;
 #else

diff --git a/tests/crc_test.c b/tests/crc_test.c
@@ -186,7 +186,7 @@ AWS_TEST_CASE(test_crc32, s_test_crc32)
 
 static int s_test_large_buffer_crc32(struct aws_allocator *allocator, void *ctx) {
     (void)ctx;
-#if SIZE_BITS == 32
+#if SIZE_BITS == 32 || defined(__OpenBSD__) /* openbsd fails to allocate big buffer */
     (void)allocator;
     return AWS_OP_SKIP;
 #else