chore: change default compiler to clang

.github/workflows/build_linux_wheel/action.yml

@@ -25,8 +25,8 @@ runs:
       shell: bash
       run: |
         echo "ARM BUILD: ${{ inputs.arm-build }}"
-    - name: Build x86_64 Manylinux wheel
-      if: ${{ inputs.arm-build == 'false' }}
+    - name: Build x86_64 Manylinux2014 wheel
+      if: ${{ inputs.arm-build == 'false' && inputs.manylinux == '2_17' }}
       uses: PyO3/maturin-action@v1
       with:
         command: build
@@ -40,6 +40,24 @@ runs:
             && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
             && unzip /tmp/protoc.zip -d /usr/local \
             && rm /tmp/protoc.zip
+    - name: Build x86_64 Manylinux {manylinux} wheel
+      if: ${{ inputs.arm-build == 'false' && inputs.manylinux != '2_17' }}
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        working-directory: python
+        target: x86_64-unknown-linux-gnu
+        manylinux: ${{ inputs.manylinux }}
+        docker-options: |
+          -e CC=clang
+          -e CXX=clang++
+        args: ${{ inputs.args }}
+        before-script-linux: |
+          set -e
+          yum install -y openssl-devel clang \
+            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
+            && unzip /tmp/protoc.zip -d /usr/local \
+            && rm /tmp/protoc.zip
     - name: Build Arm Manylinux Wheel
       if: ${{ inputs.arm-build == 'true' }}
       uses: PyO3/maturin-action@v1

.github/workflows/cargo-publish.yml

@@ -19,12 +19,12 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     env:
       # Need up-to-date compilers for kernels
-      CC: gcc-12
-      CXX: g++-12
+      CC: clang-18
+      CXX: clang-18
     defaults:
       run:
         working-directory: .

.github/workflows/python.yml

@@ -31,131 +31,131 @@ env:
 jobs:
   lint:
     timeout-minutes: 45
-    runs-on: "ubuntu-22.04"
+    runs-on: "ubuntu-24.04"
     defaults:
       run:
         shell: bash
         working-directory: python
     env:
       # Need up-to-date compilers for kernels
-      CC: gcc-12
-      CXX: g++-12
+      CC: clang-18
+      CXX: clang-18
     steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.11  # Ray does not support 3.12 yet.
-    - uses: Swatinem/rust-cache@v2
-      with:
-        workspaces: python
-    - name: Install linting tools
-      run: |
-        pip install ruff==0.4.1 maturin tensorflow tqdm ray[data]
-        pip install torch --index-url https://download.pytorch.org/whl/cpu
-    - name: Lint Python
-      run: |
-        ruff format --check python
-        ruff check python
-    - name: Install dependencies
-      run: |
-        sudo apt update
-        sudo apt install -y protobuf-compiler libssl-dev
-    - name: Lint Rust
-      run: |
-        cargo fmt --all -- --check
-        cargo clippy --all-features --tests -- -D warnings
-    - name: Build
-      run: |
-        python -m venv venv
-        source venv/bin/activate
-        pip install torch tqdm --index-url https://download.pytorch.org/whl/cpu
-        pip install maturin
-        maturin develop --extras tests,ray
-    - name: Run doctest
-      run: |
-        source venv/bin/activate
-        pytest --doctest-modules python/lance
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11 # Ray does not support 3.12 yet.
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - name: Install linting tools
+        run: |
+          pip install ruff==0.4.1 maturin tensorflow tqdm ray[data]
+          pip install torch --index-url https://download.pytorch.org/whl/cpu
+      - name: Lint Python
+        run: |
+          ruff format --check python
+          ruff check python
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Lint Rust
+        run: |
+          cargo fmt --all -- --check
+          cargo clippy --all-features --tests -- -D warnings
+      - name: Build
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          pip install torch tqdm --index-url https://download.pytorch.org/whl/cpu
+          pip install maturin
+          maturin develop --extras tests,ray
+      - name: Run doctest
+        run: |
+          source venv/bin/activate
+          pytest --doctest-modules python/lance
   linux:
     timeout-minutes: 45
     strategy:
       matrix:
-        python-minor-version: [ "9", "12" ]
+        python-minor-version: ["9", "12"]
     name: "Python Linux 3.${{ matrix.python-minor-version }} x86_64"
-    runs-on: "ubuntu-22.04"
+    runs-on: "ubuntu-24.04"
     defaults:
       run:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.${{ matrix.python-minor-version }}
-    - uses: Swatinem/rust-cache@v2
-      with:
-        workspaces: python
-        prefix-key: "manylinux2014"  # use this to flush the cache
-    - uses: ./.github/workflows/build_linux_wheel
-    - uses: ./.github/workflows/run_tests
-    - name: Generate forward compatibility files
-      run: python python/tests/forward_compat/datagen.py
-    - name: Install old wheel
-      run: |
-        python -m venv venv
-        source venv/bin/activate
-        pip install pytest pylance==0.16.0
-    - name: Run forward compatibility tests
-      run: |
-        source venv/bin/activate
-        pytest python/tests/forward_compat --run-forward
-    # Make sure wheels are not included in the Rust cache
-    - name: Delete wheels
-      run: sudo rm -rf target/wheels
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.${{ matrix.python-minor-version }}
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+          prefix-key: "manylinux2014" # use this to flush the cache
+      - uses: ./.github/workflows/build_linux_wheel
+      - uses: ./.github/workflows/run_tests
+      - name: Generate forward compatibility files
+        run: python python/tests/forward_compat/datagen.py
+      - name: Install old wheel
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          pip install pytest pylance==0.16.0
+      - name: Run forward compatibility tests
+        run: |
+          source venv/bin/activate
+          pytest python/tests/forward_compat --run-forward
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: sudo rm -rf target/wheels
   linux-arm:
     timeout-minutes: 45
     #runs-on: warp-ubuntu-latest-arm64-4x
     runs-on: buildjet-4vcpu-ubuntu-2204-arm
     name: Python Linux 3.${{ matrix.python-minor-version }} ARM
     strategy:
       matrix:
-        python-minor-version: [ "12" ]
+        python-minor-version: ["12"]
     defaults:
       run:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      run: |
-        sudo apt update
-        sudo apt install -y -qq wget software-properties-common
-        sudo add-apt-repository ppa:deadsnakes/ppa
-        sudo apt install -y python3.${{ matrix.python-minor-version }}
-        sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.${{ matrix.python-minor-version }} 1
-        sudo apt install -y python3-pip python3.${{ matrix.python-minor-version }}-distutils
-    - uses: Swatinem/rust-cache@v2
-      with:
-        workspaces: python
-        prefix-key: "manylinux2014"  # use this to flush the cache
-    - uses: ./.github/workflows/build_linux_wheel
-      with:
-        arm-build: "true"
-        manylinux: "2_24"
-    - uses: ./.github/workflows/run_tests
-    # Make sure wheels are not included in the Rust cache
-    - name: Delete wheels
-      run: sudo rm -rf target/wheels
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        run: |
+          sudo apt update
+          sudo apt install -y -qq wget software-properties-common
+          sudo add-apt-repository ppa:deadsnakes/ppa
+          sudo apt install -y python3.${{ matrix.python-minor-version }}
+          sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.${{ matrix.python-minor-version }} 1
+          sudo apt install -y python3-pip python3.${{ matrix.python-minor-version }}-distutils
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+          prefix-key: "manylinux2014" # use this to flush the cache
+      - uses: ./.github/workflows/build_linux_wheel
+        with:
+          arm-build: "true"
+          manylinux: "2_24"
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: sudo rm -rf target/wheels
   mac:
     timeout-minutes: 45
     name: Python macOS 3.12 ARM
@@ -165,22 +165,22 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.12"
-    - uses: Swatinem/rust-cache@v2
-      with:
-        workspaces: python
-    - uses: ./.github/workflows/build_mac_wheel
-    - uses: ./.github/workflows/run_tests
-    # Make sure wheels are not included in the Rust cache
-    - name: Delete wheels
-      run: rm -rf target/wheels
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_mac_wheel
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
   windows:
     runs-on: windows-latest
     timeout-minutes: 90
@@ -222,24 +222,24 @@ jobs:
           AWS_ACCESS_KEY_ID: ACCESSKEY
           AWS_SECRET_ACCESS_KEY: SECRETKEY
     steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.11" # TODO: upgrade when ray supports 3.12
-    - uses: Swatinem/rust-cache@v2
-      with:
-        workspaces: python
-        prefix-key: "manylinux2014"  # use this to flush the cache
-    - uses: ./.github/workflows/build_linux_wheel
-    - name: Install dependencies
-      run: |
-        pip install ray[data]
-        pip install torch --index-url https://download.pytorch.org/whl/cpu
-    - uses: ./.github/workflows/run_integtests
-    # Make sure wheels are not included in the Rust cache
-    - name: Delete wheels
-      run: sudo rm -rf target/wheels
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11" # TODO: upgrade when ray supports 3.12
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+          prefix-key: "manylinux2014" # use this to flush the cache
+      - uses: ./.github/workflows/build_linux_wheel
+      - name: Install dependencies
+        run: |
+          pip install ray[data]
+          pip install torch --index-url https://download.pytorch.org/whl/cpu
+      - uses: ./.github/workflows/run_integtests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: sudo rm -rf target/wheels

.github/workflows/rust.yml

@@ -38,15 +38,15 @@ jobs:
           - nightly
     env:
       # Need up-to-date compilers for kernels
-      CC: gcc-12
-      CXX: g++-12
+      CC: clang-14
+      CXX: clang-14
     steps:
       - uses: actions/checkout@v4
       # pin the toolchain version to avoid surprises
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         if: ${{ matrix.toolchain == 'stable' }}
         with:
-          toolchain: "1.78.0"
+          toolchain: "1.81.0"
           cache-workspaces: "src/rust"
           # Disable full debug symbol generation to speed up CI build and keep memory down
           # "1" means line tables only, which is useful for panic tracebacks.

rust/lance-linalg/build.rs

@@ -76,7 +76,8 @@ fn main() -> Result<(), String> {
 fn build_f16_with_flags(suffix: &str, flags: &[&str]) -> Result<(), cc::Error> {
     let mut builder = cc::Build::new();
     builder
-        // TODO: why specify the compiler?
+        // We use clang #pragma to yields better vectorization
+        // See https://github.com/lancedb/lance/pull/2885
         // .compiler("clang")
         .std("c17")
         .file("src/simd/f16.c")