[pull] main from github:main

.bazelrc

@@ -1,9 +1,30 @@
 common --enable_platform_specific_config
+common --enable_bzlmod
+# because we use --override_module with `%workspace%`, the lock file is not stable
+common --lockfile_mode=off
+
+# when building from this repository in isolation, the internal repository will not be found at ..
+# where `MODULE.bazel` looks for it. The following will get us past the module loading phase, so
+# that we can build things that do not rely on that
+common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
 
 build --repo_env=CC=clang --repo_env=CXX=clang++
 
-build:linux --cxxopt=-std=c++20
-build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
-build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
+# we use transitions that break builds of `...`, so for `test` to work with that we need the following
+test --build_tests_only
+
+# this requires developer mode, but is required to have pack installer functioning
+startup --windows_enable_symlinks
+common --enable_runfiles
+
+# with the above, we can avoid building python zips which is the default on windows as that's expensive
+build --nobuild_python_zip
+
+common --registry=file:///%workspace%/misc/bazel/registry
+common --registry=https://bcr.bazel.build
+
+common --@rules_dotnet//dotnet/settings:strict_deps=false
+common --experimental_isolated_extension_usages
+common --incompatible_use_plus_in_repo_names
 
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -0,0 +1,11 @@
+# this file should contain bazel settings required to build things from `semmle-code`
+
+common --registry=file:///%workspace%/ql/misc/bazel/registry
+common --registry=https://bcr.bazel.build
+
+# See bazelbuild/rules_dotnet#413: strict_deps in C# also appliy to 3rd-party deps, and when we pull
+# in (for example) the xunit package, there's no code in this at all, it just depends transitively on
+# its implementation packages without providing any code itself.
+# We either can depend on internal implementation details, or turn of strict deps.
+common --@rules_dotnet//dotnet/settings:strict_deps=false
+common --experimental_isolated_extension_usages

.bazelversion

@@ -1 +1 @@
-6.3.1
+5f5d70b6c4d2fb1a889479569107f1692239e8a7

.clang-format

@@ -0,0 +1 @@
+DisableFormat: true

.devcontainer/swift/root.sh

@@ -3,14 +3,26 @@ set -xe
 BAZELISK_VERSION=v1.12.0
 BAZELISK_DOWNLOAD_SHA=6b0bcb2ea15bca16fffabe6fda75803440375354c085480fe361d2cbf32501db
 
+# install git lfs apt source
+curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
+
+# install gh apt source
+(type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \
+&& sudo mkdir -p -m 755 /etc/apt/keyrings \
+&& wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
+&& sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
+&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+
 apt-get update
 export DEBIAN_FRONTEND=noninteractive
 apt-get -y install --no-install-recommends \
     zlib1g-dev \
     uuid-dev \
     python3-distutils \
     python3-pip \
-    bash-completion
+    bash-completion \
+    git-lfs \
+    gh
 
 # Install Bazel
 curl -fSsL -o /usr/local/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64

.devcontainer/swift/user.sh

@@ -1,5 +1,7 @@
 set -xe
 
+git lfs install
+
 # add the workspace to the codeql search path
 mkdir -p /home/vscode/.config/codeql
 echo "--search-path /workspaces/codeql" > /home/vscode/.config/codeql/config

.gitattributes

@@ -50,24 +50,40 @@
 *.dll -text
 *.pdb -text
 
-java/ql/test/stubs/**/*.java linguist-generated=true
-java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+/java/ql/test/stubs/**/*.java linguist-generated=true
+/java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text
 
 # Force git not to modify line endings for go or html files under the go/ql directory
-go/ql/**/*.go -text
-go/ql/**/*.html -text
+/go/ql/**/*.go -text
+/go/ql/**/*.html -text
 # Force git not to modify line endings for go dbschemes
-go/*.dbscheme -text
+/go/*.dbscheme -text
 # Preserve unusual line ending from codeql-go merge
-go/extractor/opencsv/CSVReader.java -text
+/go/extractor/opencsv/CSVReader.java -text
 
 # For some languages, upgrade script testing references really old dbscheme
 # files from legacy upgrades that have CRLF line endings. Since upgrade
 # resolution relies on object hashes, we must suppress line ending conversion
 # for those testing dbscheme files.
-*/ql/lib/upgrades/initial/*.dbscheme -text
+/*/ql/lib/upgrades/initial/*.dbscheme -text
 
-# Generated test files - these are synced from the standard JavaScript libraries using
-# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge
+# Auto-generated modeling for Python
+/python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
+
+# auto-generated bazel lock file
+/ruby/extractor/cargo-bazel-lock.json linguist-generated=true
+/ruby/extractor/cargo-bazel-lock.json -merge
+
+# auto-generated files for the C# build
+/csharp/paket.lock linguist-generated=true
+# needs eol=crlf, as `paket` touches this file and saves it as crlf
+/csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
+/csharp/paket.main.bzl linguist-generated=true
+/csharp/paket.main_extension.bzl linguist-generated=true
+
+# ripunzip tool
+/misc/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text
+
+# swift prebuilt resources
+/swift/third_party/resource-dir/*.zip filter=lfs diff=lfs merge=lfs -text

.github/dependabot.yml

@@ -17,3 +17,26 @@ updates:
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-patch', 'version-update:semver-minor']
+
+  - package-ecosystem: "gomod"
+    directory: "go/extractor"
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-name: "golang.org/x/mod"
+      - dependency-name: "golang.org/x/tools"
+    groups:
+      extractor-dependencies:
+        patterns:
+          - "golang.org/x/*"
+    reviewers:
+      - "github/codeql-go"
+
+  - package-ecosystem: "gomod"
+    directory: "go/ql/test"
+    schedule:
+      interval: "monthly"
+    ignore:
+      - dependency-name: "*"
+    reviewers:
+      - "github/codeql-go"

.github/labeler.yml

@@ -15,12 +15,12 @@ Java:
   - change-notes/**/*java.*
 
 JS:
-  - any: [ 'javascript/**/*', '!javascript/ql/experimental/adaptivethreatmodeling/**/*' ]
+  - any: [ 'javascript/**/*' ]
   - change-notes/**/*javascript*
 
 Kotlin:
   - java/kotlin-extractor/**/*
-  - java/ql/test/kotlin/**/*
+  - java/ql/test-kotlin*/**/*
 
 Python:
   - python/**/*
@@ -30,6 +30,10 @@ Ruby:
   - ruby/**/*
   - change-notes/**/*ruby*
 
+Rust:
+  - rust/**/*
+  - change-notes/**/*rust*
+
 Swift:
   - swift/**/*
   - change-notes/**/*swift*
@@ -45,11 +49,4 @@ documentation:
 
 # Since these are all shared files that need to be synced, just pick _one_ copy of each.
 "DataFlow Library":
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
-
-"ATM":
-  - javascript/ql/experimental/adaptivethreatmodeling/**/*
+  - "shared/dataflow/**/*"

.github/pull_request_template.md

@@ -0,0 +1,14 @@
+### Pull Request checklist
+
+#### All query authors
+
+- [ ] A change note is added if necessary. See [the documentation](https://github.com/github/codeql/blob/main/docs/change-notes.md) in this repository.
+- [ ] All new queries have appropriate `.qhelp`. See [the documentation](https://github.com/github/codeql/blob/main/docs/query-help-style-guide.md) in this repository.
+- [ ] QL tests are added if necessary. See [Testing custom queries](https://docs.github.com/en/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/testing-custom-queries) in the GitHub documentation.
+- [ ] New and changed queries have correct query metadata. See [the documentation](https://github.com/github/codeql/blob/main/docs/query-metadata-style-guide.md) in this repository.
+
+#### Internal query authors only
+
+- [ ] Autofixes generated based on these changes are valid, only needed if this PR makes significant changes to `.ql`, `.qll`, or `.qhelp` files. See [the documentation](https://github.com/github/codeql-team/blob/main/docs/best-practices/validating-autofix-for-query-changes.md) (internal access required).
+- [ ] Changes are validated [at scale](https://github.com/github/codeql-dca/) (internal access required).
+- [ ] Adding a new query? Consider also [adding the query to autofix](https://github.com/github/codeml-autofix/blob/main/docs/updating-query-support.md#adding-a-new-query-to-the-query-suite).

.github/workflows/build-ripunzip.yml

@@ -0,0 +1,74 @@
+name: Build runzip
+
+on:
+  workflow_dispatch:
+    inputs:
+      ripunzip-version:
+        description: "what reference to checktout from google/runzip"
+        required: false
+        default: v1.2.1
+      openssl-version:
+        description: "what reference to checkout from openssl/openssl for Linux"
+        required: false
+        default: openssl-3.3.0
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, macos-12, windows-2019]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: google/ripunzip
+          ref: ${{ inputs.ripunzip-version }}
+      # we need to avoid ripunzip dynamically linking into libssl
+      # see https://github.com/sfackler/rust-openssl/issues/183
+      - if: runner.os == 'Linux'
+        name: checkout openssl
+        uses: actions/checkout@v4
+        with:
+          repository: openssl/openssl
+          path: openssl
+          ref: ${{ inputs.openssl-version }}
+      - if: runner.os == 'Linux'
+        name: build and install openssl with fPIC
+        shell: bash
+        working-directory: openssl
+        run: |
+          ./config -fPIC --prefix=$HOME/.local --openssldir=$HOME/.local/ssl
+          make -j $(nproc)
+          make install_sw -j $(nproc)
+      - if: runner.os == 'Linux'
+        name: build (linux)
+        shell: bash
+        run: |
+          env OPENSSL_LIB_DIR=$HOME/.local/lib64 OPENSSL_INCLUDE_DIR=$HOME/.local/include OPENSSL_STATIC=yes cargo build --release
+          mv target/release/ripunzip ripunzip-linux
+      - if: runner.os == 'Windows'
+        name: build (windows)
+        shell: bash
+        run: |
+          cargo build --release
+          mv target/release/ripunzip ripunzip-windows
+      - name: build (macOS)
+        if: runner.os == 'macOS'
+        shell: bash
+        run: |
+          rustup target install x86_64-apple-darwin
+          rustup target install aarch64-apple-darwin
+          cargo build --target x86_64-apple-darwin --release
+          cargo build --target aarch64-apple-darwin --release
+          lipo -create -output ripunzip-macos \
+            -arch x86_64 target/x86_64-apple-darwin/release/ripunzip \
+            -arch arm64 target/aarch64-apple-darwin/release/ripunzip
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ripunzip-${{ runner.os }}
+          path: ripunzip-*
+      - name: Check built binary
+        shell: bash
+        run: |
+          ./ripunzip-* --version

.github/workflows/buildifier.yml

@@ -0,0 +1,28 @@
+name: Check bazel formatting
+
+on:
+  pull_request:
+    paths:
+      - "**.bazel"
+      - "**.bzl"
+    branches:
+      - main
+      - "rc/*"
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Check bazel formatting
+        uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
+        with:
+          extra_args: >
+            buildifier --all-files 2>&1 ||
+            (
+              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel/buildifier"; exit 1
+            )

.github/workflows/check-change-note.yml

@@ -1,5 +1,8 @@
 name: Check change note
 
+permissions:
+  pull-requests: read
+
 on:
   pull_request_target:
     types: [labeled, unlabeled, opened, synchronize, reopened, ready_for_review]
@@ -9,26 +12,43 @@ on:
       - "*/ql/lib/**/*.ql"
       - "*/ql/lib/**/*.qll"
       - "*/ql/lib/**/*.yml"
+      - "shared/**/*.ql"
+      - "shared/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
+      - "!rust/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:
   check-change-note:
+    env:
+      REPO: ${{ github.repository }}
+      PULL_REQUEST_NUMBER: ${{ github.event.number }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     runs-on: ubuntu-latest
     steps:
+
       - name: Fail if no change note found. To fix, either add one, or add the `no-change-note-required` label.
         if: |
           github.event.pull_request.draft == false &&
           !contains(github.event.pull_request.labels.*.name, 'no-change-note-required')
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
-          grep true -c
+          change_note_files=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '.[].filename | select(test("/change-notes/.*[.]md$"))')
+
+          if [ -z "$change_note_files" ]; then
+            echo "No change note found. Either add one, or add the 'no-change-note-required' label."
+            exit 1
+          fi
+
+          echo "Change notes found:"
+          echo "$change_note_files"
+
       - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md', 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text, or released/x.y.z.md for released change-notes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$"))' |
-          grep true -c
+          bad_change_note_file_names=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))][] | select((test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$")) | not)')
+
+          if [ -n "$bad_change_note_file_names" ]; then
+            echo "The following change note file names are invalid:"
+            echo "$bad_change_note_file_names"
+            exit 1
+          fi

.github/workflows/check-implicit-this.yml

@@ -9,6 +9,9 @@ on:
       - main
       - "rc/*"
 
+permissions:
+  contents: read
+
 jobs:
   check:
     runs-on: ubuntu-latest