Skip to content

Commit

Permalink
Make: Cleaner SQLite CI
Browse files Browse the repository at this point in the history 
Relates to #376, #373, #371
  • Loading branch information
@ashvardanian
ashvardanian committed Apr 1, 2024
1 parent 3fd5184 commit e0bc522
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 74 deletions.
20 changes: 6 additions & 14 deletions .github/workflows/release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ jobs:
touch "usearch_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/DEBIAN/control"
mkdir -p "usearch_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/lib"
mkdir "usearch_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/include"
cp include/c/usearch.h "usearch_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/include/"
cp c/usearch.h "usearch_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/include/"
cp build_release/libusearch_c.so "usearch_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/lib/"
echo -e "Package: usearch\nVersion: ${{ steps.set_version.outputs.version }}\nMaintainer: Ash Vardanian\nArchitecture: ${{ matrix.arch }}\nDescription: Faster & Smaller Single-File Search Engine for Vectors & Texts" > "usearch_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/DEBIAN/control"
dpkg-deb --build "usearch_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}"
Expand All @@ -125,15 +125,7 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
file: "*.so"
update_latest_release: true

- name: Upload archive
uses: xresloader/upload-to-github-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
file: "*.deb"
file: "*.so;*.deb"
update_latest_release: true

create_windows_dll_library:
Expand Down Expand Up @@ -184,7 +176,7 @@ jobs:
file: "usearch_windows_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}.tar"
update_latest_release: true

create_macos_library:
create_macos_dylib_library:
name: Create Library for MacOS ${{ matrix.arch }}
runs-on: macos-12
needs: versioning
Expand Down Expand Up @@ -237,7 +229,7 @@ jobs:
file: "*.zip"
update_latest_release: true

wasm_c_library:
create_wasm_library:
name: WASM builds for C libraries on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
needs: versioning
Expand Down Expand Up @@ -329,8 +321,8 @@ jobs:
versioning,
create_linux_deb_package,
create_windows_dll_library,
create_macos_library,
wasm_c_library,
create_macos_dylib_library,
create_wasm_library,
deploy_docs_pages,
]
steps:
Expand Down
10 changes: 5 additions & 5 deletions python/scripts/test_sqlite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def test_sqlite_minimal_json_cosine_vector_search():
pytest.skip("SQLite extensions are not available on this platform")
return

conn.load_extension(usearch.sqlite)
conn.load_extension(usearch.sqlite_path())
cursor = conn.cursor()

# Create a table with a JSON column for vectors
Expand Down Expand Up @@ -76,7 +76,7 @@ def test_sqlite_minimal_text_search():
pytest.skip("SQLite extensions are not available on this platform")
return

conn.load_extension(usearch.sqlite)
conn.load_extension(usearch.sqlite_path())
cursor = conn.cursor()

# Create a table with a TEXT column for strings
Expand Down Expand Up @@ -132,7 +132,7 @@ def test_sqlite_blob_bits_vector_search():
pytest.skip("SQLite extensions are not available on this platform")
return

conn.load_extension(usearch.sqlite)
conn.load_extension(usearch.sqlite_path())
cursor = conn.cursor()

# Create a table with a BLOB column for binary vectors
Expand Down Expand Up @@ -195,7 +195,7 @@ def test_sqlite_distances_in_high_dimensions(num_vectors: int, ndim: int):
pytest.skip("SQLite extensions are not available on this platform")
return

conn.load_extension(usearch.sqlite)
conn.load_extension(usearch.sqlite_path())
cursor = conn.cursor()

# Create a table with additional columns for f32 and f16 BLOBs
Expand Down Expand Up @@ -271,7 +271,7 @@ def test_sqlite_distances_in_low_dimensions(num_vectors: int):
pytest.skip("SQLite extensions are not available on this platform")
return

conn.load_extension(usearch.sqlite)
conn.load_extension(usearch.sqlite_path())
cursor = conn.cursor()

# Create a table for storing vectors and their descriptions
Expand Down
100 changes: 45 additions & 55 deletions python/usearch/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import tempfile
import warnings
import urllib.request
from typing import Optional
from typing import Optional, Tuple
from urllib.error import HTTPError


Expand All @@ -21,7 +21,6 @@ def __init__(self, version: Optional[str] = None):
if version is None:
version = __version__
self.version = version or __version__
self.download_dir = self.determine_download_dir()

@staticmethod
def determine_download_dir():
Expand All @@ -35,7 +34,13 @@ def determine_download_dir():
home_dir = os.path.expanduser("~")
return os.path.join(home_dir, ".usearch", "binaries")

def sqlite_file_name(self) -> str:
@staticmethod
def determine_download_url(version: str, filename: str) -> str:
base_url = "https://github.com/unum-cloud/usearch/releases/download"
url = f"{base_url}/v{version}/{filename}"
return url

def get_binary_name(self) -> Tuple[str, str]:
version = self.version
os_map = {"Linux": "linux", "Windows": "windows", "Darwin": "macos"}
arch_map = {
Expand All @@ -49,31 +54,9 @@ def sqlite_file_name(self) -> str:
arch = platform.machine()
arch_part = arch_map.get(arch, "")
extension = {"Linux": "so", "Windows": "dll", "Darwin": "dylib"}.get(platform.system(), "")
filename = f"usearch_sqlite_{os_part}_{arch_part}_{version}.{extension}"
return filename

def sqlite_download_url(self) -> str:
version = self.version
filename = self.sqlite_file_name()
base_url = "https://github.com/unum-cloud/usearch/releases/download"
url = f"{base_url}/v{version}/{filename}"
return url

def download_binary(self, url: str, dest_folder: str) -> str:
"""
Downloads a file from a given URL to a specified destination folder.
Args:
url (str): The URL to download the file from.
dest_folder (str): The folder where the file will be saved.
Returns:
The path to the downloaded file.
"""
filename = url.split("/")[-1]
dest_path = os.path.join(dest_folder, filename)
urllib.request.urlretrieve(url, dest_path)
return dest_path
source_filename = f"usearch_sqlite_{os_part}_{arch_part}_{version}.{extension}"
target_filename = f"usearch_sqlite.{extension}"
return source_filename, target_filename

def sqlite_found_or_downloaded(self) -> Optional[str]:
"""
Expand All @@ -85,37 +68,44 @@ def sqlite_found_or_downloaded(self) -> Optional[str]:
"""
# Search local directories
local_dirs = ["build", "build_artifacts", "build_release", "build_debug"]
extensions = {"Linux": ".so", "Windows": ".dll", "Darwin": ".dylib"}
os_type = platform.system()
file_extension = extensions.get(os_type, "")
source_filename, target_filename = self.get_binary_name()

# Check local development directories first
for directory in local_dirs:
for root, _, files in os.walk(directory):
for file in files:
if file.endswith(file_extension) and "usearch_sqlite" in file:
return os.path.join(root, file).removesuffix(file_extension)

# Check a temporary directory (assuming the binary might be downloaded from a GitHub release)
local_path = os.path.join(self.download_dir, self.sqlite_file_name())
if os.path.exists(local_path):
return local_path.removesuffix(file_extension)

# If not found locally, warn the user and download from GitHub
warnings.warn("Will download `usearch_sqlite` binary from GitHub.", UserWarning)
try:
binary_path = self.download_binary(self.sqlite_download_url(), self.download_dir)
except HTTPError as e:
# If the download fails due to HTTPError (e.g., 404 Not Found), like a missing lib version
if e.code == 404:
warnings.warn(f"Download failed: {e.url} could not be found.", UserWarning)
else:
warnings.warn(f"Download failed with HTTP error: {e.code} {e.reason}", UserWarning)
return None
for local_dir in local_dirs:

local_path = os.path.join(local_dir, target_filename)
if os.path.exists(local_path):
path_wout_extension, _, _ = local_path.rpartition(".")
return path_wout_extension

# Most build systems on POSIX would prefix the library name with "lib"
local_path = os.path.join(local_dir, "lib" + target_filename)
if os.path.exists(local_path):
path_wout_extension, _, _ = local_path.rpartition(".")
return path_wout_extension

# Check local installation directories, in case the build is already installed
download_dir = self.determine_download_dir()
local_path = os.path.join(download_dir, target_filename)
if not os.path.exists(local_path):

# If not found locally, warn the user and download from GitHub
warnings.warn("Will download `usearch_sqlite` binary from GitHub.", UserWarning)
try:
source_url = self.determine_download_url(self.version, source_filename)
urllib.request.urlretrieve(source_url, local_path)
except HTTPError as e:
# If the download fails due to HTTPError (e.g., 404 Not Found), like a missing lib version
if e.code == 404:
warnings.warn(f"Download failed: {e.url} could not be found.", UserWarning)
else:
warnings.warn(f"Download failed with HTTP error: {e.code} {e.reason}", UserWarning)
return None

# Handle the case where binary_path does not exist after supposed successful download
if os.path.exists(binary_path):
return binary_path.removesuffix(file_extension)
if os.path.exists(local_path):
path_wout_extension, _, _ = local_path.rpartition(".")
return path_wout_extension
else:
warnings.warn("Failed to download `usearch_sqlite` binary from GitHub.", UserWarning)
return None
Expand Down

0 comments on commit e0bc522

Please sign in to comment.