From c2f18f45326dd6f0694ca5a45f48e58b3e618dbe Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Wed, 11 Jan 2023 21:17:02 -0600 Subject: [PATCH 01/11] Implemented pack command and pack bytes --- redis/connection.py | 31 ++- redis/utils.py | 6 + redisrs-py/.github/workflows/CI.yml | 70 +++++ redisrs-py/.gitignore | 72 +++++ redisrs-py/Cargo.lock | 394 ++++++++++++++++++++++++++++ redisrs-py/Cargo.toml | 13 + redisrs-py/pyproject.toml | 14 + redisrs-py/src/lib.rs | 62 +++++ redisrs-py/test/bench.py | 231 ++++++++++++++++ redisrs-py/test/test.py | 43 +++ 10 files changed, 934 insertions(+), 2 deletions(-) mode change 100755 => 100644 redis/connection.py create mode 100644 redisrs-py/.github/workflows/CI.yml create mode 100644 redisrs-py/.gitignore create mode 100644 redisrs-py/Cargo.lock create mode 100644 redisrs-py/Cargo.toml create mode 100644 redisrs-py/pyproject.toml create mode 100644 redisrs-py/src/lib.rs create mode 100644 redisrs-py/test/bench.py create mode 100644 redisrs-py/test/test.py diff --git a/redis/connection.py b/redis/connection.py old mode 100755 new mode 100644 index 126ea5db32..509ffec24d --- a/redis/connection.py +++ b/redis/connection.py @@ -31,7 +31,7 @@ TimeoutError, ) from redis.retry import Retry -from redis.utils import CRYPTOGRAPHY_AVAILABLE, HIREDIS_AVAILABLE, str_if_bytes +from redis.utils import CRYPTOGRAPHY_AVAILABLE, HIREDIS_AVAILABLE, REDISRS_PY_AVAILABLE, str_if_bytes try: import ssl @@ -54,6 +54,11 @@ if HIREDIS_AVAILABLE: import hiredis + +if REDISRS_PY_AVAILABLE: + import redisrs_py + + SYM_STAR = b"*" SYM_DOLLAR = b"$" SYM_CRLF = b"\r\n" @@ -504,6 +509,23 @@ def read_response(self, disable_decoding=False): DefaultParser = PythonParser +def pack_command_redisrs(*args): + """Pack a series of arguments into the Redis protocol""" + output = [] + if isinstance(args[0], str): + args = tuple(args[0].encode().split()) + args[1:] + elif b" " in args[0]: + args = tuple(args[0].split()) + args[1:] + output.append(redisrs_py.pack_command(args)) + return output + + +if REDISRS_PY_AVAILABLE: + print("REDISRS_PY_AVAILABLE") + DefaultCommandPacker = pack_command_redisrs +else: + DefaultCommandPacker = None + class Connection: "Manages TCP communication to and from a Redis server" @@ -531,6 +553,7 @@ def __init__( retry=None, redis_connect_func=None, credential_provider: Optional[CredentialProvider] = None, + pack_command=DefaultCommandPacker, ): """ Initialize a new Connection. @@ -585,6 +608,10 @@ def __init__( self.set_parser(parser_class) self._connect_callbacks = [] self._buffer_cutoff = 6000 + if pack_command is not None: + self.pack_command = pack_command + else: + self.pack_command = self._pack_command_python def __repr__(self): repr_args = ",".join([f"{k}={v}" for k, v in self.repr_pieces()]) @@ -865,7 +892,7 @@ def read_response(self, disable_decoding=False): raise response return response - def pack_command(self, *args): + def _pack_command_python(self, *args): """Pack a series of arguments into the Redis protocol""" output = [] # the client might have included 1 or more literal arguments in diff --git a/redis/utils.py b/redis/utils.py index 693d4e64b5..b3e6b5ef1c 100644 --- a/redis/utils.py +++ b/redis/utils.py @@ -17,6 +17,12 @@ except ImportError: CRYPTOGRAPHY_AVAILABLE = False +try: + import redisrs_py + REDISRS_PY_AVAILABLE = True +except ImportError: + REDISRS_PY_AVAILABLE = False + def from_url(url, **kwargs): """ diff --git a/redisrs-py/.github/workflows/CI.yml b/redisrs-py/.github/workflows/CI.yml new file mode 100644 index 0000000000..074743e816 --- /dev/null +++ b/redisrs-py/.github/workflows/CI.yml @@ -0,0 +1,70 @@ +name: CI + +on: + push: + branches: + - main + - master + pull_request: + workflow_dispatch: + +jobs: + linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: PyO3/maturin-action@v1 + with: + manylinux: auto + command: build + args: --release --sdist -o dist --find-interpreter + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + - uses: PyO3/maturin-action@v1 + with: + command: build + args: --release -o dist --find-interpreter + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - uses: PyO3/maturin-action@v1 + with: + command: build + args: --release -o dist --universal2 --find-interpreter + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + release: + name: Release + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/')" + needs: [ macos, windows, linux ] + steps: + - uses: actions/download-artifact@v3 + with: + name: wheels + - name: Publish to PyPI + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + with: + command: upload + args: --skip-existing * \ No newline at end of file diff --git a/redisrs-py/.gitignore b/redisrs-py/.gitignore new file mode 100644 index 0000000000..af3ca5ef1c --- /dev/null +++ b/redisrs-py/.gitignore @@ -0,0 +1,72 @@ +/target + +# Byte-compiled / optimized / DLL files +__pycache__/ +.pytest_cache/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +.venv/ +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +include/ +man/ +venv/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt +pip-selfcheck.json + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +.DS_Store + +# Sphinx documentation +docs/_build/ + +# PyCharm +.idea/ + +# VSCode +.vscode/ + +# Pyenv +.python-version \ No newline at end of file diff --git a/redisrs-py/Cargo.lock b/redisrs-py/Cargo.lock new file mode 100644 index 0000000000..aaf449fc34 --- /dev/null +++ b/redisrs-py/Cargo.lock @@ -0,0 +1,394 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bytes" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indoc" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da2d6f23ffea9d7e76c53eee25dfb67bcd8fde7f1198b0855350698c9f07c780" + +[[package]] +name = "itoa" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" + +[[package]] +name = "libc" +version = "0.2.139" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "proc-macro2" +version = "1.0.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "parking_lot", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redis" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1aada340fba5deba625c84d109d0a83cc3565452d38083417992a702c2428d" +dependencies = [ + "combine", + "itoa", + "percent-encoding", + "ryu", + "sha1_smol", + "url", +] + +[[package]] +name = "redisrs-py" +version = "0.1.0" +dependencies = [ + "pyo3", + "redis", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ryu" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "syn" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9410d0f6853b1d94f0e519fb95df60f29d2c1eff2d921ffdf01a4c8a3b54f12d" + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unindent" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" diff --git a/redisrs-py/Cargo.toml b/redisrs-py/Cargo.toml new file mode 100644 index 0000000000..80ae3facb6 --- /dev/null +++ b/redisrs-py/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "redisrs-py" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "redisrs_py" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.17.3", features = ["extension-module"] } +redis = { version= "0.22.2"} diff --git a/redisrs-py/pyproject.toml b/redisrs-py/pyproject.toml new file mode 100644 index 0000000000..c8e7d8e26c --- /dev/null +++ b/redisrs-py/pyproject.toml @@ -0,0 +1,14 @@ +[build-system] +requires = ["maturin>=0.14,<0.15"] +build-backend = "maturin" + +[project] +name = "redisrs-py" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + + diff --git a/redisrs-py/src/lib.rs b/redisrs-py/src/lib.rs new file mode 100644 index 0000000000..a6bbb4e789 --- /dev/null +++ b/redisrs-py/src/lib.rs @@ -0,0 +1,62 @@ +extern crate redis; + +use pyo3::prelude::*; +use pyo3::{ types::{PyTuple, PyBytes, PyByteArray, PyString, PyInt, PyLong, PyFloat } }; + + +#[pyfunction] +fn pack_command<'a>(py: Python<'a>, items: &'a PyTuple) -> &'a PyBytes { + // Redis-py:connection.py:Encoder accepts only: bytes, string, int or float. + // + let mut cmd = redis::Cmd::new().to_owned(); + for item in items { + if item.is_instance_of::().unwrap() || item.is_instance_of::().unwrap() { + let bytes: &[u8] = item.extract().unwrap(); + cmd.arg(bytes); + } else if item.is_instance_of::().unwrap() { + let str_item : String = item.extract().unwrap(); + cmd.arg(str_item); + } else if item.is_instance_of::().unwrap() || item.is_instance_of::().unwrap() { + let num_item : i64 = item.extract().unwrap(); + cmd.arg(num_item); + } else if item.is_instance_of::().unwrap() { + let float_item : f64 = item.extract().unwrap(); + cmd.arg(float_item); + } else { + println!("not yet"); + } + } + + PyBytes::new(py, &cmd.get_packed_command()[..]) +} + + +#[pyfunction] +fn pack_bytes<'a>(py: Python<'a>, bytes_cmd: &'a [u8]) -> &'a PyBytes { + + let mut cmd = redis::Cmd::new().to_owned(); + let mut start = 0; + for (i, &item) in bytes_cmd.iter().enumerate() { + if item == b' ' { + if i > start { + cmd.arg(&bytes_cmd[start..i]); + } + start = i + 1; + } + } + + if start < bytes_cmd.len() { + cmd.arg(&bytes_cmd[start..bytes_cmd.len()]); + } + + PyBytes::new(py, &cmd.get_packed_command()[..]) +} + + +/// A Python module implemented in Rust. +#[pymodule] +fn redisrs_py(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(pack_command, m)?)?; + m.add_function(wrap_pyfunction!(pack_bytes, m)?)?; + Ok(()) +} \ No newline at end of file diff --git a/redisrs-py/test/bench.py b/redisrs-py/test/bench.py new file mode 100644 index 0000000000..a4f4ee1573 --- /dev/null +++ b/redisrs-py/test/bench.py @@ -0,0 +1,231 @@ +from random import choice +from string import ascii_uppercase +import timeit +import redisrs_py +import hiredis + +############################################################### +# COPY-PASTE +############################################################### + +encoding="utf-8" +encoding_errors="strict" +decode_responses=False + +#pure copy-paste from connetion.py + +class Encoder: + "Encode strings to bytes-like and decode bytes-like to strings" + + def __init__(self, encoding, encoding_errors, decode_responses): + self.encoding = encoding + self.encoding_errors = encoding_errors + self.decode_responses = decode_responses + + def encode(self, value): + "Return a bytestring or bytes-like representation of the value" + if isinstance(value, (bytes, memoryview)): + return value + elif isinstance(value, bool): + # special case bool since it is a subclass of int + raise Exception( + "Invalid input of type: 'bool'. Convert to a " + "bytes, string, int or float first." + ) + elif isinstance(value, (int, float)): + value = repr(value).encode() + elif not isinstance(value, str): + # a value we don't know how to deal with. throw an error + typename = type(value).__name__ + raise Exception( + f"Invalid input of type: '{typename}'. {value}" + f"Convert to a bytes, string, int or float first." + ) + if isinstance(value, str): + value = value.encode(self.encoding, self.encoding_errors) + return value + + def decode(self, value, force=False): + "Return a unicode string from the bytes-like representation" + if self.decode_responses or force: + if isinstance(value, memoryview): + value = value.tobytes() + if isinstance(value, bytes): + value = value.decode(self.encoding, self.encoding_errors) + return value + +SYM_STAR = b'*' +SYM_DOLLAR = b'$' +SYM_CRLF = b'\r\n' +SYM_EMPTY = b'' + +encoder = Encoder(encoding, encoding_errors, decode_responses) + +#almost pure copy-paste from connetion.py: got rid of self.encoder and self.buffer_cutoff + +def pack_command(*args): + """Pack a series of arguments into the Redis protocol""" + output = [] + # the client might have included 1 or more literal arguments in + # the command name, e.g., 'CONFIG GET'. The Redis server expects these + # arguments to be sent separately, so split the first argument + # manually. These arguments should be bytestrings so that they are + # not encoded. + if isinstance(args[0], str): + args = tuple(args[0].encode().split()) + args[1:] + elif b" " in args[0]: + args = tuple(args[0].split()) + args[1:] + buff = SYM_EMPTY.join((SYM_STAR, str(len(args)).encode(), SYM_CRLF)) + + buffer_cutoff = 6000 + global encoder + for arg in map(encoder.encode, args): + # to avoid large string mallocs, chunk the command into the + # output list if we're sending large values or memoryviews + arg_length = len(arg) + if ( + len(buff) > buffer_cutoff + or arg_length > buffer_cutoff + or isinstance(arg, memoryview) + ): + buff = SYM_EMPTY.join( + (buff, SYM_DOLLAR, str(arg_length).encode(), SYM_CRLF) + ) + output.append(buff) + output.append(arg) + buff = SYM_CRLF + else: + buff = SYM_EMPTY.join( + ( + buff, + SYM_DOLLAR, + str(arg_length).encode(), + SYM_CRLF, + arg, + SYM_CRLF, + ) + ) + output.append(buff) + return output + + +############################################################### +# WRAPPER FUNCTIONS +# pack_command_* - encoding on module extention side +# pack_bytes_* encoding on python side +############################################################### + + +def pack_bytes_hiredis(*args): + """Pack a series of arguments into the Redis protocol""" + output = [] + if isinstance(args[0], str): + args = tuple(args[0].encode().split()) + args[1:] + elif b" " in args[0]: + args = tuple(args[0].split()) + args[1:] + + global encoder + cmd = b' '.join(map(encoder.encode, args)) + + output.append(hiredis.pack_bytes(cmd)) + return output + + +def pack_command_hiredis(*args): + """Pack a series of arguments into the Redis protocol""" + output = [] + if isinstance(args[0], str): + args = tuple(args[0].encode().split()) + args[1:] + elif b" " in args[0]: + args = tuple(args[0].split()) + args[1:] + output.append(hiredis.pack_command(args)) + return output + +#Rust +def pack_bytes_rust(*args): + """Pack a series of arguments into the Redis protocol""" + output = [] + if isinstance(args[0], str): + args = tuple(args[0].encode().split()) + args[1:] + elif b" " in args[0]: + args = tuple(args[0].split()) + args[1:] + + global encoder + cmd = b' '.join(map(encoder.encode, args)) + + output.append(redisrs_py.pack_bytes(cmd)) + return output + + +def pack_command_rust(*args): + """Pack a series of arguments into the Redis protocol""" + output = [] + if isinstance(args[0], str): + args = tuple(args[0].encode().split()) + args[1:] + elif b" " in args[0]: + args = tuple(args[0].split()) + args[1:] + output.append(redisrs_py.pack_command(args)) + return output + + +############################################################### +# BENCHMARKING +############################################################### + + +TOTAL_CMDS = 100000 + +def get_entry(number_of_fields): + fields = (b'HSET', f'foo42') + for i in range(number_of_fields): + fields += (f"field{i}",) + fields += (''.join(choice(ascii_uppercase) for _ in range(10)),) + return fields + + +def get_input(number_of_fields, total=TOTAL_CMDS): + entry = get_entry(number_of_fields) + input = [] + for _ in range(total): + input.append(entry) + return input + + +input_0 = [] +for i in range(TOTAL_CMDS): + input_0.append( + (b'HSET', f'foo{i}', 42, 'zoo', 'a', 67, 43, 3.56, 'foo1', 'qwertyuiop', 'foo2', 3.1412345678901, 52, 'zxcvbnmmpei', 53, 'dhjihoihpouihj', 'kjl;kjhj;lkjlk;', '567890798783', 'kjkjkjkjk', 79878933334890808709890.0) + ) +input_17 = get_input(17) +input_50 = get_input(50) +input_100 = get_input(100) + + +ITERATIONS = 5 + + +def bench(input_name, description): + print(description) + res = timeit.timeit(f'[pack_command(*cmd) for cmd in {input_name}]', globals=globals(), number=ITERATIONS) + print(f'BASELINE:redis.py: {res}') + + print("IMPLEMENTATIONS:") + res = timeit.timeit(f'[pack_bytes_hiredis(*cmd) for cmd in {input_name}]', globals=globals(), number=ITERATIONS) + print(f'hiredis.pack_bytes: {res}') + + res = timeit.timeit(f'[pack_bytes_rust(*cmd) for cmd in {input_name}]', globals=globals(), number=ITERATIONS) + print(f'redisrs-py.pack_bytes: {res}') + + if input_name == 'input_0': + print('hiredis.pack_command - not supported (yet?!)') + else: + res = timeit.timeit(f'[pack_command_hiredis(*cmd) for cmd in {input_name}]', globals=globals(), number=ITERATIONS) + print(f'hiredis.pack_command(partial encoding in hiredis-C): {res}') + + res = timeit.timeit(f'[pack_command_rust(*cmd) for cmd in {input_name}]', globals=globals(), number=ITERATIONS) + print(f'redisrs_py.pack_command(encoding in Rust): {res}') + +bench("input_0", "========= 10 fields of various types: str, int, float, bytes") +bench("input_17", "\n========= 17 10-bytes fileds per entry") +bench("input_50", "\n========= 50 10-bytes fileds per entry") +bench("input_100", "\n========= 100 10-bytes fileds per entry") diff --git a/redisrs-py/test/test.py b/redisrs-py/test/test.py new file mode 100644 index 0000000000..7c59755189 --- /dev/null +++ b/redisrs-py/test/test.py @@ -0,0 +1,43 @@ +import redis + +import redisrs_py +import hiredis +r = redis.from_url('redis://localhost:12005') +print(r.ping()) + + +# print(r.hset("foo1", "42", 4.09,)) +# print(r.hset("foo2", items=[42, "zoo", 'a', 67, 42, 3.56])) +print(r.hset("foo3", mapping={42: "zoo", 'a': 67, 43: 3.56})) + +# print(redisrs_py.pack_command(('ping',))) + +# print(redisrs_py.pack_command((b'HSET', 'foo3', 42, 'zoo', 'a', 67, 43, 3.56))) + +# args = (b'HSET', 'foo3', 42, 'zoo', 'a', 67, 43, 3.56) +# #args = ['bbb'] +# print(hiredis.pack_command(args)) +# bytes_cmd = b'hset key1 foo bar zoo 4567890 baz 9009870987098 zoo 3.14569808' +# print(hiredis.pack_bytes(bytes_cmd)) + +import unittest + +class TestPackBytes(unittest.TestCase): + def test_basic(self): + cmd = b'HSET foo3 42 zoo a 67 43 3.56' + expected_packed = b'*8\r\n$4\r\nHSET\r\n$4\r\nfoo3\r\n$2\r\n42\r\n$3\r\nzoo\r\n$1\r\na\r\n$2\r\n67\r\n$2\r\n43\r\n$4\r\n3.56\r\n' + packed_cmd = hiredis.pack_bytes(cmd) + self.assertEqual(packed_cmd, expected_packed) + packed_cmd = redisrs_py.pack_bytes(cmd) + self.assertEqual(packed_cmd, expected_packed) + + def test_multiple_spaces(self): + cmd = b' HSET foo3 42 zoo a 67 43 3.56 ' + expected_packed = b'*8\r\n$4\r\nHSET\r\n$4\r\nfoo3\r\n$2\r\n42\r\n$3\r\nzoo\r\n$1\r\na\r\n$2\r\n67\r\n$2\r\n43\r\n$4\r\n3.56\r\n' + packed_cmd = hiredis.pack_bytes(cmd) + self.assertEqual(packed_cmd, expected_packed) + packed_cmd = redisrs_py.pack_bytes(cmd) + self.assertEqual(packed_cmd, expected_packed) + +if __name__ == '__main__': + unittest.main() From 668eeb4c6966ff875cfdcee7b0136c0460035da2 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Sat, 14 Jan 2023 05:00:42 -0600 Subject: [PATCH 02/11] 1) refactored the command packer construction process 2) now hiredis.pack_bytes is the default choice. Though it's still possible to run redisrs-py (fix the flag in utils.py) or hiredis.pack_command (flag in connection.py) --- benchmarks/basic_operations.py | 11 ++++++- redis/connection.py | 60 +++++++++++++++++++++++++++------- redis/utils.py | 2 +- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/benchmarks/basic_operations.py b/benchmarks/basic_operations.py index c9f5853652..67b526ed7e 100644 --- a/benchmarks/basic_operations.py +++ b/benchmarks/basic_operations.py @@ -173,13 +173,22 @@ def lpop(conn, num, pipeline_size, data_size): if pipeline_size > 1: conn.execute() +from random import choice +from string import ascii_uppercase + +def get_data(number_of_fields): + fields = {} + for i in range(number_of_fields): + fields[f"field{i}"] = ''.join(choice(ascii_uppercase) for _ in range(10)) + return fields @timer def hmset(conn, num, pipeline_size, data_size): if pipeline_size > 1: conn = conn.pipeline() - set_data = {"str_value": "string", "int_value": 123456, "float_value": 123456.0} + # set_data = {"str_value": "string", "int_value": 123456, "float_value": 123456.0} + set_data = get_data(17) #{"str_value": "string", "int_value": 123456, "float_value": 123456.0} for i in range(num): conn.hmset("hmset_key", set_data) if pipeline_size > 1 and i % pipeline_size == 0: diff --git a/redis/connection.py b/redis/connection.py index 509ffec24d..f91aa9e592 100644 --- a/redis/connection.py +++ b/redis/connection.py @@ -501,7 +501,7 @@ def read_response(self, disable_decoding=False): ): raise response[0] return response - +# HIREDIS_AVAILABLE = False if HIREDIS_AVAILABLE: DefaultParser = HiredisParser @@ -520,11 +520,18 @@ def pack_command_redisrs(*args): return output -if REDISRS_PY_AVAILABLE: - print("REDISRS_PY_AVAILABLE") - DefaultCommandPacker = pack_command_redisrs -else: - DefaultCommandPacker = None +def pack_command_hiredis(*args): + """Pack a series of arguments into the Redis protocol""" + output = [] + if isinstance(args[0], str): + args = tuple(args[0].encode().split()) + args[1:] + elif b" " in args[0]: + args = tuple(args[0].split()) + args[1:] + output.append(hiredis.pack_command(args)) + return output + +# temporary, until we decide on actual impl +PACK_BYTES = True class Connection: "Manages TCP communication to and from a Redis server" @@ -553,7 +560,7 @@ def __init__( retry=None, redis_connect_func=None, credential_provider: Optional[CredentialProvider] = None, - pack_command=DefaultCommandPacker, + pack_command=None, ): """ Initialize a new Connection. @@ -608,10 +615,7 @@ def __init__( self.set_parser(parser_class) self._connect_callbacks = [] self._buffer_cutoff = 6000 - if pack_command is not None: - self.pack_command = pack_command - else: - self.pack_command = self._pack_command_python + self.pack_command = self._construct_pack_command(pack_command) def __repr__(self): repr_args = ",".join([f"{k}={v}" for k, v in self.repr_pieces()]) @@ -628,6 +632,25 @@ def __del__(self): self.disconnect() except Exception: pass + + def _construct_pack_command(self, packer): + # print-s are temporary + if packer is not None: + return packer + else: + if REDISRS_PY_AVAILABLE: + print("PACK COMMAND BY REDISRS_PY") + return pack_command_redisrs + elif HIREDIS_AVAILABLE and hiredis.__version__ >= "2.1.2": + if PACK_BYTES: + print("PACK BYTES BY HIREDIS") + return self._pack_bytes_hiredis + else: + print("PACK COMMAND BY HIREDIS") + return pack_command_hiredis + else: + print("PACK COMMAND BY PYTHON") + return self._pack_command_python def register_connect_callback(self, callback): self._connect_callbacks.append(weakref.WeakMethod(callback)) @@ -892,6 +915,21 @@ def read_response(self, disable_decoding=False): raise response return response + def _pack_bytes_hiredis(self, *args): + """Pack a series of arguments into the Redis protocol""" + output = [] + if isinstance(args[0], str): + args = tuple(args[0].encode().split()) + args[1:] + elif b" " in args[0]: + args = tuple(args[0].split()) + args[1:] + + global encoder + cmd = b' '.join(map(self.encoder.encode, args)) + + output.append(hiredis.pack_bytes(cmd)) + return output + + def _pack_command_python(self, *args): """Pack a series of arguments into the Redis protocol""" output = [] diff --git a/redis/utils.py b/redis/utils.py index b3e6b5ef1c..48a642adbc 100644 --- a/redis/utils.py +++ b/redis/utils.py @@ -22,7 +22,7 @@ REDISRS_PY_AVAILABLE = True except ImportError: REDISRS_PY_AVAILABLE = False - +REDISRS_PY_AVAILABLE = False def from_url(url, **kwargs): """ From 8cff9dfabb875450dde359c82f49918b55c45399 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Mon, 16 Jan 2023 10:37:26 -0600 Subject: [PATCH 03/11] Switch to hiredis.pack_command --- redis/connection.py | 41 ++++++++++++----------------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/redis/connection.py b/redis/connection.py index f91aa9e592..3a7ed13c49 100644 --- a/redis/connection.py +++ b/redis/connection.py @@ -3,11 +3,13 @@ import io import os import socket +import sys import threading import weakref from itertools import chain from queue import Empty, Full, LifoQueue from time import time +from packaging.version import Version from typing import Optional from urllib.parse import parse_qs, unquote, urlparse @@ -501,7 +503,6 @@ def read_response(self, disable_decoding=False): ): raise response[0] return response -# HIREDIS_AVAILABLE = False if HIREDIS_AVAILABLE: DefaultParser = HiredisParser @@ -509,29 +510,22 @@ def read_response(self, disable_decoding=False): DefaultParser = PythonParser -def pack_command_redisrs(*args): - """Pack a series of arguments into the Redis protocol""" - output = [] - if isinstance(args[0], str): - args = tuple(args[0].encode().split()) + args[1:] - elif b" " in args[0]: - args = tuple(args[0].split()) + args[1:] - output.append(redisrs_py.pack_command(args)) - return output - - def pack_command_hiredis(*args): """Pack a series of arguments into the Redis protocol""" output = [] + if isinstance(args[0], str): args = tuple(args[0].encode().split()) + args[1:] elif b" " in args[0]: args = tuple(args[0].split()) + args[1:] - output.append(hiredis.pack_command(args)) + try: + output.append(hiredis.pack_command(args)) + except TypeError as err: + _, value, traceback = sys.exc_info() + raise DataError(value).with_traceback(traceback) + return output -# temporary, until we decide on actual impl -PACK_BYTES = True class Connection: "Manages TCP communication to and from a Redis server" @@ -634,23 +628,12 @@ def __del__(self): pass def _construct_pack_command(self, packer): - # print-s are temporary if packer is not None: return packer + elif HIREDIS_AVAILABLE and Version(hiredis.__version__) >= Version("2.1.2"): + return pack_command_hiredis else: - if REDISRS_PY_AVAILABLE: - print("PACK COMMAND BY REDISRS_PY") - return pack_command_redisrs - elif HIREDIS_AVAILABLE and hiredis.__version__ >= "2.1.2": - if PACK_BYTES: - print("PACK BYTES BY HIREDIS") - return self._pack_bytes_hiredis - else: - print("PACK COMMAND BY HIREDIS") - return pack_command_hiredis - else: - print("PACK COMMAND BY PYTHON") - return self._pack_command_python + return self._pack_command_python def register_connect_callback(self, callback): self._connect_callbacks.append(weakref.WeakMethod(callback)) From 7a266074e33313f0b1e1e8f2a14551f1e49438a8 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Wed, 18 Jan 2023 10:14:05 -0600 Subject: [PATCH 04/11] Remove the rust extension module. --- redisrs-py/.github/workflows/CI.yml | 70 ----- redisrs-py/.gitignore | 72 ----- redisrs-py/Cargo.lock | 394 ---------------------------- redisrs-py/Cargo.toml | 13 - redisrs-py/pyproject.toml | 14 - redisrs-py/src/lib.rs | 62 ----- redisrs-py/test/bench.py | 231 ---------------- redisrs-py/test/test.py | 43 --- 8 files changed, 899 deletions(-) delete mode 100644 redisrs-py/.github/workflows/CI.yml delete mode 100644 redisrs-py/.gitignore delete mode 100644 redisrs-py/Cargo.lock delete mode 100644 redisrs-py/Cargo.toml delete mode 100644 redisrs-py/pyproject.toml delete mode 100644 redisrs-py/src/lib.rs delete mode 100644 redisrs-py/test/bench.py delete mode 100644 redisrs-py/test/test.py diff --git a/redisrs-py/.github/workflows/CI.yml b/redisrs-py/.github/workflows/CI.yml deleted file mode 100644 index 074743e816..0000000000 --- a/redisrs-py/.github/workflows/CI.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: CI - -on: - push: - branches: - - main - - master - pull_request: - workflow_dispatch: - -jobs: - linux: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: PyO3/maturin-action@v1 - with: - manylinux: auto - command: build - args: --release --sdist -o dist --find-interpreter - - name: Upload wheels - uses: actions/upload-artifact@v3 - with: - name: wheels - path: dist - - windows: - runs-on: windows-latest - steps: - - uses: actions/checkout@v3 - - uses: PyO3/maturin-action@v1 - with: - command: build - args: --release -o dist --find-interpreter - - name: Upload wheels - uses: actions/upload-artifact@v3 - with: - name: wheels - path: dist - - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - - uses: PyO3/maturin-action@v1 - with: - command: build - args: --release -o dist --universal2 --find-interpreter - - name: Upload wheels - uses: actions/upload-artifact@v3 - with: - name: wheels - path: dist - - release: - name: Release - runs-on: ubuntu-latest - if: "startsWith(github.ref, 'refs/tags/')" - needs: [ macos, windows, linux ] - steps: - - uses: actions/download-artifact@v3 - with: - name: wheels - - name: Publish to PyPI - uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - with: - command: upload - args: --skip-existing * \ No newline at end of file diff --git a/redisrs-py/.gitignore b/redisrs-py/.gitignore deleted file mode 100644 index af3ca5ef1c..0000000000 --- a/redisrs-py/.gitignore +++ /dev/null @@ -1,72 +0,0 @@ -/target - -# Byte-compiled / optimized / DLL files -__pycache__/ -.pytest_cache/ -*.py[cod] - -# C extensions -*.so - -# Distribution / packaging -.Python -.venv/ -env/ -bin/ -build/ -develop-eggs/ -dist/ -eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -include/ -man/ -venv/ -*.egg-info/ -.installed.cfg -*.egg - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt -pip-selfcheck.json - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.cache -nosetests.xml -coverage.xml - -# Translations -*.mo - -# Mr Developer -.mr.developer.cfg -.project -.pydevproject - -# Rope -.ropeproject - -# Django stuff: -*.log -*.pot - -.DS_Store - -# Sphinx documentation -docs/_build/ - -# PyCharm -.idea/ - -# VSCode -.vscode/ - -# Pyenv -.python-version \ No newline at end of file diff --git a/redisrs-py/Cargo.lock b/redisrs-py/Cargo.lock deleted file mode 100644 index aaf449fc34..0000000000 --- a/redisrs-py/Cargo.lock +++ /dev/null @@ -1,394 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bytes" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "combine" -version = "4.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" -dependencies = [ - "bytes", - "memchr", -] - -[[package]] -name = "form_urlencoded" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "idna" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "indoc" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2d6f23ffea9d7e76c53eee25dfb67bcd8fde7f1198b0855350698c9f07c780" - -[[package]] -name = "itoa" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" - -[[package]] -name = "libc" -version = "0.2.139" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" - -[[package]] -name = "lock_api" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "memchr" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg", -] - -[[package]] -name = "once_cell" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-sys", -] - -[[package]] -name = "percent-encoding" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" - -[[package]] -name = "proc-macro2" -version = "1.0.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "pyo3" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543" -dependencies = [ - "cfg-if", - "indoc", - "libc", - "memoffset", - "parking_lot", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", - "unindent", -] - -[[package]] -name = "pyo3-build-config" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8" -dependencies = [ - "once_cell", - "target-lexicon", -] - -[[package]] -name = "pyo3-ffi" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc" -dependencies = [ - "libc", - "pyo3-build-config", -] - -[[package]] -name = "pyo3-macros" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28" -dependencies = [ - "proc-macro2", - "pyo3-macros-backend", - "quote", - "syn", -] - -[[package]] -name = "pyo3-macros-backend" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "quote" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "redis" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1aada340fba5deba625c84d109d0a83cc3565452d38083417992a702c2428d" -dependencies = [ - "combine", - "itoa", - "percent-encoding", - "ryu", - "sha1_smol", - "url", -] - -[[package]] -name = "redisrs-py" -version = "0.1.0" -dependencies = [ - "pyo3", - "redis", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - -[[package]] -name = "ryu" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "sha1_smol" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" - -[[package]] -name = "smallvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" - -[[package]] -name = "syn" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "target-lexicon" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9410d0f6853b1d94f0e519fb95df60f29d2c1eff2d921ffdf01a4c8a3b54f12d" - -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" - -[[package]] -name = "unicode-bidi" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" - -[[package]] -name = "unicode-ident" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unindent" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" - -[[package]] -name = "url" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" diff --git a/redisrs-py/Cargo.toml b/redisrs-py/Cargo.toml deleted file mode 100644 index 80ae3facb6..0000000000 --- a/redisrs-py/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "redisrs-py" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[lib] -name = "redisrs_py" -crate-type = ["cdylib"] - -[dependencies] -pyo3 = { version = "0.17.3", features = ["extension-module"] } -redis = { version= "0.22.2"} diff --git a/redisrs-py/pyproject.toml b/redisrs-py/pyproject.toml deleted file mode 100644 index c8e7d8e26c..0000000000 --- a/redisrs-py/pyproject.toml +++ /dev/null @@ -1,14 +0,0 @@ -[build-system] -requires = ["maturin>=0.14,<0.15"] -build-backend = "maturin" - -[project] -name = "redisrs-py" -requires-python = ">=3.7" -classifiers = [ - "Programming Language :: Rust", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", -] - - diff --git a/redisrs-py/src/lib.rs b/redisrs-py/src/lib.rs deleted file mode 100644 index a6bbb4e789..0000000000 --- a/redisrs-py/src/lib.rs +++ /dev/null @@ -1,62 +0,0 @@ -extern crate redis; - -use pyo3::prelude::*; -use pyo3::{ types::{PyTuple, PyBytes, PyByteArray, PyString, PyInt, PyLong, PyFloat } }; - - -#[pyfunction] -fn pack_command<'a>(py: Python<'a>, items: &'a PyTuple) -> &'a PyBytes { - // Redis-py:connection.py:Encoder accepts only: bytes, string, int or float. - // - let mut cmd = redis::Cmd::new().to_owned(); - for item in items { - if item.is_instance_of::().unwrap() || item.is_instance_of::().unwrap() { - let bytes: &[u8] = item.extract().unwrap(); - cmd.arg(bytes); - } else if item.is_instance_of::().unwrap() { - let str_item : String = item.extract().unwrap(); - cmd.arg(str_item); - } else if item.is_instance_of::().unwrap() || item.is_instance_of::().unwrap() { - let num_item : i64 = item.extract().unwrap(); - cmd.arg(num_item); - } else if item.is_instance_of::().unwrap() { - let float_item : f64 = item.extract().unwrap(); - cmd.arg(float_item); - } else { - println!("not yet"); - } - } - - PyBytes::new(py, &cmd.get_packed_command()[..]) -} - - -#[pyfunction] -fn pack_bytes<'a>(py: Python<'a>, bytes_cmd: &'a [u8]) -> &'a PyBytes { - - let mut cmd = redis::Cmd::new().to_owned(); - let mut start = 0; - for (i, &item) in bytes_cmd.iter().enumerate() { - if item == b' ' { - if i > start { - cmd.arg(&bytes_cmd[start..i]); - } - start = i + 1; - } - } - - if start < bytes_cmd.len() { - cmd.arg(&bytes_cmd[start..bytes_cmd.len()]); - } - - PyBytes::new(py, &cmd.get_packed_command()[..]) -} - - -/// A Python module implemented in Rust. -#[pymodule] -fn redisrs_py(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(pack_command, m)?)?; - m.add_function(wrap_pyfunction!(pack_bytes, m)?)?; - Ok(()) -} \ No newline at end of file diff --git a/redisrs-py/test/bench.py b/redisrs-py/test/bench.py deleted file mode 100644 index a4f4ee1573..0000000000 --- a/redisrs-py/test/bench.py +++ /dev/null @@ -1,231 +0,0 @@ -from random import choice -from string import ascii_uppercase -import timeit -import redisrs_py -import hiredis - -############################################################### -# COPY-PASTE -############################################################### - -encoding="utf-8" -encoding_errors="strict" -decode_responses=False - -#pure copy-paste from connetion.py - -class Encoder: - "Encode strings to bytes-like and decode bytes-like to strings" - - def __init__(self, encoding, encoding_errors, decode_responses): - self.encoding = encoding - self.encoding_errors = encoding_errors - self.decode_responses = decode_responses - - def encode(self, value): - "Return a bytestring or bytes-like representation of the value" - if isinstance(value, (bytes, memoryview)): - return value - elif isinstance(value, bool): - # special case bool since it is a subclass of int - raise Exception( - "Invalid input of type: 'bool'. Convert to a " - "bytes, string, int or float first." - ) - elif isinstance(value, (int, float)): - value = repr(value).encode() - elif not isinstance(value, str): - # a value we don't know how to deal with. throw an error - typename = type(value).__name__ - raise Exception( - f"Invalid input of type: '{typename}'. {value}" - f"Convert to a bytes, string, int or float first." - ) - if isinstance(value, str): - value = value.encode(self.encoding, self.encoding_errors) - return value - - def decode(self, value, force=False): - "Return a unicode string from the bytes-like representation" - if self.decode_responses or force: - if isinstance(value, memoryview): - value = value.tobytes() - if isinstance(value, bytes): - value = value.decode(self.encoding, self.encoding_errors) - return value - -SYM_STAR = b'*' -SYM_DOLLAR = b'$' -SYM_CRLF = b'\r\n' -SYM_EMPTY = b'' - -encoder = Encoder(encoding, encoding_errors, decode_responses) - -#almost pure copy-paste from connetion.py: got rid of self.encoder and self.buffer_cutoff - -def pack_command(*args): - """Pack a series of arguments into the Redis protocol""" - output = [] - # the client might have included 1 or more literal arguments in - # the command name, e.g., 'CONFIG GET'. The Redis server expects these - # arguments to be sent separately, so split the first argument - # manually. These arguments should be bytestrings so that they are - # not encoded. - if isinstance(args[0], str): - args = tuple(args[0].encode().split()) + args[1:] - elif b" " in args[0]: - args = tuple(args[0].split()) + args[1:] - buff = SYM_EMPTY.join((SYM_STAR, str(len(args)).encode(), SYM_CRLF)) - - buffer_cutoff = 6000 - global encoder - for arg in map(encoder.encode, args): - # to avoid large string mallocs, chunk the command into the - # output list if we're sending large values or memoryviews - arg_length = len(arg) - if ( - len(buff) > buffer_cutoff - or arg_length > buffer_cutoff - or isinstance(arg, memoryview) - ): - buff = SYM_EMPTY.join( - (buff, SYM_DOLLAR, str(arg_length).encode(), SYM_CRLF) - ) - output.append(buff) - output.append(arg) - buff = SYM_CRLF - else: - buff = SYM_EMPTY.join( - ( - buff, - SYM_DOLLAR, - str(arg_length).encode(), - SYM_CRLF, - arg, - SYM_CRLF, - ) - ) - output.append(buff) - return output - - -############################################################### -# WRAPPER FUNCTIONS -# pack_command_* - encoding on module extention side -# pack_bytes_* encoding on python side -############################################################### - - -def pack_bytes_hiredis(*args): - """Pack a series of arguments into the Redis protocol""" - output = [] - if isinstance(args[0], str): - args = tuple(args[0].encode().split()) + args[1:] - elif b" " in args[0]: - args = tuple(args[0].split()) + args[1:] - - global encoder - cmd = b' '.join(map(encoder.encode, args)) - - output.append(hiredis.pack_bytes(cmd)) - return output - - -def pack_command_hiredis(*args): - """Pack a series of arguments into the Redis protocol""" - output = [] - if isinstance(args[0], str): - args = tuple(args[0].encode().split()) + args[1:] - elif b" " in args[0]: - args = tuple(args[0].split()) + args[1:] - output.append(hiredis.pack_command(args)) - return output - -#Rust -def pack_bytes_rust(*args): - """Pack a series of arguments into the Redis protocol""" - output = [] - if isinstance(args[0], str): - args = tuple(args[0].encode().split()) + args[1:] - elif b" " in args[0]: - args = tuple(args[0].split()) + args[1:] - - global encoder - cmd = b' '.join(map(encoder.encode, args)) - - output.append(redisrs_py.pack_bytes(cmd)) - return output - - -def pack_command_rust(*args): - """Pack a series of arguments into the Redis protocol""" - output = [] - if isinstance(args[0], str): - args = tuple(args[0].encode().split()) + args[1:] - elif b" " in args[0]: - args = tuple(args[0].split()) + args[1:] - output.append(redisrs_py.pack_command(args)) - return output - - -############################################################### -# BENCHMARKING -############################################################### - - -TOTAL_CMDS = 100000 - -def get_entry(number_of_fields): - fields = (b'HSET', f'foo42') - for i in range(number_of_fields): - fields += (f"field{i}",) - fields += (''.join(choice(ascii_uppercase) for _ in range(10)),) - return fields - - -def get_input(number_of_fields, total=TOTAL_CMDS): - entry = get_entry(number_of_fields) - input = [] - for _ in range(total): - input.append(entry) - return input - - -input_0 = [] -for i in range(TOTAL_CMDS): - input_0.append( - (b'HSET', f'foo{i}', 42, 'zoo', 'a', 67, 43, 3.56, 'foo1', 'qwertyuiop', 'foo2', 3.1412345678901, 52, 'zxcvbnmmpei', 53, 'dhjihoihpouihj', 'kjl;kjhj;lkjlk;', '567890798783', 'kjkjkjkjk', 79878933334890808709890.0) - ) -input_17 = get_input(17) -input_50 = get_input(50) -input_100 = get_input(100) - - -ITERATIONS = 5 - - -def bench(input_name, description): - print(description) - res = timeit.timeit(f'[pack_command(*cmd) for cmd in {input_name}]', globals=globals(), number=ITERATIONS) - print(f'BASELINE:redis.py: {res}') - - print("IMPLEMENTATIONS:") - res = timeit.timeit(f'[pack_bytes_hiredis(*cmd) for cmd in {input_name}]', globals=globals(), number=ITERATIONS) - print(f'hiredis.pack_bytes: {res}') - - res = timeit.timeit(f'[pack_bytes_rust(*cmd) for cmd in {input_name}]', globals=globals(), number=ITERATIONS) - print(f'redisrs-py.pack_bytes: {res}') - - if input_name == 'input_0': - print('hiredis.pack_command - not supported (yet?!)') - else: - res = timeit.timeit(f'[pack_command_hiredis(*cmd) for cmd in {input_name}]', globals=globals(), number=ITERATIONS) - print(f'hiredis.pack_command(partial encoding in hiredis-C): {res}') - - res = timeit.timeit(f'[pack_command_rust(*cmd) for cmd in {input_name}]', globals=globals(), number=ITERATIONS) - print(f'redisrs_py.pack_command(encoding in Rust): {res}') - -bench("input_0", "========= 10 fields of various types: str, int, float, bytes") -bench("input_17", "\n========= 17 10-bytes fileds per entry") -bench("input_50", "\n========= 50 10-bytes fileds per entry") -bench("input_100", "\n========= 100 10-bytes fileds per entry") diff --git a/redisrs-py/test/test.py b/redisrs-py/test/test.py deleted file mode 100644 index 7c59755189..0000000000 --- a/redisrs-py/test/test.py +++ /dev/null @@ -1,43 +0,0 @@ -import redis - -import redisrs_py -import hiredis -r = redis.from_url('redis://localhost:12005') -print(r.ping()) - - -# print(r.hset("foo1", "42", 4.09,)) -# print(r.hset("foo2", items=[42, "zoo", 'a', 67, 42, 3.56])) -print(r.hset("foo3", mapping={42: "zoo", 'a': 67, 43: 3.56})) - -# print(redisrs_py.pack_command(('ping',))) - -# print(redisrs_py.pack_command((b'HSET', 'foo3', 42, 'zoo', 'a', 67, 43, 3.56))) - -# args = (b'HSET', 'foo3', 42, 'zoo', 'a', 67, 43, 3.56) -# #args = ['bbb'] -# print(hiredis.pack_command(args)) -# bytes_cmd = b'hset key1 foo bar zoo 4567890 baz 9009870987098 zoo 3.14569808' -# print(hiredis.pack_bytes(bytes_cmd)) - -import unittest - -class TestPackBytes(unittest.TestCase): - def test_basic(self): - cmd = b'HSET foo3 42 zoo a 67 43 3.56' - expected_packed = b'*8\r\n$4\r\nHSET\r\n$4\r\nfoo3\r\n$2\r\n42\r\n$3\r\nzoo\r\n$1\r\na\r\n$2\r\n67\r\n$2\r\n43\r\n$4\r\n3.56\r\n' - packed_cmd = hiredis.pack_bytes(cmd) - self.assertEqual(packed_cmd, expected_packed) - packed_cmd = redisrs_py.pack_bytes(cmd) - self.assertEqual(packed_cmd, expected_packed) - - def test_multiple_spaces(self): - cmd = b' HSET foo3 42 zoo a 67 43 3.56 ' - expected_packed = b'*8\r\n$4\r\nHSET\r\n$4\r\nfoo3\r\n$2\r\n42\r\n$3\r\nzoo\r\n$1\r\na\r\n$2\r\n67\r\n$2\r\n43\r\n$4\r\n3.56\r\n' - packed_cmd = hiredis.pack_bytes(cmd) - self.assertEqual(packed_cmd, expected_packed) - packed_cmd = redisrs_py.pack_bytes(cmd) - self.assertEqual(packed_cmd, expected_packed) - -if __name__ == '__main__': - unittest.main() From 5893cd06ba25ea9a13ea7589dbc8a6319e200366 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Wed, 18 Jan 2023 10:22:46 -0600 Subject: [PATCH 05/11] 1) Introduce HIREDIS_PACK_AVAILABLE environment variable. 2) Extract serialization functionality out of Connection class. --- redis/connection.py | 168 ++++++++++++++++++++--------------------- redis/utils.py | 9 +-- setup.py | 2 +- tests/test_encoding.py | 4 +- 4 files changed, 87 insertions(+), 96 deletions(-) diff --git a/redis/connection.py b/redis/connection.py index 3a7ed13c49..967a9051ed 100644 --- a/redis/connection.py +++ b/redis/connection.py @@ -9,7 +9,6 @@ from itertools import chain from queue import Empty, Full, LifoQueue from time import time -from packaging.version import Version from typing import Optional from urllib.parse import parse_qs, unquote, urlparse @@ -33,7 +32,8 @@ TimeoutError, ) from redis.retry import Retry -from redis.utils import CRYPTOGRAPHY_AVAILABLE, HIREDIS_AVAILABLE, REDISRS_PY_AVAILABLE, str_if_bytes + +from redis.utils import CRYPTOGRAPHY_AVAILABLE, HIREDIS_AVAILABLE, HIREDIS_PACK_AVAILABLE, str_if_bytes try: import ssl @@ -56,11 +56,6 @@ if HIREDIS_AVAILABLE: import hiredis - -if REDISRS_PY_AVAILABLE: - import redisrs_py - - SYM_STAR = b"*" SYM_DOLLAR = b"$" SYM_CRLF = b"\r\n" @@ -161,7 +156,7 @@ def parse_error(self, response): "Parse an error response" error_code = response.split(" ")[0] if error_code in self.EXCEPTION_CLASSES: - response = response[len(error_code) + 1 :] + response = response[len(error_code) + 1:] exception_class = self.EXCEPTION_CLASSES[error_code] if isinstance(exception_class, dict): exception_class = exception_class.get(response, ResponseError) @@ -504,27 +499,80 @@ def read_response(self, disable_decoding=False): raise response[0] return response + if HIREDIS_AVAILABLE: DefaultParser = HiredisParser else: DefaultParser = PythonParser -def pack_command_hiredis(*args): - """Pack a series of arguments into the Redis protocol""" - output = [] +class HiredisRespSerializer: + def pack(self, *args): + """Pack a series of arguments into the Redis protocol""" + output = [] + + if isinstance(args[0], str): + args = tuple(args[0].encode().split()) + args[1:] + elif b" " in args[0]: + args = tuple(args[0].split()) + args[1:] + try: + output.append(hiredis.pack_command(args)) + except TypeError as err: + _, value, traceback = sys.exc_info() + raise DataError(value).with_traceback(traceback) + + return output + - if isinstance(args[0], str): - args = tuple(args[0].encode().split()) + args[1:] - elif b" " in args[0]: - args = tuple(args[0].split()) + args[1:] - try: - output.append(hiredis.pack_command(args)) - except TypeError as err: - _, value, traceback = sys.exc_info() - raise DataError(value).with_traceback(traceback) +class PythonRespSerializer: + def __init__(self, buffer_cutoff, encode) -> None: + self._buffer_cutoff = buffer_cutoff + self.encode = encode - return output + def pack(self, *args): + """Pack a series of arguments into the Redis protocol""" + output = [] + # the client might have included 1 or more literal arguments in + # the command name, e.g., 'CONFIG GET'. The Redis server expects these + # arguments to be sent separately, so split the first argument + # manually. These arguments should be bytestrings so that they are + # not encoded. + if isinstance(args[0], str): + args = tuple(args[0].encode().split()) + args[1:] + elif b" " in args[0]: + args = tuple(args[0].split()) + args[1:] + + buff = SYM_EMPTY.join((SYM_STAR, str(len(args)).encode(), SYM_CRLF)) + + buffer_cutoff = self._buffer_cutoff + for arg in map(self.encode, args): + # to avoid large string mallocs, chunk the command into the + # output list if we're sending large values or memoryviews + arg_length = len(arg) + if ( + len(buff) > buffer_cutoff + or arg_length > buffer_cutoff + or isinstance(arg, memoryview) + ): + buff = SYM_EMPTY.join( + (buff, SYM_DOLLAR, str(arg_length).encode(), SYM_CRLF) + ) + output.append(buff) + output.append(arg) + buff = SYM_CRLF + else: + buff = SYM_EMPTY.join( + ( + buff, + SYM_DOLLAR, + str(arg_length).encode(), + SYM_CRLF, + arg, + SYM_CRLF, + ) + ) + output.append(buff) + return output class Connection: @@ -554,7 +602,7 @@ def __init__( retry=None, redis_connect_func=None, credential_provider: Optional[CredentialProvider] = None, - pack_command=None, + command_packer=None, ): """ Initialize a new Connection. @@ -609,7 +657,7 @@ def __init__( self.set_parser(parser_class) self._connect_callbacks = [] self._buffer_cutoff = 6000 - self.pack_command = self._construct_pack_command(pack_command) + self._command_packer = self._construct_command_packer(command_packer) def __repr__(self): repr_args = ",".join([f"{k}={v}" for k, v in self.repr_pieces()]) @@ -626,14 +674,14 @@ def __del__(self): self.disconnect() except Exception: pass - - def _construct_pack_command(self, packer): + + def _construct_command_packer(self, packer): if packer is not None: return packer - elif HIREDIS_AVAILABLE and Version(hiredis.__version__) >= Version("2.1.2"): - return pack_command_hiredis + elif HIREDIS_PACK_AVAILABLE: + return HiredisRespSerializer() else: - return self._pack_command_python + return PythonRespSerializer(self._buffer_cutoff, self.encoder.encode) def register_connect_callback(self, callback): self._connect_callbacks.append(weakref.WeakMethod(callback)) @@ -855,7 +903,7 @@ def send_packed_command(self, command, check_health=True): def send_command(self, *args, **kwargs): """Pack and send a command to the Redis server""" self.send_packed_command( - self.pack_command(*args), check_health=kwargs.get("check_health", True) + self._command_packer.pack(*args), check_health=kwargs.get("check_health", True) ) def can_read(self, timeout=0): @@ -898,65 +946,9 @@ def read_response(self, disable_decoding=False): raise response return response - def _pack_bytes_hiredis(self, *args): - """Pack a series of arguments into the Redis protocol""" - output = [] - if isinstance(args[0], str): - args = tuple(args[0].encode().split()) + args[1:] - elif b" " in args[0]: - args = tuple(args[0].split()) + args[1:] - - global encoder - cmd = b' '.join(map(self.encoder.encode, args)) - - output.append(hiredis.pack_bytes(cmd)) - return output - - - def _pack_command_python(self, *args): + def pack_command(self, *args): """Pack a series of arguments into the Redis protocol""" - output = [] - # the client might have included 1 or more literal arguments in - # the command name, e.g., 'CONFIG GET'. The Redis server expects these - # arguments to be sent separately, so split the first argument - # manually. These arguments should be bytestrings so that they are - # not encoded. - if isinstance(args[0], str): - args = tuple(args[0].encode().split()) + args[1:] - elif b" " in args[0]: - args = tuple(args[0].split()) + args[1:] - - buff = SYM_EMPTY.join((SYM_STAR, str(len(args)).encode(), SYM_CRLF)) - - buffer_cutoff = self._buffer_cutoff - for arg in map(self.encoder.encode, args): - # to avoid large string mallocs, chunk the command into the - # output list if we're sending large values or memoryviews - arg_length = len(arg) - if ( - len(buff) > buffer_cutoff - or arg_length > buffer_cutoff - or isinstance(arg, memoryview) - ): - buff = SYM_EMPTY.join( - (buff, SYM_DOLLAR, str(arg_length).encode(), SYM_CRLF) - ) - output.append(buff) - output.append(arg) - buff = SYM_CRLF - else: - buff = SYM_EMPTY.join( - ( - buff, - SYM_DOLLAR, - str(arg_length).encode(), - SYM_CRLF, - arg, - SYM_CRLF, - ) - ) - output.append(buff) - return output + return self._command_packer.pack(*args) def pack_commands(self, commands): """Pack multiple commands into the Redis protocol""" @@ -966,7 +958,7 @@ def pack_commands(self, commands): buffer_cutoff = self._buffer_cutoff for cmd in commands: - for chunk in self.pack_command(*cmd): + for chunk in self._command_packer.pack(*cmd): chunklen = len(chunk) if ( buffer_length > buffer_cutoff diff --git a/redis/utils.py b/redis/utils.py index 48a642adbc..6600b09204 100644 --- a/redis/utils.py +++ b/redis/utils.py @@ -1,5 +1,6 @@ from contextlib import contextmanager from functools import wraps +from packaging.version import Version from typing import Any, Dict, Mapping, Union try: @@ -7,8 +8,10 @@ # Only support Hiredis >= 1.0: HIREDIS_AVAILABLE = not hiredis.__version__.startswith("0.") + HIREDIS_PACK_AVAILABLE = Version(hiredis.__version__) >= Version("2.1.2") except ImportError: HIREDIS_AVAILABLE = False + HIREDIS_PACK_AVAILABLE = False try: import cryptography # noqa @@ -17,12 +20,6 @@ except ImportError: CRYPTOGRAPHY_AVAILABLE = False -try: - import redisrs_py - REDISRS_PY_AVAILABLE = True -except ImportError: - REDISRS_PY_AVAILABLE = False -REDISRS_PY_AVAILABLE = False def from_url(url, **kwargs): """ diff --git a/setup.py b/setup.py index a702c9afb4..16a9156641 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ long_description_content_type="text/markdown", keywords=["Redis", "key-value store", "database"], license="MIT", - version="4.4.1", + version="4.4.2", packages=find_packages( include=[ "redis", diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 2867640742..066a2dca44 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -2,6 +2,7 @@ import redis from redis.connection import Connection +from redis.utils import HIREDIS_PACK_AVAILABLE from .conftest import _get_client @@ -74,7 +75,8 @@ def test_replace(self, request): r.set("a", b"foo\xff") assert r.get("a") == "foo\ufffd" - +@pytest.mark.skipif(HIREDIS_PACK_AVAILABLE, + reason="Packing via hiredis does not preserve memoriviews") class TestMemoryviewsAreNotPacked: def test_memoryviews_are_not_packed(self): c = Connection() From d7a9a46119cb2206adf783324b3abaf2aef49f21 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Mon, 23 Jan 2023 11:39:25 -0600 Subject: [PATCH 06/11] 1) Fix typo. 2) Add change log entry. 3) Revert the benchmark changes --- CHANGES | 1 + benchmarks/basic_operations.py | 11 +---------- tests/test_encoding.py | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/CHANGES b/CHANGES index fca8d3168e..b74516b007 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,4 @@ + * Use hiredis-py 2.2.0 pack command if available. * Make PythonParser resumable in case of error (#2510) * Add `timeout=None` in `SentinelConnectionManager.read_response` * Documentation fix: password protected socket connection (#2374) diff --git a/benchmarks/basic_operations.py b/benchmarks/basic_operations.py index 67b526ed7e..c9f5853652 100644 --- a/benchmarks/basic_operations.py +++ b/benchmarks/basic_operations.py @@ -173,22 +173,13 @@ def lpop(conn, num, pipeline_size, data_size): if pipeline_size > 1: conn.execute() -from random import choice -from string import ascii_uppercase - -def get_data(number_of_fields): - fields = {} - for i in range(number_of_fields): - fields[f"field{i}"] = ''.join(choice(ascii_uppercase) for _ in range(10)) - return fields @timer def hmset(conn, num, pipeline_size, data_size): if pipeline_size > 1: conn = conn.pipeline() - # set_data = {"str_value": "string", "int_value": 123456, "float_value": 123456.0} - set_data = get_data(17) #{"str_value": "string", "int_value": 123456, "float_value": 123456.0} + set_data = {"str_value": "string", "int_value": 123456, "float_value": 123456.0} for i in range(num): conn.hmset("hmset_key", set_data) if pipeline_size > 1 and i % pipeline_size == 0: diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 066a2dca44..39e78065e0 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -76,7 +76,7 @@ def test_replace(self, request): assert r.get("a") == "foo\ufffd" @pytest.mark.skipif(HIREDIS_PACK_AVAILABLE, - reason="Packing via hiredis does not preserve memoriviews") + reason="Packing via hiredis does not preserve memoryviews") class TestMemoryviewsAreNotPacked: def test_memoryviews_are_not_packed(self): c = Connection() From 737b2069e1c2eaa7ce3f8d33fff1c69e2f5c23f1 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Mon, 23 Jan 2023 17:43:35 -0600 Subject: [PATCH 07/11] Ditch the hiredis version check for pack_command. --- redis/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/redis/utils.py b/redis/utils.py index 6600b09204..dbb1d9b927 100644 --- a/redis/utils.py +++ b/redis/utils.py @@ -1,6 +1,5 @@ from contextlib import contextmanager from functools import wraps -from packaging.version import Version from typing import Any, Dict, Mapping, Union try: @@ -8,7 +7,7 @@ # Only support Hiredis >= 1.0: HIREDIS_AVAILABLE = not hiredis.__version__.startswith("0.") - HIREDIS_PACK_AVAILABLE = Version(hiredis.__version__) >= Version("2.1.2") + HIREDIS_PACK_AVAILABLE = hasattr(hiredis, 'pack_command') except ImportError: HIREDIS_AVAILABLE = False HIREDIS_PACK_AVAILABLE = False From 5b2e85d534c057a274e4eb8d93e325bdf7bf5450 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Mon, 23 Jan 2023 18:42:42 -0600 Subject: [PATCH 08/11] Fix linter errors --- redis/connection.py | 9 +++++---- tests/test_encoding.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/redis/connection.py b/redis/connection.py index 967a9051ed..8a5f07fad7 100644 --- a/redis/connection.py +++ b/redis/connection.py @@ -32,8 +32,8 @@ TimeoutError, ) from redis.retry import Retry - -from redis.utils import CRYPTOGRAPHY_AVAILABLE, HIREDIS_AVAILABLE, HIREDIS_PACK_AVAILABLE, str_if_bytes +from redis.utils import CRYPTOGRAPHY_AVAILABLE, HIREDIS_AVAILABLE, \ + HIREDIS_PACK_AVAILABLE, str_if_bytes try: import ssl @@ -517,7 +517,7 @@ def pack(self, *args): args = tuple(args[0].split()) + args[1:] try: output.append(hiredis.pack_command(args)) - except TypeError as err: + except TypeError: _, value, traceback = sys.exc_info() raise DataError(value).with_traceback(traceback) @@ -903,7 +903,8 @@ def send_packed_command(self, command, check_health=True): def send_command(self, *args, **kwargs): """Pack and send a command to the Redis server""" self.send_packed_command( - self._command_packer.pack(*args), check_health=kwargs.get("check_health", True) + self._command_packer.pack(*args), + check_health=kwargs.get("check_health", True) ) def can_read(self, timeout=0): diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 39e78065e0..fc109eceef 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -75,6 +75,7 @@ def test_replace(self, request): r.set("a", b"foo\xff") assert r.get("a") == "foo\ufffd" + @pytest.mark.skipif(HIREDIS_PACK_AVAILABLE, reason="Packing via hiredis does not preserve memoryviews") class TestMemoryviewsAreNotPacked: From b4cb3eb3f397da4653d2532cab4107b64e7b2c0f Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Tue, 31 Jan 2023 22:45:58 -0600 Subject: [PATCH 09/11] Revert version changes --- CHANGES | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index b74516b007..f508bb9121 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,4 @@ - * Use hiredis-py 2.2.0 pack command if available. + * Use hiredis-py pack_command if available. * Make PythonParser resumable in case of error (#2510) * Add `timeout=None` in `SentinelConnectionManager.read_response` * Documentation fix: password protected socket connection (#2374) diff --git a/setup.py b/setup.py index 16a9156641..9cf0dcff26 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ long_description_content_type="text/markdown", keywords=["Redis", "key-value store", "database"], license="MIT", - version="4.4.2", + version="4.4.0", packages=find_packages( include=[ "redis", From 1687da5b8a4cf650f426b460fecb327a8cb72535 Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Thu, 2 Feb 2023 18:05:10 -0600 Subject: [PATCH 10/11] Fix linter issues --- redis/connection.py | 12 ++++++++---- redis/utils.py | 2 +- tests/test_encoding.py | 6 ++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/redis/connection.py b/redis/connection.py index 8a5f07fad7..a2e8051884 100644 --- a/redis/connection.py +++ b/redis/connection.py @@ -32,8 +32,12 @@ TimeoutError, ) from redis.retry import Retry -from redis.utils import CRYPTOGRAPHY_AVAILABLE, HIREDIS_AVAILABLE, \ - HIREDIS_PACK_AVAILABLE, str_if_bytes +from redis.utils import ( + CRYPTOGRAPHY_AVAILABLE, + HIREDIS_AVAILABLE, + HIREDIS_PACK_AVAILABLE, + str_if_bytes, +) try: import ssl @@ -156,7 +160,7 @@ def parse_error(self, response): "Parse an error response" error_code = response.split(" ")[0] if error_code in self.EXCEPTION_CLASSES: - response = response[len(error_code) + 1:] + response = response[len(error_code) + 1 :] exception_class = self.EXCEPTION_CLASSES[error_code] if isinstance(exception_class, dict): exception_class = exception_class.get(response, ResponseError) @@ -904,7 +908,7 @@ def send_command(self, *args, **kwargs): """Pack and send a command to the Redis server""" self.send_packed_command( self._command_packer.pack(*args), - check_health=kwargs.get("check_health", True) + check_health=kwargs.get("check_health", True), ) def can_read(self, timeout=0): diff --git a/redis/utils.py b/redis/utils.py index dbb1d9b927..d95e62c042 100644 --- a/redis/utils.py +++ b/redis/utils.py @@ -7,7 +7,7 @@ # Only support Hiredis >= 1.0: HIREDIS_AVAILABLE = not hiredis.__version__.startswith("0.") - HIREDIS_PACK_AVAILABLE = hasattr(hiredis, 'pack_command') + HIREDIS_PACK_AVAILABLE = hasattr(hiredis, "pack_command") except ImportError: HIREDIS_AVAILABLE = False HIREDIS_PACK_AVAILABLE = False diff --git a/tests/test_encoding.py b/tests/test_encoding.py index fc109eceef..cb9c4e20be 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -76,8 +76,10 @@ def test_replace(self, request): assert r.get("a") == "foo\ufffd" -@pytest.mark.skipif(HIREDIS_PACK_AVAILABLE, - reason="Packing via hiredis does not preserve memoryviews") +@pytest.mark.skipif( + HIREDIS_PACK_AVAILABLE, + reason="Packing via hiredis does not preserve memoryviews", +) class TestMemoryviewsAreNotPacked: def test_memoryviews_are_not_packed(self): c = Connection() From 3c3d706ba3b70456985e695ecd5dacf843be718c Mon Sep 17 00:00:00 2001 From: Sergey Prokazov Date: Fri, 3 Feb 2023 00:21:15 -0600 Subject: [PATCH 11/11] Looks like the current redis-py version is 4.4.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9cf0dcff26..a702c9afb4 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ long_description_content_type="text/markdown", keywords=["Redis", "key-value store", "database"], license="MIT", - version="4.4.0", + version="4.4.1", packages=find_packages( include=[ "redis",