From c9656eef6eb05d2e118f0b20346eb78551778774 Mon Sep 17 00:00:00 2001 From: Benedikt Bartscher Date: Sun, 24 Nov 2024 20:55:27 +0100 Subject: [PATCH 1/7] benchmark compression and pickle protocols --- benchmarks/benchmark_pickle.py | 155 ++++++++++++++++++++++ benchmarks/conftest.py | 44 +++++++ poetry.lock | 227 ++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 4 files changed, 425 insertions(+), 2 deletions(-) create mode 100644 benchmarks/benchmark_pickle.py diff --git a/benchmarks/benchmark_pickle.py b/benchmarks/benchmark_pickle.py new file mode 100644 index 000000000..7773d3b2e --- /dev/null +++ b/benchmarks/benchmark_pickle.py @@ -0,0 +1,155 @@ +"""Benchmarks for pickling and unpickling states.""" + +import logging +import pickle +import time +import uuid +from typing import Tuple + +import pytest +from pytest_benchmark.fixture import BenchmarkFixture +from redis import Redis + +from reflex.state import State +from reflex.utils.prerequisites import get_redis_sync + +log = logging.getLogger(__name__) + + +SLOW_REDIS_MAP: dict[bytes, bytes] = {} + + +class SlowRedis: + """Simulate a slow Redis client which uses a global dict and sleeps based on size.""" + + def __init__(self): + """Initialize the slow Redis client.""" + pass + + def set(self, key: bytes, value: bytes) -> None: + """Set a key-value pair in the slow Redis client. + + Args: + key: The key. + value: The value. + """ + SLOW_REDIS_MAP[key] = value + size = len(value) + sleep_time = (size / 1e6) + 0.05 + time.sleep(sleep_time) + + def get(self, key: bytes) -> bytes: + """Get a value from the slow Redis client. + + Args: + key: The key. + + Returns: + The value. + """ + value = SLOW_REDIS_MAP[key] + size = len(value) + sleep_time = (size / 1e6) + 0.05 + time.sleep(sleep_time) + return value + + +@pytest.mark.parametrize( + "protocol", + argvalues=[ + pickle.DEFAULT_PROTOCOL, + pickle.HIGHEST_PROTOCOL, + ], + ids=[ + "pickle_default", + "pickle_highest", + ], +) +@pytest.mark.parametrize( + "redis", + [ + Redis, + SlowRedis, + None, + ], + ids=[ + "redis", + "slow_redis", + "no_redis", + ], +) +@pytest.mark.parametrize( + "should_compress", [True, False], ids=["compress", "no_compress"] +) +@pytest.mark.benchmark(disable_gc=True) +def test_pickle( + request: pytest.FixtureRequest, + benchmark: BenchmarkFixture, + big_state: State, + big_state_size: Tuple[int, str], + protocol: int, + redis: Redis | SlowRedis | None, + should_compress: bool, +) -> None: + """Benchmark pickling a big state. + + Args: + request: The pytest fixture request object. + benchmark: The benchmark fixture. + big_state: The big state fixture. + big_state_size: The big state size fixture. + protocol: The pickle protocol. + redis: Whether to use Redis. + should_compress: Whether to compress the pickled state. + """ + if should_compress: + try: + from blosc2 import compress, decompress + except ImportError: + pytest.skip("Blosc is not available.") + + def dump(obj: State) -> bytes: + return compress(pickle.dumps(obj, protocol=protocol)) + + def load(data: bytes) -> State: + return pickle.loads(decompress(data)) + + else: + + def dump(obj: State) -> bytes: + return pickle.dumps(obj, protocol=protocol) + + def load(data: bytes) -> State: + return pickle.loads(data) + + if redis: + if redis == Redis: + redis_client = get_redis_sync() + if redis_client is None: + pytest.skip("Redis is not available.") + else: + redis_client = SlowRedis() + + key = str(uuid.uuid4()).encode() + + def run(obj: State) -> None: + _ = redis_client.set(key, dump(obj)) + _ = load(redis_client.get(key)) # type: ignore + + else: + + def run(obj: State) -> None: + _ = load(dump(obj)) + + # calculate size before benchmark to not affect it + out = dump(big_state) + size = len(out) + log.warning(f"{protocol=}, {redis=}, {should_compress=}, {size=}") + + benchmark.extra_info["size"] = size + benchmark.extra_info["redis"] = redis + benchmark.extra_info["pickle_protocol"] = protocol + redis_group = redis.__name__ if redis else "no_redis" # type: ignore + benchmark.group = f"{redis_group}_{big_state_size[1]}" + + _ = benchmark(run, big_state) diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py index 11e3bd4d7..9ef7d501c 100644 --- a/benchmarks/conftest.py +++ b/benchmarks/conftest.py @@ -1,7 +1,11 @@ """Shared conftest for all benchmark tests.""" +from typing import Tuple + +import pandas as pd import pytest +from reflex.state import State from reflex.testing import AppHarness, AppHarnessProd @@ -18,3 +22,43 @@ def app_harness_env(request): The AppHarness class to use for the test. """ return request.param + + +@pytest.fixture(params=[(10, "SmallState"), (2000, "BigState")], ids=["small", "big"]) +def big_state_size(request: pytest.FixtureRequest) -> int: + """The size of the DataFrame. + + Args: + request: The pytest fixture request object. + + Returns: + The size of the BigState + """ + return request.param + + +@pytest.fixture +def big_state(big_state_size: Tuple[int, str]) -> State: + """A big state with a dictionary and a list of DataFrames. + + Args: + big_state_size: The size of the big state. + + Returns: + A big state instance. + """ + size, _ = big_state_size + + class BigState(State): + """A big state.""" + + d: dict[str, int] + d_repeated: dict[str, int] + df: list[pd.DataFrame] + + d = {str(i): i for i in range(size)} + d_repeated = {str(i): i for i in range(size)} + df = [pd.DataFrame({"a": [i]}) for i in range(size)] + + state = BigState(d=d, df=df, d_repeated=d_repeated) + return state diff --git a/poetry.lock b/poetry.lock index cc778d19b..f5e0bde75 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "alembic" @@ -119,6 +119,44 @@ files = [ {file = "bidict-0.23.1.tar.gz", hash = "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71"}, ] +[[package]] +name = "blosc2" +version = "3.0.0" +description = "A high-performance compressed ndarray library with a flexible compute engine" +optional = false +python-versions = ">=3.11" +files = [ + {file = "blosc2-3.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:027158b6c6de85a5c7d2e60066819df58efb6a0d0108b613db2118453d0e3a8f"}, + {file = "blosc2-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c3514427c04f8f3180b07807eab736d694db21dea518ffde6e36359657deb922"}, + {file = "blosc2-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3213763e8ed8f0dd81393e5ecf70fa6a498160aaec6611eb7d37d462b2814824"}, + {file = "blosc2-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d2a11c4241e4f718931d0ed7ce2519badd9a326c1cf631ac1355b6921d732f9"}, + {file = "blosc2-3.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:5a23b23dab715a3918564c70ffc2f75d0c103f80be41718959fa55bc805f66b6"}, + {file = "blosc2-3.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ff945f653c111aa02008bc530b48565499a806899f3ff5ff6ea7f9e73e9a660b"}, + {file = "blosc2-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a8ba5eb416d125f4dac9ae89fe1915b7ef0d84ee876e42815cb3093f998cb4e5"}, + {file = "blosc2-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecba37aa87b2a2974a8bc1316d7c85e1d42e2f4e0aef38267ae1e3d55d8cdf30"}, + {file = "blosc2-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5567bc9d966f7215a81f4e2ba24628d5d9e66c87b79a3d011ffdd2ffe40b74ee"}, + {file = "blosc2-3.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3a209aec1f365063f0a887a3fb6f37690da11d0f35411cc538bab25fd3d71498"}, + {file = "blosc2-3.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b7e5220b83cf336391f883eba745206450efae766ed40c237b9ea41877306ae0"}, + {file = "blosc2-3.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8da04add23d4229bbfdd2a3be3baaa0c8e3035eec02ff484a9ca97c76aff2ed7"}, + {file = "blosc2-3.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d949e86522a6001ed8e8bca0a29c8638c5b3a0900376a48a9536864cccb4b89"}, + {file = "blosc2-3.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5e7cdad6c2361d2e2ed2de33f18ef073a3d4dc7a9beb4d1db172005e09b229e"}, + {file = "blosc2-3.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:37de3eab82df2738b2d81fa2fbfa46efdb108b7b54b37f1255f358f90ab7212d"}, + {file = "blosc2-3.0.0.tar.gz", hash = "sha256:d8c03a09ed11b644b48bf050bd108972ec56ac9cbc3f2aedca077255ed81ac69"}, +] + +[package.dependencies] +httpx = "*" +msgpack = "*" +ndindex = "*" +numexpr = "*" +numpy = ">=1.25.0" +py-cpuinfo = "*" + +[package.extras] +dev = ["h5py", "hdf5plugin", "jupyterlab", "matplotlib", "pandas", "plotly", "pre-commit", "pyarrow", "ruff", "s3fs", "xarray", "zarr"] +doc = ["myst-parser", "nbsphinx", "numpydoc", "pydata-sphinx-theme", "sphinx", "sphinx-panels", "sphinx-paramlinks"] +test = ["psutil", "pytest", "torch"] + [[package]] name = "build" version = "1.2.2.post1" @@ -1117,6 +1155,142 @@ files = [ {file = "more_itertools-10.5.0-py3-none-any.whl", hash = "sha256:037b0d3203ce90cca8ab1defbbdac29d5f993fc20131f3664dc8d6acfa872aef"}, ] +[[package]] +name = "msgpack" +version = "1.1.0" +description = "MessagePack serializer" +optional = false +python-versions = ">=3.8" +files = [ + {file = "msgpack-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7ad442d527a7e358a469faf43fda45aaf4ac3249c8310a82f0ccff9164e5dccd"}, + {file = "msgpack-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:74bed8f63f8f14d75eec75cf3d04ad581da6b914001b474a5d3cd3372c8cc27d"}, + {file = "msgpack-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:914571a2a5b4e7606997e169f64ce53a8b1e06f2cf2c3a7273aa106236d43dd5"}, + {file = "msgpack-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c921af52214dcbb75e6bdf6a661b23c3e6417f00c603dd2070bccb5c3ef499f5"}, + {file = "msgpack-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8ce0b22b890be5d252de90d0e0d119f363012027cf256185fc3d474c44b1b9e"}, + {file = "msgpack-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:73322a6cc57fcee3c0c57c4463d828e9428275fb85a27aa2aa1a92fdc42afd7b"}, + {file = "msgpack-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e1f3c3d21f7cf67bcf2da8e494d30a75e4cf60041d98b3f79875afb5b96f3a3f"}, + {file = "msgpack-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:64fc9068d701233effd61b19efb1485587560b66fe57b3e50d29c5d78e7fef68"}, + {file = "msgpack-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:42f754515e0f683f9c79210a5d1cad631ec3d06cea5172214d2176a42e67e19b"}, + {file = "msgpack-1.1.0-cp310-cp310-win32.whl", hash = "sha256:3df7e6b05571b3814361e8464f9304c42d2196808e0119f55d0d3e62cd5ea044"}, + {file = "msgpack-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:685ec345eefc757a7c8af44a3032734a739f8c45d1b0ac45efc5d8977aa4720f"}, + {file = "msgpack-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d364a55082fb2a7416f6c63ae383fbd903adb5a6cf78c5b96cc6316dc1cedc7"}, + {file = "msgpack-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79ec007767b9b56860e0372085f8504db5d06bd6a327a335449508bbee9648fa"}, + {file = "msgpack-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6ad622bf7756d5a497d5b6836e7fc3752e2dd6f4c648e24b1803f6048596f701"}, + {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e59bca908d9ca0de3dc8684f21ebf9a690fe47b6be93236eb40b99af28b6ea6"}, + {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e1da8f11a3dd397f0a32c76165cf0c4eb95b31013a94f6ecc0b280c05c91b59"}, + {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:452aff037287acb1d70a804ffd022b21fa2bb7c46bee884dbc864cc9024128a0"}, + {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8da4bf6d54ceed70e8861f833f83ce0814a2b72102e890cbdfe4b34764cdd66e"}, + {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:41c991beebf175faf352fb940bf2af9ad1fb77fd25f38d9142053914947cdbf6"}, + {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a52a1f3a5af7ba1c9ace055b659189f6c669cf3657095b50f9602af3a3ba0fe5"}, + {file = "msgpack-1.1.0-cp311-cp311-win32.whl", hash = "sha256:58638690ebd0a06427c5fe1a227bb6b8b9fdc2bd07701bec13c2335c82131a88"}, + {file = "msgpack-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd2906780f25c8ed5d7b323379f6138524ba793428db5d0e9d226d3fa6aa1788"}, + {file = "msgpack-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:d46cf9e3705ea9485687aa4001a76e44748b609d260af21c4ceea7f2212a501d"}, + {file = "msgpack-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5dbad74103df937e1325cc4bfeaf57713be0b4f15e1c2da43ccdd836393e2ea2"}, + {file = "msgpack-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58dfc47f8b102da61e8949708b3eafc3504509a5728f8b4ddef84bd9e16ad420"}, + {file = "msgpack-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4676e5be1b472909b2ee6356ff425ebedf5142427842aa06b4dfd5117d1ca8a2"}, + {file = "msgpack-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17fb65dd0bec285907f68b15734a993ad3fc94332b5bb21b0435846228de1f39"}, + {file = "msgpack-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a51abd48c6d8ac89e0cfd4fe177c61481aca2d5e7ba42044fd218cfd8ea9899f"}, + {file = "msgpack-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2137773500afa5494a61b1208619e3871f75f27b03bcfca7b3a7023284140247"}, + {file = "msgpack-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:398b713459fea610861c8a7b62a6fec1882759f308ae0795b5413ff6a160cf3c"}, + {file = "msgpack-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:06f5fd2f6bb2a7914922d935d3b8bb4a7fff3a9a91cfce6d06c13bc42bec975b"}, + {file = "msgpack-1.1.0-cp312-cp312-win32.whl", hash = "sha256:ad33e8400e4ec17ba782f7b9cf868977d867ed784a1f5f2ab46e7ba53b6e1e1b"}, + {file = "msgpack-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:115a7af8ee9e8cddc10f87636767857e7e3717b7a2e97379dc2054712693e90f"}, + {file = "msgpack-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:071603e2f0771c45ad9bc65719291c568d4edf120b44eb36324dcb02a13bfddf"}, + {file = "msgpack-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0f92a83b84e7c0749e3f12821949d79485971f087604178026085f60ce109330"}, + {file = "msgpack-1.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a1964df7b81285d00a84da4e70cb1383f2e665e0f1f2a7027e683956d04b734"}, + {file = "msgpack-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59caf6a4ed0d164055ccff8fe31eddc0ebc07cf7326a2aaa0dbf7a4001cd823e"}, + {file = "msgpack-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0907e1a7119b337971a689153665764adc34e89175f9a34793307d9def08e6ca"}, + {file = "msgpack-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65553c9b6da8166e819a6aa90ad15288599b340f91d18f60b2061f402b9a4915"}, + {file = "msgpack-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7a946a8992941fea80ed4beae6bff74ffd7ee129a90b4dd5cf9c476a30e9708d"}, + {file = "msgpack-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4b51405e36e075193bc051315dbf29168d6141ae2500ba8cd80a522964e31434"}, + {file = "msgpack-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4c01941fd2ff87c2a934ee6055bda4ed353a7846b8d4f341c428109e9fcde8c"}, + {file = "msgpack-1.1.0-cp313-cp313-win32.whl", hash = "sha256:7c9a35ce2c2573bada929e0b7b3576de647b0defbd25f5139dcdaba0ae35a4cc"}, + {file = "msgpack-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:bce7d9e614a04d0883af0b3d4d501171fbfca038f12c77fa838d9f198147a23f"}, + {file = "msgpack-1.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c40ffa9a15d74e05ba1fe2681ea33b9caffd886675412612d93ab17b58ea2fec"}, + {file = "msgpack-1.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1ba6136e650898082d9d5a5217d5906d1e138024f836ff48691784bbe1adf96"}, + {file = "msgpack-1.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0856a2b7e8dcb874be44fea031d22e5b3a19121be92a1e098f46068a11b0870"}, + {file = "msgpack-1.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:471e27a5787a2e3f974ba023f9e265a8c7cfd373632247deb225617e3100a3c7"}, + {file = "msgpack-1.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:646afc8102935a388ffc3914b336d22d1c2d6209c773f3eb5dd4d6d3b6f8c1cb"}, + {file = "msgpack-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:13599f8829cfbe0158f6456374e9eea9f44eee08076291771d8ae93eda56607f"}, + {file = "msgpack-1.1.0-cp38-cp38-win32.whl", hash = "sha256:8a84efb768fb968381e525eeeb3d92857e4985aacc39f3c47ffd00eb4509315b"}, + {file = "msgpack-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:879a7b7b0ad82481c52d3c7eb99bf6f0645dbdec5134a4bddbd16f3506947feb"}, + {file = "msgpack-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:53258eeb7a80fc46f62fd59c876957a2d0e15e6449a9e71842b6d24419d88ca1"}, + {file = "msgpack-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7e7b853bbc44fb03fbdba34feb4bd414322180135e2cb5164f20ce1c9795ee48"}, + {file = "msgpack-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3e9b4936df53b970513eac1758f3882c88658a220b58dcc1e39606dccaaf01c"}, + {file = "msgpack-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46c34e99110762a76e3911fc923222472c9d681f1094096ac4102c18319e6468"}, + {file = "msgpack-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a706d1e74dd3dea05cb54580d9bd8b2880e9264856ce5068027eed09680aa74"}, + {file = "msgpack-1.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:534480ee5690ab3cbed89d4c8971a5c631b69a8c0883ecfea96c19118510c846"}, + {file = "msgpack-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8cf9e8c3a2153934a23ac160cc4cba0ec035f6867c8013cc6077a79823370346"}, + {file = "msgpack-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3180065ec2abbe13a4ad37688b61b99d7f9e012a535b930e0e683ad6bc30155b"}, + {file = "msgpack-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c5a91481a3cc573ac8c0d9aace09345d989dc4a0202b7fcb312c88c26d4e71a8"}, + {file = "msgpack-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f80bc7d47f76089633763f952e67f8214cb7b3ee6bfa489b3cb6a84cfac114cd"}, + {file = "msgpack-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:4d1b7ff2d6146e16e8bd665ac726a89c74163ef8cd39fa8c1087d4e52d3a2325"}, + {file = "msgpack-1.1.0.tar.gz", hash = "sha256:dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e"}, +] + +[[package]] +name = "ndindex" +version = "1.9.2" +description = "A Python library for manipulating indices of ndarrays." +optional = false +python-versions = ">=3.9" +files = [ + {file = "ndindex-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6c0bd9fb02b5ac6e3e8b70404930e1f6824ad565363bf8f0e817d6f6a3a47593"}, + {file = "ndindex-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3502df515c225e653b55fc08e714d4ae56306ad46454759d90156530cb4e2f40"}, + {file = "ndindex-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a36a48abc26dbf405aaa121ee3b4245390801a07856192f4f6c79d4db403520"}, + {file = "ndindex-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0adcfa8f213493399458184175e411ce8040790f5de00d5cbca6ec6a778bfa71"}, + {file = "ndindex-1.9.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5f53810a8eb73ad7054c1d628365a86c3f550a14e59ed7d9904ee7de21a3a432"}, + {file = "ndindex-1.9.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d227c1c8e3c83ded04499b9a43c58862aebc948f2fe61d4c8622f711a56032b6"}, + {file = "ndindex-1.9.2-cp310-cp310-win32.whl", hash = "sha256:2707453757dabd7efa6c8f2d8acda4cdc8e39108528479a8c8fcc9126a693024"}, + {file = "ndindex-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:42614f829e614808987dd68377cacda32634c2d8399b239a7a43f78bd8e3bdda"}, + {file = "ndindex-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1fa2e521a872870d55fa6fa85399f16c1c20bbe4e3e315bbfc80e3ea92561334"}, + {file = "ndindex-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6b0ef52d15fa8755d00a6868c799ff4227f1f453901a6f4de395586f9a435b7a"}, + {file = "ndindex-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f647eda61cae68a260017118ad8daac4d580ad221ff922bbaa1526e30e350feb"}, + {file = "ndindex-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:384520b4d9f52cb2fd1d324e6f278ec422b2cc2885e95a00587394bf6f56a798"}, + {file = "ndindex-1.9.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e2dd75a6e25269b66607f1722acd72e8f086837b2c58275d31b3bdfdf8a095bf"}, + {file = "ndindex-1.9.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8634be005b18034846163067bce78a0209fab65e4bc77e0bc333aa160ef12b7a"}, + {file = "ndindex-1.9.2-cp311-cp311-win32.whl", hash = "sha256:89172e90e56a409197cbbe12a49aa16c83879274ca4f61fd8a03b30c6c90e3ca"}, + {file = "ndindex-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:d23f07831d28bb3c04c234936b6038078cd7c0c4966d2e2e37738edad6435f9f"}, + {file = "ndindex-1.9.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:466d2e30a3c2afac6dac64e5ada19db30d23164befa7f69d29f209fb512b3d2f"}, + {file = "ndindex-1.9.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3e87eefa75af0f974cf2c5af14a488ee97dfdc7fb6da67f19f9dc600da5ae041"}, + {file = "ndindex-1.9.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9d98a41ff276fc623f3e068d40381ee06289644b530c3535bc00a8cbc5526c6"}, + {file = "ndindex-1.9.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05d42c5cd86d923f1606c3a9defbaeb3ece8f7b444f95a46ec6f1fb511e971f7"}, + {file = "ndindex-1.9.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:959f8babfc3055933079296a33449e02c24074027c39ce974cf73757c7d5ea21"}, + {file = "ndindex-1.9.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d68d8285f3ab8a78b0db990fb25eddc637df4b00467fbf36a4656c7ee46ddc5d"}, + {file = "ndindex-1.9.2-cp312-cp312-win32.whl", hash = "sha256:c87aa8218b6eaaa9eacb2f68f1ce71be0e368280ef926d0ed9ad71d2fbe24fe6"}, + {file = "ndindex-1.9.2-cp312-cp312-win_amd64.whl", hash = "sha256:d15f3a8566910ec25898e3d77b3b7c258b37f84a235d49cb17dfddc9036127b4"}, + {file = "ndindex-1.9.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a33452b4a7b8510f809f8b59dbac5b1d7b3d8a68c9e42dee2c28c5131bbbfbc6"}, + {file = "ndindex-1.9.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19d70a304f942c0aee89418d9c487de9c2dcfcda9073976a590c7fed587bd674"}, + {file = "ndindex-1.9.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0bba0f387d1d8204378e82b886c19f46ae39347ee7113c9028317270c0060be"}, + {file = "ndindex-1.9.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1876dcd82d6d1cbc9c2e6819bc7a5ca3686a5430f0a07520b94f78ff78097d2d"}, + {file = "ndindex-1.9.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:3c564768ffa7228d1768eb6df1a77d03d39efc9c98746d8c1b511ffcc23d5f9f"}, + {file = "ndindex-1.9.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ddbdfee4560c3f7823de88257680c8fd6d0220a7b23bfd27b0f3fc7afa27bee1"}, + {file = "ndindex-1.9.2-cp313-cp313-win32.whl", hash = "sha256:6274886f1348128fc4e10aef925272f904ac467175af52338d56f1cb763caf1a"}, + {file = "ndindex-1.9.2-cp313-cp313-win_amd64.whl", hash = "sha256:98be658c00ec0827e398b0fe5c13d207ff70b027187d728cb22155cce3bf6fbe"}, + {file = "ndindex-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:35543c2638beeb1119c0842f3cdc394b9a242d312f7a5ec9d967486d66c3094f"}, + {file = "ndindex-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a2d9c237269fffdc02b810127d3cacaf82ac7cbee24ac9743e80859e789448b8"}, + {file = "ndindex-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d64688b27f89d529dfabeae5f1216e0673652b21143e802cf427ded8a4954ecd"}, + {file = "ndindex-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d3e507c079dfe12cf8144de0cfd9e670d0c234a3200d578556965723f0387fb"}, + {file = "ndindex-1.9.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:7daafad7af277e92d155178f625a3c951ed2dfb34878f1b7e6e952246eca0314"}, + {file = "ndindex-1.9.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4110c98e77cd529ec1ea8065bf6ce96a5c613ac47ad17e6f97943a80409de721"}, + {file = "ndindex-1.9.2-cp39-cp39-win32.whl", hash = "sha256:9c92921b1453b050b7531f55c568e64d93d6943e16bd9249a896258a6d4489bb"}, + {file = "ndindex-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:cd1b3d39cb0d3ef6606d7450b79729e1934bf88fd1a20e123d164bae8e74d01b"}, + {file = "ndindex-1.9.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:f32019691f52314d6b7ae0168f6c148f94fecda92cd84cc0fd94331982a55f37"}, + {file = "ndindex-1.9.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:c6aa27f552a810e60e207fca372db94147811ca9228a728d3529a908197d35ce"}, + {file = "ndindex-1.9.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:67e02762cd342d0e722f47a26ba546d5709e4b17205bda504228ed455f31b690"}, + {file = "ndindex-1.9.2-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4750ded2e1d2941cda4604aade162452ed5eccffd6737e072359a5a633acd5eb"}, + {file = "ndindex-1.9.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e8c33c2feedefde0789505ffb4b1324c6cb1adf553923f6bff7084910d37740b"}, + {file = "ndindex-1.9.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cd82f7b46f0df51631273f72e1c5c8a03c30ce0e0c6da6f8b0ce70235cda31bb"}, + {file = "ndindex-1.9.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:9dff41fcb353fdb7589f6c7c69fd34748254cddd9c35824d3c81447de8906f5d"}, + {file = "ndindex-1.9.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4cabded9f47e988b9564df7f37662a53dae5ac8e3419e784c3db07dc230d0480"}, + {file = "ndindex-1.9.2-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04ce2e8712a4908641a1e23bba692fc88ba147e5926b6346dd526605de7a7980"}, + {file = "ndindex-1.9.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7ca14a442375f911dd930b7a246b24d6dfc64543ca202d20d65e5b1b96e9dbbf"}, + {file = "ndindex-1.9.2.tar.gz", hash = "sha256:b8658a06e52d6c47445c2ec11d292e1d52c3af259214c8b52e3a1aab733daa72"}, +] + +[package.extras] +arrays = ["numpy"] + [[package]] name = "nh3" version = "0.2.19" @@ -1147,6 +1321,7 @@ files = [ {file = "nh3-0.2.19-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:00810cd5275f5c3f44b9eb0e521d1a841ee2f8023622de39ffc7d88bd533d8e0"}, {file = "nh3-0.2.19-cp38-abi3-win32.whl", hash = "sha256:7e98621856b0a911c21faa5eef8f8ea3e691526c2433f9afc2be713cb6fbdb48"}, {file = "nh3-0.2.19-cp38-abi3-win_amd64.whl", hash = "sha256:75c7cafb840f24430b009f7368945cb5ca88b2b54bb384ebfba495f16bc9c121"}, + {file = "nh3-0.2.19.tar.gz", hash = "sha256:790056b54c068ff8dceb443eaefb696b84beff58cca6c07afd754d17692a4804"}, ] [[package]] @@ -1160,6 +1335,54 @@ files = [ {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, ] +[[package]] +name = "numexpr" +version = "2.10.2" +description = "Fast numerical expression evaluator for NumPy" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numexpr-2.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5b0e82d2109c1d9e63fcd5ea177d80a11b881157ab61178ddbdebd4c561ea46"}, + {file = "numexpr-2.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc2b8035a0c2cdc352e58c3875cb668836018065cbf5752cb531015d9a568d8"}, + {file = "numexpr-2.10.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0db5ff5183935d1612653559c319922143e8fa3019007696571b13135f216458"}, + {file = "numexpr-2.10.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15f59655458056fdb3a621b1bb8e071581ccf7e823916c7568bb7c9a3e393025"}, + {file = "numexpr-2.10.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ce8cccf944339051e44a49a124a06287fe3066d0acbff33d1aa5aee10a96abb7"}, + {file = "numexpr-2.10.2-cp310-cp310-win32.whl", hash = "sha256:ba85371c9a8d03e115f4dfb6d25dfbce05387002b9bc85016af939a1da9624f0"}, + {file = "numexpr-2.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:deb64235af9eeba59fcefa67e82fa80cfc0662e1b0aa373b7118a28da124d51d"}, + {file = "numexpr-2.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6b360eb8d392483410fe6a3d5a7144afa298c9a0aa3e9fe193e89590b47dd477"}, + {file = "numexpr-2.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d9a42f5c24880350d88933c4efee91b857c378aaea7e8b86221fff569069841e"}, + {file = "numexpr-2.10.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83fcb11988b57cc25b028a36d285287d706d1f536ebf2662ea30bd990e0de8b9"}, + {file = "numexpr-2.10.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4213a92efa9770bc28e3792134e27c7e5c7e97068bdfb8ba395baebbd12f991b"}, + {file = "numexpr-2.10.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ebdbef5763ca057eea0c2b5698e4439d084a0505d9d6e94f4804f26e8890c45e"}, + {file = "numexpr-2.10.2-cp311-cp311-win32.whl", hash = "sha256:3bf01ec502d89944e49e9c1b5cc7c7085be8ca2eb9dd46a0eafd218afbdbd5f5"}, + {file = "numexpr-2.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:e2d0ae24b0728e4bc3f1d3f33310340d67321d36d6043f7ce26897f4f1042db0"}, + {file = "numexpr-2.10.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5323a46e75832334f1af86da1ef6ff0add00fbacdd266250be872b438bdf2be"}, + {file = "numexpr-2.10.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a42963bd4c62d8afa4f51e7974debfa39a048383f653544ab54f50a2f7ec6c42"}, + {file = "numexpr-2.10.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5191ba8f2975cb9703afc04ae845a929e193498c0e8bcd408ecb147b35978470"}, + {file = "numexpr-2.10.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:97298b14f0105a794bea06fd9fbc5c423bd3ff4d88cbc618860b83eb7a436ad6"}, + {file = "numexpr-2.10.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9d7805ccb6be2d3b0f7f6fad3707a09ac537811e8e9964f4074d28cb35543db"}, + {file = "numexpr-2.10.2-cp312-cp312-win32.whl", hash = "sha256:cb845b2d4f9f8ef0eb1c9884f2b64780a85d3b5ae4eeb26ae2b0019f489cd35e"}, + {file = "numexpr-2.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:57b59cbb5dcce4edf09cd6ce0b57ff60312479930099ca8d944c2fac896a1ead"}, + {file = "numexpr-2.10.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a37d6a51ec328c561b2ca8a2bef07025642eca995b8553a5267d0018c732976d"}, + {file = "numexpr-2.10.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:81d1dde7dd6166d8ff5727bb46ab42a6b0048db0e97ceb84a121334a404a800f"}, + {file = "numexpr-2.10.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5b3f814437d5a10797f8d89d2037cca2c9d9fa578520fc911f894edafed6ea3e"}, + {file = "numexpr-2.10.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9309f2e43fe6e4560699ef5c27d7a848b3ff38549b6b57194207cf0e88900527"}, + {file = "numexpr-2.10.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ebb73b93f5c4d6994f357fa5a47a9f7a5485577e633b3c46a603cb01445bbb19"}, + {file = "numexpr-2.10.2-cp313-cp313-win32.whl", hash = "sha256:ec04c9a3c050c175348801e27c18c68d28673b7bfb865ef88ce333be523bbc01"}, + {file = "numexpr-2.10.2-cp313-cp313-win_amd64.whl", hash = "sha256:d7a3fc83c959288544db3adc70612475d8ad53a66c69198105c74036182d10dd"}, + {file = "numexpr-2.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0495f8111c3633e265248709b8b3b521bbfa646ba384909edd10e2b9a588a83a"}, + {file = "numexpr-2.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2aa05ac71bee3b1253e73173c4d7fa96a09a18970c0226f1c2c07a71ffe988dc"}, + {file = "numexpr-2.10.2-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3a23c3002ab330056fbdd2785871937a6f2f2fa85d06c8d0ff74ea8418119d1"}, + {file = "numexpr-2.10.2-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a018a7d81326f4c73d8b5aee61794d7d8514512f43957c0db61eb2a8a86848c7"}, + {file = "numexpr-2.10.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:037859b17a0abe2b489d4c2cfdadd2bf458ec80dd83f338ea5544c7987e06b85"}, + {file = "numexpr-2.10.2-cp39-cp39-win32.whl", hash = "sha256:eb278ccda6f893a312aa0452701bb17d098b7b14eb7c9381517d509cce0a39a3"}, + {file = "numexpr-2.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:734b64c6d6a597601ce9d0ef7b666e678ec015b446f1d1412c23903c021436c3"}, + {file = "numexpr-2.10.2.tar.gz", hash = "sha256:b0aff6b48ebc99d2f54f27b5f73a58cb92fde650aeff1b397c71c8788b4fff1a"}, +] + +[package.dependencies] +numpy = ">=1.23.0" + [[package]] name = "numpy" version = "2.0.2" @@ -3076,4 +3299,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "d62cd1897d8f73e9aad9e907beb82be509dc5e33d8f37b36ebf26ad1f3075a9f" +content-hash = "8b45b61e191f2980107d56680f0fd7bdfb93e305cea268566adf5cf1f20bf554" diff --git a/pyproject.toml b/pyproject.toml index 8fd1886f0..44643442f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ selenium = ">=4.11.0,<5.0" pytest-benchmark = ">=4.0.0,<6.0" playwright = ">=1.46.0" pytest-playwright = ">=0.5.1" +blosc2 = { version = ">=2.7.1", python = ">=3.11" } [tool.poetry.scripts] reflex = "reflex.reflex:cli" From ec1a2d3a974b551cdbd617072534ee323b918acf Mon Sep 17 00:00:00 2001 From: Benedikt Bartscher Date: Mon, 25 Nov 2024 00:07:25 +0100 Subject: [PATCH 2/7] implement state compression --- reflex/config.py | 6 ++++++ reflex/state.py | 54 +++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/reflex/config.py b/reflex/config.py index 0579b019f..fa54fc182 100644 --- a/reflex/config.py +++ b/reflex/config.py @@ -567,6 +567,12 @@ class EnvironmentVariables: # The maximum size of the reflex state in kilobytes. REFLEX_STATE_SIZE_LIMIT: EnvVar[int] = env_var(1000) + # Whether to compress the reflex state. + REFLEX_COMPRESS_STATE: EnvVar[bool] = env_var(False) + + # Threshold for the reflex state compression in bytes. + REFLEX_COMPRESS_THRESHOLD: EnvVar[int] = env_var(1024) + environment = EnvironmentVariables() diff --git a/reflex/state.py b/reflex/state.py index 1cd3e2c3e..d0e76370d 100644 --- a/reflex/state.py +++ b/reflex/state.py @@ -17,6 +17,7 @@ import uuid from abc import ABC, abstractmethod from collections import defaultdict from hashlib import md5 +from io import BytesIO from pathlib import Path from types import FunctionType, MethodType from typing import ( @@ -143,6 +144,10 @@ HANDLED_PICKLE_ERRORS = ( ValueError, ) +STATE_NOT_COMPRESSED = b"\x01" +STATE_COMPRESSED = b"\x02" +STATE_CHUNK_SIZE = 1024 + def _no_chain_background_task( state_cls: Type["BaseState"], name: str, fn: Callable @@ -2218,6 +2223,8 @@ class BaseState(Base, ABC, extra=pydantic.Extra.allow): error = "" try: payload = pickle.dumps((self._to_schema(), self)) + if environment.REFLEX_PERF_MODE.get() != PerformanceMode.OFF: + self._check_state_size(len(payload)) except HANDLED_PICKLE_ERRORS as og_pickle_error: error = ( f"Failed to serialize state {self.get_full_name()} due to unpicklable object. " @@ -2236,12 +2243,23 @@ class BaseState(Base, ABC, extra=pydantic.Extra.allow): error += f"Dill was also unable to pickle the state: {ex}" console.warn(error) + size = len(payload) if environment.REFLEX_PERF_MODE.get() != PerformanceMode.OFF: - self._check_state_size(len(payload)) + self._check_state_size(size) if not payload: raise StateSerializationError(error) + if environment.REFLEX_COMPRESS_STATE.get(): + if size > environment.REFLEX_COMPRESS_THRESHOLD.get(): + from blosc2 import compress + + payload = compress(payload) + prefix = STATE_COMPRESSED + else: + prefix = STATE_NOT_COMPRESSED + payload = prefix + payload + return payload @classmethod @@ -2264,14 +2282,40 @@ class BaseState(Base, ABC, extra=pydantic.Extra.allow): StateSchemaMismatchError: If the state schema does not match the expected schema. """ if data is not None and fp is None: - (substate_schema, state) = pickle.loads(data) + if environment.REFLEX_COMPRESS_STATE.get(): + # get first byte to determine if compressed + is_compressed = data[:1] == STATE_COMPRESSED + # remove compression byte + data = data[1:] + if is_compressed: + from blosc2 import decompress + + data = decompress(data) + data = pickle.loads(data) # type: ignore elif fp is not None and data is None: - (substate_schema, state) = pickle.load(fp) + if environment.REFLEX_COMPRESS_STATE.get(): + # read first byte to determine if compressed + is_compressed = fp.read(1) == STATE_COMPRESSED + if is_compressed: + from blosc2 import SChunk + + schunk = SChunk(chunksize=STATE_CHUNK_SIZE) + + while chunk := fp.read(STATE_CHUNK_SIZE): + schunk.append_data(chunk) + + fp = BytesIO() + + for chunk_index in range(schunk.nchunks): + fp.write(schunk.decompress_chunk(chunk_index)) + + data = pickle.load(fp) else: raise ValueError("Only one of `data` or `fp` must be provided") - if substate_schema != state._to_schema(): + substate_schema, state = data # type: ignore + if substate_schema != state._to_schema(): # type: ignore raise StateSchemaMismatchError() - return state + return state # type: ignore class State(BaseState): From 8e625025ba75d9a2a66b8708e0b57b01a50d40b2 Mon Sep 17 00:00:00 2001 From: Benedikt Bartscher Date: Sat, 21 Dec 2024 17:38:51 +0100 Subject: [PATCH 3/7] fix: ignore multiple size in state compression --- reflex/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reflex/state.py b/reflex/state.py index d0e76370d..305c4da01 100644 --- a/reflex/state.py +++ b/reflex/state.py @@ -2254,7 +2254,7 @@ class BaseState(Base, ABC, extra=pydantic.Extra.allow): if size > environment.REFLEX_COMPRESS_THRESHOLD.get(): from blosc2 import compress - payload = compress(payload) + payload = compress(payload, _ignore_multiple_size=True) prefix = STATE_COMPRESSED else: prefix = STATE_NOT_COMPRESSED From e2a58c2d6f4971c8a10e8d38961ee57f3ae55836 Mon Sep 17 00:00:00 2001 From: Benedikt Bartscher Date: Sat, 21 Dec 2024 17:41:05 +0100 Subject: [PATCH 4/7] cleanup rebase conflict --- reflex/state.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/reflex/state.py b/reflex/state.py index 305c4da01..59c84d50e 100644 --- a/reflex/state.py +++ b/reflex/state.py @@ -2223,8 +2223,6 @@ class BaseState(Base, ABC, extra=pydantic.Extra.allow): error = "" try: payload = pickle.dumps((self._to_schema(), self)) - if environment.REFLEX_PERF_MODE.get() != PerformanceMode.OFF: - self._check_state_size(len(payload)) except HANDLED_PICKLE_ERRORS as og_pickle_error: error = ( f"Failed to serialize state {self.get_full_name()} due to unpicklable object. " From 7853dc9da71d1d9cbc769ee55fb7f3b7a4f81463 Mon Sep 17 00:00:00 2001 From: Benedikt Bartscher Date: Sat, 21 Dec 2024 17:47:16 +0100 Subject: [PATCH 5/7] add type ignores.. :/ --- benchmarks/benchmark_pickle.py | 4 ++-- reflex/state.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmarks/benchmark_pickle.py b/benchmarks/benchmark_pickle.py index 7773d3b2e..c282874b8 100644 --- a/benchmarks/benchmark_pickle.py +++ b/benchmarks/benchmark_pickle.py @@ -109,10 +109,10 @@ def test_pickle( pytest.skip("Blosc is not available.") def dump(obj: State) -> bytes: - return compress(pickle.dumps(obj, protocol=protocol)) + return compress(pickle.dumps(obj, protocol=protocol)) # type: ignore def load(data: bytes) -> State: - return pickle.loads(decompress(data)) + return pickle.loads(decompress(data)) # type: ignore else: diff --git a/reflex/state.py b/reflex/state.py index 59c84d50e..3fb5d8dad 100644 --- a/reflex/state.py +++ b/reflex/state.py @@ -2256,7 +2256,7 @@ class BaseState(Base, ABC, extra=pydantic.Extra.allow): prefix = STATE_COMPRESSED else: prefix = STATE_NOT_COMPRESSED - payload = prefix + payload + payload = prefix + payload # type: ignore return payload @@ -2288,7 +2288,7 @@ class BaseState(Base, ABC, extra=pydantic.Extra.allow): if is_compressed: from blosc2 import decompress - data = decompress(data) + data = decompress(data) # type: ignore data = pickle.loads(data) # type: ignore elif fp is not None and data is None: if environment.REFLEX_COMPRESS_STATE.get(): @@ -2305,7 +2305,7 @@ class BaseState(Base, ABC, extra=pydantic.Extra.allow): fp = BytesIO() for chunk_index in range(schunk.nchunks): - fp.write(schunk.decompress_chunk(chunk_index)) + fp.write(schunk.decompress_chunk(chunk_index)) # type: ignore data = pickle.load(fp) else: From 51831f13c668984c5dc0707c5f129c848db7b21f Mon Sep 17 00:00:00 2001 From: Benedikt Bartscher Date: Thu, 30 Jan 2025 15:30:34 +0100 Subject: [PATCH 6/7] fix PGH003, i love it --- benchmarks/benchmark_pickle.py | 8 ++++---- reflex/state.py | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/benchmark_pickle.py b/benchmarks/benchmark_pickle.py index c282874b8..d03c21a42 100644 --- a/benchmarks/benchmark_pickle.py +++ b/benchmarks/benchmark_pickle.py @@ -109,10 +109,10 @@ def test_pickle( pytest.skip("Blosc is not available.") def dump(obj: State) -> bytes: - return compress(pickle.dumps(obj, protocol=protocol)) # type: ignore + return compress(pickle.dumps(obj, protocol=protocol)) # pyright: ignore[reportReturnType] def load(data: bytes) -> State: - return pickle.loads(decompress(data)) # type: ignore + return pickle.loads(decompress(data)) # pyright: ignore[reportAny,reportArgumentType] else: @@ -134,7 +134,7 @@ def test_pickle( def run(obj: State) -> None: _ = redis_client.set(key, dump(obj)) - _ = load(redis_client.get(key)) # type: ignore + _ = load(redis_client.get(key)) # pyright: ignore[reportArgumentType] else: @@ -149,7 +149,7 @@ def test_pickle( benchmark.extra_info["size"] = size benchmark.extra_info["redis"] = redis benchmark.extra_info["pickle_protocol"] = protocol - redis_group = redis.__name__ if redis else "no_redis" # type: ignore + redis_group = redis.__name__ if redis else "no_redis" # pyright: ignore[reportUnknownMemberType,reportAttributeAccessIssue,reportUnknownVariableType] benchmark.group = f"{redis_group}_{big_state_size[1]}" _ = benchmark(run, big_state) diff --git a/reflex/state.py b/reflex/state.py index 439d2dfb5..89093deb0 100644 --- a/reflex/state.py +++ b/reflex/state.py @@ -2314,7 +2314,7 @@ class BaseState(Base, ABC, extra=pydantic.Extra.allow): prefix = STATE_COMPRESSED else: prefix = STATE_NOT_COMPRESSED - payload = prefix + payload # type: ignore + payload = prefix + payload # pyright: ignore[reportOperatorIssue,reportUnknownVariableType] return payload @@ -2346,8 +2346,8 @@ class BaseState(Base, ABC, extra=pydantic.Extra.allow): if is_compressed: from blosc2 import decompress - data = decompress(data) # type: ignore - data = pickle.loads(data) # type: ignore + data = decompress(data) # pyright: ignore[reportAssignmentType] + data = pickle.loads(data) # pyright: ignore[reportArgumentType] elif fp is not None and data is None: if environment.REFLEX_COMPRESS_STATE.get(): # read first byte to determine if compressed @@ -2363,13 +2363,13 @@ class BaseState(Base, ABC, extra=pydantic.Extra.allow): fp = BytesIO() for chunk_index in range(schunk.nchunks): - fp.write(schunk.decompress_chunk(chunk_index)) # type: ignore + fp.write(schunk.decompress_chunk(chunk_index)) # pyright: ignore[reportArgumentType,reportUnusedCallResult] data = pickle.load(fp) else: raise ValueError("Only one of `data` or `fp` must be provided") - substate_schema, state = data # type: ignore - if substate_schema != state._to_schema(): # type: ignore + substate_schema, state = data # pyright: ignore[reportUnknownVariableType,reportGeneralTypeIssues] + if substate_schema != state._to_schema(): # pyright: ignore[reportAttributeAccessIssue,reportUnknownMemberType] raise StateSchemaMismatchError() return state # type: ignore From ad7dbc2436c674a658cf106220e04bdaa7830a2b Mon Sep 17 00:00:00 2001 From: Benedikt Bartscher Date: Sat, 8 Feb 2025 21:00:13 +0100 Subject: [PATCH 7/7] explicit pyright ignores --- reflex/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reflex/state.py b/reflex/state.py index f43d6d151..d58205bac 100644 --- a/reflex/state.py +++ b/reflex/state.py @@ -2277,7 +2277,7 @@ class BaseState(Base, ABC, extra=pydantic.Extra.allow): substate_schema, state = data # pyright: ignore[reportUnknownVariableType,reportGeneralTypeIssues] if substate_schema != state._to_schema(): # pyright: ignore[reportAttributeAccessIssue,reportUnknownMemberType] raise StateSchemaMismatchError() - return state # type: ignore + return state # pyright: ignore[reportUnknownVariableType,reportReturnType] T_STATE = TypeVar("T_STATE", bound=BaseState)