From b130dd07cdd69cb6671ed013fd4b3053a0f8fe09 Mon Sep 17 00:00:00 2001 From: jackie-pc Date: Tue, 13 Feb 2024 12:02:37 -0800 Subject: [PATCH 1/3] print background compile errors ASAP (#2596) --- reflex/app_module_for_backend.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/reflex/app_module_for_backend.py b/reflex/app_module_for_backend.py index 57395e447..d4bc3d1dc 100644 --- a/reflex/app_module_for_backend.py +++ b/reflex/app_module_for_backend.py @@ -11,7 +11,10 @@ if "app" != constants.CompileVars.APP: app_module = get_app(reload=False) app = getattr(app_module, constants.CompileVars.APP) -ThreadPoolExecutor(max_workers=1).submit(app.compile_) +# Force background compile errors to print eagerly +ThreadPoolExecutor(max_workers=1).submit(app.compile_).add_done_callback( + lambda f: f.result() +) # ensure only "app" is exposed. del app_module From 6b6eea4d7d3b3476738f26460524774adce3ca2b Mon Sep 17 00:00:00 2001 From: jackie-pc Date: Tue, 13 Feb 2024 12:16:35 -0800 Subject: [PATCH 2/3] Revert "Revert "Revert "use process pool to compile faster (#2377)" (#2434)" (#2497)" (#2595) --- reflex/app.py | 202 ++++++++++++++++-------------------- reflex/compiler/compiler.py | 110 -------------------- 2 files changed, 90 insertions(+), 222 deletions(-) diff --git a/reflex/app.py b/reflex/app.py index 25d9d07c6..33d773a6f 100644 --- a/reflex/app.py +++ b/reflex/app.py @@ -6,9 +6,7 @@ import concurrent.futures import contextlib import copy import functools -import multiprocessing import os -import platform from typing import ( Any, AsyncIterator, @@ -38,7 +36,6 @@ from reflex.admin import AdminDash from reflex.base import Base from reflex.compiler import compiler from reflex.compiler import utils as compiler_utils -from reflex.compiler.compiler import ExecutorSafeFunctions from reflex.components import connection_modal from reflex.components.base.app_wrap import AppWrap from reflex.components.base.fragment import Fragment @@ -679,24 +676,15 @@ class App(Base): TimeElapsedColumn(), ) - # try to be somewhat accurate - but still not 100% - adhoc_steps_without_executor = 6 - fixed_pages_within_executor = 7 - progress.start() - task = progress.add_task( - "Compiling:", - total=len(self.pages) - + fixed_pages_within_executor - + adhoc_steps_without_executor, - ) - # Get the env mode. config = get_config() # Store the compile results. compile_results = [] + # Compile the pages in parallel. custom_components = set() + # TODO Anecdotally, processes=2 works 10% faster (cpu_count=12) all_imports = {} app_wrappers: Dict[tuple[int, str], Component] = { # Default app wrap component renders {children} @@ -706,137 +694,127 @@ class App(Base): # If a theme component was provided, wrap the app with it app_wrappers[(20, "Theme")] = self.theme - progress.advance(task) + with progress, concurrent.futures.ThreadPoolExecutor() as thread_pool: + fixed_pages = 7 + task = progress.add_task("Compiling:", total=len(self.pages) + fixed_pages) - for _route, component in self.pages.items(): - # Merge the component style with the app style. - component.add_style(self.style) - - component.apply_theme(self.theme) - - # Add component.get_imports() to all_imports. - all_imports.update(component.get_imports()) - - # Add the app wrappers from this component. - app_wrappers.update(component.get_app_wrap_components()) - - # Add the custom components from the page to the set. - custom_components |= component.get_custom_components() - - progress.advance(task) - - # Perform auto-memoization of stateful components. - ( - stateful_components_path, - stateful_components_code, - page_components, - ) = compiler.compile_stateful_components(self.pages.values()) - - progress.advance(task) - - # Catch "static" apps (that do not define a rx.State subclass) which are trying to access rx.State. - if code_uses_state_contexts(stateful_components_code) and self.state is None: - raise RuntimeError( - "To access rx.State in frontend components, at least one " - "subclass of rx.State must be defined in the app." - ) - compile_results.append((stateful_components_path, stateful_components_code)) - - app_root = self._app_root(app_wrappers=app_wrappers) - - progress.advance(task) - - # Prepopulate the global ExecutorSafeFunctions class with input data required by the compile functions. - # This is required for multiprocessing to work, in presence of non-picklable inputs. - for route, component in zip(self.pages, page_components): - ExecutorSafeFunctions.COMPILE_PAGE_ARGS_BY_ROUTE[route] = ( - route, - component, - self.state, - ) - - ExecutorSafeFunctions.COMPILE_APP_APP_ROOT = app_root - ExecutorSafeFunctions.CUSTOM_COMPONENTS = custom_components - ExecutorSafeFunctions.HEAD_COMPONENTS = self.head_components - ExecutorSafeFunctions.STYLE = self.style - ExecutorSafeFunctions.STATE = self.state - - # Use a forking process pool, if possible. Much faster, especially for large sites. - # Fallback to ThreadPoolExecutor as something that will always work. - executor = None - if platform.system() in ("Linux", "Darwin"): - executor = concurrent.futures.ProcessPoolExecutor( - mp_context=multiprocessing.get_context("fork") - ) - else: - executor = concurrent.futures.ThreadPoolExecutor() - - with executor: - result_futures = [] - - def _mark_complete(_=None): + def mark_complete(_=None): progress.advance(task) - def _submit_work(fn, *args, **kwargs): - f = executor.submit(fn, *args, **kwargs) - f.add_done_callback(_mark_complete) + for _route, component in self.pages.items(): + # Merge the component style with the app style. + component.add_style(self.style) + + component.apply_theme(self.theme) + + # Add component.get_imports() to all_imports. + all_imports.update(component.get_imports()) + + # Add the app wrappers from this component. + app_wrappers.update(component.get_app_wrap_components()) + + # Add the custom components from the page to the set. + custom_components |= component.get_custom_components() + + # Perform auto-memoization of stateful components. + ( + stateful_components_path, + stateful_components_code, + page_components, + ) = compiler.compile_stateful_components(self.pages.values()) + + # Catch "static" apps (that do not define a rx.State subclass) which are trying to access rx.State. + if ( + code_uses_state_contexts(stateful_components_code) + and self.state is None + ): + raise RuntimeError( + "To access rx.State in frontend components, at least one " + "subclass of rx.State must be defined in the app." + ) + compile_results.append((stateful_components_path, stateful_components_code)) + + result_futures = [] + + def submit_work(fn, *args, **kwargs): + """Submit work to the thread pool and add a callback to mark the task as complete. + + The Future will be added to the `result_futures` list. + + Args: + fn: The function to submit. + *args: The args to submit. + **kwargs: The kwargs to submit. + """ + f = thread_pool.submit(fn, *args, **kwargs) + f.add_done_callback(mark_complete) result_futures.append(f) # Compile all page components. - for route in self.pages: - _submit_work(ExecutorSafeFunctions.compile_page, route) + for route, component in zip(self.pages, page_components): + submit_work( + compiler.compile_page, + route, + component, + self.state, + ) # Compile the app wrapper. - _submit_work(ExecutorSafeFunctions.compile_app) + app_root = self._app_root(app_wrappers=app_wrappers) + submit_work(compiler.compile_app, app_root) # Compile the custom components. - _submit_work(ExecutorSafeFunctions.compile_custom_components) + submit_work(compiler.compile_components, custom_components) # Compile the root stylesheet with base styles. - _submit_work(compiler.compile_root_stylesheet, self.stylesheets) + submit_work(compiler.compile_root_stylesheet, self.stylesheets) # Compile the root document. - _submit_work(ExecutorSafeFunctions.compile_document_root) + submit_work(compiler.compile_document_root, self.head_components) # Compile the theme. - _submit_work(ExecutorSafeFunctions.compile_theme) + submit_work(compiler.compile_theme, style=self.style) # Compile the contexts. - _submit_work(ExecutorSafeFunctions.compile_contexts) + submit_work(compiler.compile_contexts, self.state) # Compile the Tailwind config. if config.tailwind is not None: config.tailwind["content"] = config.tailwind.get( "content", constants.Tailwind.CONTENT ) - _submit_work(compiler.compile_tailwind, config.tailwind) + submit_work(compiler.compile_tailwind, config.tailwind) else: - _submit_work(compiler.remove_tailwind_from_postcss) + submit_work(compiler.remove_tailwind_from_postcss) + + # Get imports from AppWrap components. + all_imports.update(app_root.get_imports()) + + # Iterate through all the custom components and add their imports to the all_imports. + for component in custom_components: + all_imports.update(component.get_imports()) # Wait for all compilation tasks to complete. for future in concurrent.futures.as_completed(result_futures): compile_results.append(future.result()) - # Get imports from AppWrap components. - all_imports.update(app_root.get_imports()) + # Empty the .web pages directory. + compiler.purge_web_pages_dir() - # Iterate through all the custom components and add their imports to the all_imports. - for component in custom_components: - all_imports.update(component.get_imports()) + # Avoid flickering when installing frontend packages + progress.stop() - progress.advance(task) + # Install frontend packages. + self.get_frontend_packages(all_imports) - # Empty the .web pages directory. - compiler.purge_web_pages_dir() - - progress.advance(task) - progress.stop() - - # Install frontend packages. - self.get_frontend_packages(all_imports) - - for output_path, code in compile_results: - compiler_utils.write_page(output_path, code) + # Write the pages at the end to trigger the NextJS hot reload only once. + write_page_futures = [] + for output_path, code in compile_results: + write_page_futures.append( + thread_pool.submit(compiler_utils.write_page, output_path, code) + ) + for future in concurrent.futures.as_completed(write_page_futures): + future.result() @contextlib.asynccontextmanager async def modify_state(self, token: str) -> AsyncIterator[BaseState]: diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index 9f4b7cbe5..31987dd78 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -467,113 +467,3 @@ def purge_web_pages_dir(): # Empty out the web pages directory. utils.empty_dir(constants.Dirs.WEB_PAGES, keep_files=["_app.js"]) - - -class ExecutorSafeFunctions: - """Helper class to allow parallelisation of parts of the compilation process. - - This class (and its class attributes) are available at global scope. - - In a multiprocessing context (like when using a ProcessPoolExecutor), the content of this - global class is logically replicated to any FORKED process. - - How it works: - * Before the child process is forked, ensure that we stash any input data required by any future - function call in the child process. - * After the child process is forked, the child process will have a copy of the global class, which - includes the previously stashed input data. - * Any task submitted to the child process simply needs a way to communicate which input data the - requested function call requires. - - Why do we need this? Passing input data directly to child process often not possible because the input data is not picklable. - The mechanic described here removes the need to pickle the input data at all. - - Limitations: - * This can never support returning unpicklable OUTPUT data. - * Any object mutations done by the child process will not propagate back to the parent process (fork goes one way!). - - """ - - COMPILE_PAGE_ARGS_BY_ROUTE = {} - COMPILE_APP_APP_ROOT: Component | None = None - CUSTOM_COMPONENTS: set[CustomComponent] | None = None - HEAD_COMPONENTS: list[Component] | None = None - STYLE: ComponentStyle | None = None - STATE: type[BaseState] | None = None - - @classmethod - def compile_page(cls, route: str): - """Compile a page. - - Args: - route: The route of the page to compile. - - Returns: - The path and code of the compiled page. - """ - return compile_page(*cls.COMPILE_PAGE_ARGS_BY_ROUTE[route]) - - @classmethod - def compile_app(cls): - """Compile the app. - - Returns: - The path and code of the compiled app. - - Raises: - ValueError: If the app root is not set. - """ - if cls.COMPILE_APP_APP_ROOT is None: - raise ValueError("COMPILE_APP_APP_ROOT should be set") - return compile_app(cls.COMPILE_APP_APP_ROOT) - - @classmethod - def compile_custom_components(cls): - """Compile the custom components. - - Returns: - The path and code of the compiled custom components. - - Raises: - ValueError: If the custom components are not set. - """ - if cls.CUSTOM_COMPONENTS is None: - raise ValueError("CUSTOM_COMPONENTS should be set") - return compile_components(cls.CUSTOM_COMPONENTS) - - @classmethod - def compile_document_root(cls): - """Compile the document root. - - Returns: - The path and code of the compiled document root. - - Raises: - ValueError: If the head components are not set. - """ - if cls.HEAD_COMPONENTS is None: - raise ValueError("HEAD_COMPONENTS should be set") - return compile_document_root(cls.HEAD_COMPONENTS) - - @classmethod - def compile_theme(cls): - """Compile the theme. - - Returns: - The path and code of the compiled theme. - - Raises: - ValueError: If the style is not set. - """ - if cls.STYLE is None: - raise ValueError("STYLE should be set") - return compile_theme(cls.STYLE) - - @classmethod - def compile_contexts(cls): - """Compile the contexts. - - Returns: - The path and code of the compiled contexts. - """ - return compile_contexts(cls.STATE) From dec777485fa3bfc32e4b544193d5ff1d271518bf Mon Sep 17 00:00:00 2001 From: Tom Gotsman <64492814+tgberkeley@users.noreply.github.com> Date: Tue, 13 Feb 2024 12:35:31 -0800 Subject: [PATCH 3/3] rx.download accepts `data` arg as either str or bytes (#2493) * initial attempt that works for dataframe and text downloads * changes for masens comments * Instead of using blob, just send a data: URL from the backend * Enable rx.download directly with Var If the Var is string-like and starts with `data:`, then no special processing occurs. Otherwise, the value is passed to JSON.stringify and downloaded as text/plain. * event: update docstring and comments on rx.download Raise ValueError when URL and data are both provided, or the data provided is not one of the expected types. --------- Co-authored-by: Tom Gotsman Co-authored-by: Masen Furer --- reflex/.templates/web/utils/state.js | 6 +-- reflex/event.py | 56 ++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/reflex/.templates/web/utils/state.js b/reflex/.templates/web/utils/state.js index eda4f1cf5..b61d1da67 100644 --- a/reflex/.templates/web/utils/state.js +++ b/reflex/.templates/web/utils/state.js @@ -152,12 +152,12 @@ export const applyEvent = async (event, socket) => { navigator.clipboard.writeText(content); return false; } + if (event.name == "_download") { const a = document.createElement('a'); a.hidden = true; - a.href = event.payload.url; - if (event.payload.filename) - a.download = event.payload.filename; + a.href = event.payload.url + a.download = event.payload.filename; a.click(); a.remove(); return false; diff --git a/reflex/event.py b/reflex/event.py index 28d1f3ecc..afdc2547a 100644 --- a/reflex/event.py +++ b/reflex/event.py @@ -2,6 +2,7 @@ from __future__ import annotations import inspect +from base64 import b64encode from types import FunctionType from typing import ( TYPE_CHECKING, @@ -552,21 +553,26 @@ def set_clipboard(content: str) -> EventSpec: ) -def download(url: str | Var, filename: Optional[str | Var] = None) -> EventSpec: - """Download the file at a given path. +def download( + url: str | Var | None = None, + filename: Optional[str | Var] = None, + data: str | bytes | Var | None = None, +) -> EventSpec: + """Download the file at a given path or with the specified data. Args: - url : The URL to the file to download. - filename : The name that the file should be saved as after download. + url: The URL to the file to download. + filename: The name that the file should be saved as after download. + data: The data to download. Raises: - ValueError: If the URL provided is invalid. + ValueError: If the URL provided is invalid, both URL and data are provided, + or the data is not an expected type. Returns: EventSpec: An event to download the associated file. """ - if isinstance(url, Var) and filename is None: - filename = "" + from reflex.components.core.cond import cond if isinstance(url, str): if not url.startswith("/"): @@ -576,6 +582,42 @@ def download(url: str | Var, filename: Optional[str | Var] = None) -> EventSpec: if filename is None: filename = url.rpartition("/")[-1] + if filename is None: + filename = "" + + if data is not None: + if url is not None: + raise ValueError("Cannot provide both URL and data to download.") + + if isinstance(data, str): + # Caller provided a plain text string to download. + url = "data:text/plain," + data + elif isinstance(data, Var): + # Need to check on the frontend if the Var already looks like a data: URI. + is_data_url = data._replace( + _var_name=( + f"typeof {data._var_full_name} == 'string' && " + f"{data._var_full_name}.startsWith('data:')" + ), + _var_type=bool, + _var_is_string=False, + _var_full_name_needs_state_prefix=False, + ) + # If it's a data: URI, use it as is, otherwise convert the Var to JSON in a data: URI. + url = cond( # type: ignore + is_data_url, + data, + "data:text/plain," + data.to_string(), # type: ignore + ) + elif isinstance(data, bytes): + # Caller provided bytes, so base64 encode it as a data: URI. + b64_data = b64encode(data).decode("utf-8") + url = "data:application/octet-stream;base64," + b64_data + else: + raise ValueError( + f"Invalid data type {type(data)} for download. Use `str` or `bytes`." + ) + return server_side( "_download", get_fn_signature(download),