basic functionality

reflex/app.py

@@ -3,13 +3,14 @@
 from __future__ import annotations
 
 import asyncio
-import concurrent.futures
 import contextlib
 import copy
+import dataclasses
 import functools
 import inspect
 import io
-import multiprocessing
+import multiprocess
+from pathos import multiprocessing, pools
 import os
 import platform
 import sys
@@ -169,6 +170,21 @@ class OverlayFragment(Fragment):
     pass
 
 
+@dataclasses.dataclass(
+    frozen=True,
+)
+class UncompiledPage:
+    """An uncompiled page."""
+
+    component: Component
+    route: str
+    title: str
+    description: str
+    image: str
+    on_load: Union[EventHandler, EventSpec, list[EventHandler | EventSpec], None]
+    meta: list[dict[str, str]]
+
+
 class App(MiddlewareMixin, LifespanMixin, Base):
     """The main Reflex app that encapsulates the backend and frontend.
 
@@ -219,6 +235,9 @@ class App(MiddlewareMixin, LifespanMixin, Base):
     # Attributes to add to the html root tag of every page.
     html_custom_attrs: Optional[Dict[str, str]] = None
 
+    # A map from a route to an uncompiled page. PRIVATE.
+    uncompiled_pages: Dict[str, UncompiledPage] = {}
+
     # A map from a page route to the component to render. Users should use `add_page`. PRIVATE.
     pages: Dict[str, Component] = {}
 
@@ -492,13 +511,13 @@ class App(MiddlewareMixin, LifespanMixin, Base):
         # Check if the route given is valid
         verify_route_validity(route)
 
-        if route in self.pages and os.getenv(constants.RELOAD_CONFIG):
+        if route in self.uncompiled_pages and os.getenv(constants.RELOAD_CONFIG):
             # when the app is reloaded(typically for app harness tests), we should maintain
             # the latest render function of a route.This applies typically to decorated pages
             # since they are only added when app._compile is called.
-            self.pages.pop(route)
+            self.uncompiled_pages.pop(route)
 
-        if route in self.pages:
+        if route in self.uncompiled_pages:
             route_name = (
                 f"`{route}` or `/`"
                 if route == constants.PageNames.INDEX_ROUTE
@@ -514,8 +533,33 @@ class App(MiddlewareMixin, LifespanMixin, Base):
         state = self.state if self.state else State
         state.setup_dynamic_args(get_route_args(route))
 
+        if on_load:
+            self.load_events[route] = (
+                on_load if isinstance(on_load, list) else [on_load]
+            )
+
+        self.uncompiled_pages[route] = UncompiledPage(
+            component=component,
+            route=route,
+            title=title,
+            description=description,
+            image=image,
+            on_load=on_load,
+            meta=meta,
+        )
+
+    def _compile_page(self, route: str):
+        """Compile a page.
+
+        Args:
+            route: The route of the page to compile.
+        """
+        uncompiled_page = self.uncompiled_pages[route]
+
+        on_load = uncompiled_page.on_load
+
         # Generate the component if it is a callable.
-        component = self._generate_component(component)
+        component = self._generate_component(uncompiled_page.component)
 
         # unpack components that return tuples in an rx.fragment.
         if isinstance(component, tuple):
@@ -538,16 +582,16 @@ class App(MiddlewareMixin, LifespanMixin, Base):
 
         meta_args = {
             "title": (
-                title
+                uncompiled_page.title
-                if title is not None
+                if uncompiled_page.title is not None
                 else format.make_default_page_title(get_config().app_name, route)
             ),
-            "image": image,
+            "image": uncompiled_page.image,
-            "meta": meta,
+            "meta": uncompiled_page.meta,
         }
 
-        if description is not None:
+        if uncompiled_page.description is not None:
-            meta_args["description"] = description
+            meta_args["description"] = uncompiled_page.description
 
         # Add meta information to the component.
         compiler_utils.add_meta(
@@ -559,12 +603,6 @@ class App(MiddlewareMixin, LifespanMixin, Base):
         self._check_routes_conflict(route)
         self.pages[route] = component
 
-        # Add the load events.
-        if on_load:
-            if not isinstance(on_load, list):
-                on_load = [on_load]
-            self.load_events[route] = on_load
-
     def get_load_events(self, route: str) -> list[EventHandler | EventSpec]:
         """Get the load events for a route.
 
@@ -839,11 +877,15 @@ class App(MiddlewareMixin, LifespanMixin, Base):
         """
         from reflex.utils.exceptions import ReflexRuntimeError
 
+        print("Compiling the app...")
+
+        self._enable_state()
+
         def get_compilation_time() -> str:
             return str(datetime.now().time()).split(".")[0]
 
         # Render a default 404 page if the user didn't supply one
-        if constants.Page404.SLUG not in self.pages:
+        if constants.Page404.SLUG not in self.uncompiled_pages:
             self.add_custom_404_page()
 
         # Add the optional endpoints (_upload)
@@ -869,7 +911,7 @@ class App(MiddlewareMixin, LifespanMixin, Base):
         progress.start()
         task = progress.add_task(
             f"[{get_compilation_time()}] Compiling:",
-            total=len(self.pages)
+            total=len(self.uncompiled_pages)
             + fixed_pages_within_executor
             + adhoc_steps_without_executor,
         )
@@ -898,7 +940,88 @@ class App(MiddlewareMixin, LifespanMixin, Base):
         all_imports = {}
         custom_components = set()
 
-        for _route, component in self.pages.items():
+        progress.advance(task)
+
+        # Compile the root document before fork.
+        compile_results.append(
+            compiler.compile_document_root(
+                self.head_components,
+                html_lang=self.html_lang,
+                html_custom_attrs=self.html_custom_attrs,  # type: ignore
+            )
+        )
+
+        # Fix #2992 by removing the top-level appearance prop
+        if self.theme is not None:
+            self.theme.appearance = None
+
+        progress.advance(task)
+
+        for route, uncompiled_page in self.uncompiled_pages.items():
+            ExecutorSafeFunctions.UNCOMPILED_PAGES[route] = uncompiled_page
+
+        ExecutorSafeFunctions.STYLE = self.style
+
+        # Use a forking process pool, if possible.  Much faster, especially for large sites.
+        # Fallback to ThreadPoolExecutor as something that will always work.
+        executor = None
+        if (
+            platform.system() in ("Linux", "Darwin")
+            and os.environ.get("REFLEX_COMPILE_PROCESSES") is not None
+        ):
+            executor = pools.ProcessPool()
+        else:
+            executor = pools.ThreadPool()
+
+        pages_results = []
+
+        with executor:
+            result_futures = []
+            pages_futures = []
+
+            # def _mark_complete(_=None):
+            #     progress.advance(task)
+
+            def _submit_work(fn, *args, **kwargs):
+                f = executor.apipe(fn, *args, **kwargs)
+                # f.add_done_callback(_mark_complete)
+                result_futures.append(f)
+
+            # Compile all page components.
+            for route in self.uncompiled_pages:
+                f = executor.apipe(
+                    ExecutorSafeFunctions.compile_uncompiled_page, route
+                )
+                # f.add_done_callback(_mark_complete)
+                pages_futures.append((route, f))
+
+            # Compile the root stylesheet with base styles.
+            _submit_work(compiler.compile_root_stylesheet, self.stylesheets)
+
+            # Compile the theme.
+            _submit_work(ExecutorSafeFunctions.compile_theme)
+
+            # Compile the Tailwind config.
+            if config.tailwind is not None:
+                config.tailwind["content"] = config.tailwind.get(
+                    "content", constants.Tailwind.CONTENT
+                )
+                _submit_work(compiler.compile_tailwind, config.tailwind)
+            else:
+                _submit_work(compiler.remove_tailwind_from_postcss)
+
+            # Wait for all compilation tasks to complete.
+            for future in result_futures:
+                compile_results.append(future.get())
+
+            for route, future in pages_futures:
+                print(f"Compiled {route}")
+                pages_results.append(future.get())
+
+        for route, component, compiled_page in pages_results:
+            self.pages[compiled_page] = component
+            compile_results.append(compiled_page)
+
             # Merge the component style with the app style.
             component._add_style_recursive(self.style, self.theme)
 
@@ -911,8 +1034,6 @@ class App(MiddlewareMixin, LifespanMixin, Base):
             # Add the custom components from the page to the set.
             custom_components |= component._get_all_custom_components()
 
-        progress.advance(task)
-
         # Perform auto-memoization of stateful components.
         (
             stateful_components_path,
@@ -930,114 +1051,30 @@ class App(MiddlewareMixin, LifespanMixin, Base):
             )
         compile_results.append((stateful_components_path, stateful_components_code))
 
-        # Compile the root document before fork.
-        compile_results.append(
-            compiler.compile_document_root(
-                self.head_components,
-                html_lang=self.html_lang,
-                html_custom_attrs=self.html_custom_attrs,  # type: ignore
-            )
-        )
-
-        # Compile the contexts before fork.
-        compile_results.append(
-            compiler.compile_contexts(self.state, self.theme),
-        )
-        # Fix #2992 by removing the top-level appearance prop
-        if self.theme is not None:
-            self.theme.appearance = None
-
         app_root = self._app_root(app_wrappers=app_wrappers)
 
-        progress.advance(task)
-
-        # Prepopulate the global ExecutorSafeFunctions class with input data required by the compile functions.
-        # This is required for multiprocessing to work, in presence of non-picklable inputs.
-        for route, component in zip(self.pages, page_components):
-            ExecutorSafeFunctions.COMPILE_PAGE_ARGS_BY_ROUTE[route] = (
-                route,
-                component,
-                self.state,
-            )
-
-        ExecutorSafeFunctions.COMPILE_APP_APP_ROOT = app_root
-        ExecutorSafeFunctions.CUSTOM_COMPONENTS = custom_components
-        ExecutorSafeFunctions.STYLE = self.style
-
-        # Use a forking process pool, if possible.  Much faster, especially for large sites.
-        # Fallback to ThreadPoolExecutor as something that will always work.
-        executor = None
-        if (
-            platform.system() in ("Linux", "Darwin")
-            and os.environ.get("REFLEX_COMPILE_PROCESSES") is not None
-        ):
-            executor = concurrent.futures.ProcessPoolExecutor(
-                max_workers=int(os.environ.get("REFLEX_COMPILE_PROCESSES", 0)) or None,
-                mp_context=multiprocessing.get_context("fork"),
-            )
-        else:
-            executor = concurrent.futures.ThreadPoolExecutor(
-                max_workers=int(os.environ.get("REFLEX_COMPILE_THREADS", 0)) or None,
-            )
-
-        with executor:
-            result_futures = []
-            custom_components_future = None
-
-            def _mark_complete(_=None):
-                progress.advance(task)
-
-            def _submit_work(fn, *args, **kwargs):
-                f = executor.submit(fn, *args, **kwargs)
-                f.add_done_callback(_mark_complete)
-                result_futures.append(f)
-
-            # Compile all page components.
-            for route in self.pages:
-                _submit_work(ExecutorSafeFunctions.compile_page, route)
-
-            # Compile the app wrapper.
-            _submit_work(ExecutorSafeFunctions.compile_app)
-
-            # Compile the custom components.
-            custom_components_future = executor.submit(
-                ExecutorSafeFunctions.compile_custom_components,
-            )
-            custom_components_future.add_done_callback(_mark_complete)
-
-            # Compile the root stylesheet with base styles.
-            _submit_work(compiler.compile_root_stylesheet, self.stylesheets)
-
-            # Compile the theme.
-            _submit_work(ExecutorSafeFunctions.compile_theme)
-
-            # Compile the Tailwind config.
-            if config.tailwind is not None:
-                config.tailwind["content"] = config.tailwind.get(
-                    "content", constants.Tailwind.CONTENT
-                )
-                _submit_work(compiler.compile_tailwind, config.tailwind)
-            else:
-                _submit_work(compiler.remove_tailwind_from_postcss)
-
-            # Wait for all compilation tasks to complete.
-            for future in concurrent.futures.as_completed(result_futures):
-                compile_results.append(future.result())
-
-            # Special case for custom_components, since we need the compiled imports
-            # to install proper frontend packages.
-            (
-                *custom_components_result,
-                custom_components_imports,
-            ) = custom_components_future.result()
-            compile_results.append(custom_components_result)
-            all_imports.update(custom_components_imports)
-
         # Get imports from AppWrap components.
         all_imports.update(app_root._get_all_imports())
 
         progress.advance(task)
 
+        # Compile the contexts before fork.
+        compile_results.append(
+            compiler.compile_contexts(self.state, self.theme),
+        )
+
+        # Compile the app root.
+        compile_results.append(
+            compiler.compile_app(app_root),
+        )
+
+        # Compile custom components.
+        *custom_components_result, custom_components_imports = (
+            compiler.compile_components(custom_components)
+        )
+        compile_results.append(custom_components_result)
+        all_imports.update(custom_components_imports)
+
         progress.advance(task)
         progress.stop()
 

reflex/compiler/compiler.py

@@ -5,10 +5,11 @@ from __future__ import annotations
 import os
 from datetime import datetime
 from pathlib import Path
-from typing import Dict, Iterable, Optional, Type, Union
+from typing import TYPE_CHECKING, Dict, Iterable, Optional, Type, Union
 
 from reflex import constants
 from reflex.compiler import templates, utils
+from reflex.components.base.fragment import Fragment
 from reflex.components.component import (
     BaseComponent,
     Component,
@@ -511,6 +512,58 @@ def purge_web_pages_dir():
     utils.empty_dir(get_web_dir() / constants.Dirs.PAGES, keep_files=["_app.js"])
 
 
+if TYPE_CHECKING:
+    from reflex.app import UncompiledPage
+    from reflex.event import EventHandler, EventSpec
+
+
+def compile_uncompiled_page(
+    route: str, page: UncompiledPage
+) -> tuple[EventHandler | EventSpec | list[EventHandler | EventSpec] | None, Fragment]:
+    """Compiles an uncompiled page into a component and adds meta information.
+
+    Args:
+        route (str): The route of the page.
+        page (UncompiledPage): The uncompiled page object.
+
+    Returns:
+        tuple[EventHandler | EventSpec | list[EventHandler | EventSpec] | None, Fragment]: The on_load event handler or spec, and the compiled component.
+    """
+    # Generate the component if it is a callable.
+    component = page.component
+    component = component if isinstance(component, Component) else component()
+
+    # unpack components that return tuples in an rx.fragment.
+    if isinstance(component, tuple):
+        component = Fragment.create(*component)
+
+    from reflex.app import OverlayFragment
+    from reflex.utils.format import make_default_page_title
+
+    component = OverlayFragment.create(component)
+
+    meta_args = {
+        "title": (
+            page.title
+            if page.title is not None
+            else make_default_page_title(get_config().app_name, route)
+        ),
+        "image": page.image,
+        "meta": page.meta,
+    }
+
+    if page.description is not None:
+        meta_args["description"] = page.description
+
+    # Add meta information to the component.
+    utils.add_meta(
+        component,
+        **meta_args,
+    )
+
+    return component
+
+
 class ExecutorSafeFunctions:
     """Helper class to allow parallelisation of parts of the compilation process.
 
@@ -536,9 +589,9 @@ class ExecutorSafeFunctions:
 
     """
 
-    COMPILE_PAGE_ARGS_BY_ROUTE = {}
+    UNCOMPILED_PAGES = {}
-    COMPILE_APP_APP_ROOT: Component | None = None
+    COMPILED_COMPONENTS = {}
-    CUSTOM_COMPONENTS: set[CustomComponent] | None = None
+    STATE: Type[BaseState] | None = None
     STYLE: ComponentStyle | None = None
 
     @classmethod
@@ -551,35 +604,21 @@ class ExecutorSafeFunctions:
         Returns:
             The path and code of the compiled page.
         """
-        return compile_page(*cls.COMPILE_PAGE_ARGS_BY_ROUTE[route])
+        return compile_page(route, cls.COMPILED_COMPONENTS[route], cls.STATE)
 
     @classmethod
-    def compile_app(cls):
+    def compile_uncompiled_page(cls, route: str):
-        """Compile the app.
+        """Compile an uncompiled page.
+
+        Args:
+            route: The route of the page to compile.
 
         Returns:
-            The path and code of the compiled app.
+            The path and code of the compiled page.
-
-        Raises:
-            ValueError: If the app root is not set.
         """
-        if cls.COMPILE_APP_APP_ROOT is None:
+        component = compile_uncompiled_page(route, cls.UNCOMPILED_PAGES[route])
-            raise ValueError("COMPILE_APP_APP_ROOT should be set")
+        component = component if isinstance(component, Component) else component()
-        return compile_app(cls.COMPILE_APP_APP_ROOT)
+        return route, component, compile_page(route, component, cls.STATE)
-
-    @classmethod
-    def compile_custom_components(cls):
-        """Compile the custom components.
-
-        Returns:
-            The path and code of the compiled custom components.
-
-        Raises:
-            ValueError: If the custom components are not set.
-        """
-        if cls.CUSTOM_COMPONENTS is None:
-            raise ValueError("CUSTOM_COMPONENTS should be set")
-        return compile_components(cls.CUSTOM_COMPONENTS)
 
     @classmethod
     def compile_theme(cls):

reflex/style.py

@@ -257,7 +257,9 @@ class Style(dict):
         _var = Var.create(value, _var_is_string=False)
         if _var is not None:
             # Carry the imports/hooks when setting a Var as a value.
-            self._var_data = VarData.merge(self._var_data, _var._var_data)
+            self._var_data = VarData.merge(
+                self._var_data if hasattr(self, "_var_data") else None, _var._var_data
+            )
         super().__setitem__(key, value)