From fed75ea7f85fc772d4496859c209423b9a5d0cee Mon Sep 17 00:00:00 2001
From: Alek Petuskey <alek@pynecone.io>
Date: Thu, 31 Aug 2023 14:20:44 -0700
Subject: [PATCH] Incrementally Add New Packages (#1607)

---
 reflex/.templates/web/package.json         | 20 +-----
 reflex/app.py                              | 42 ++++++++++--
 reflex/compiler/utils.py                   | 17 ++++-
 reflex/components/component.py             | 31 +++++++--
 reflex/components/datadisplay/code.py      |  2 +-
 reflex/components/datadisplay/datatable.py |  4 +-
 reflex/components/forms/debounce.py        |  2 +-
 reflex/components/forms/upload.py          |  2 +-
 reflex/components/graphing/plotly.py       |  6 +-
 reflex/components/graphing/victory.py      |  2 +-
 reflex/components/libs/react_player.py     |  2 +-
 reflex/components/media/icon.py            |  2 +-
 reflex/components/typography/markdown.py   |  9 ++-
 reflex/py.typed                            |  0
 reflex/reflex.py                           |  3 +
 reflex/utils/build.py                      |  3 -
 reflex/utils/exec.py                       | 75 ++++++++++++++++++++--
 reflex/utils/prerequisites.py              | 19 ++++--
 reflex/vars.py                             |  6 ++
 19 files changed, 192 insertions(+), 55 deletions(-)
 delete mode 100644 reflex/py.typed

diff --git a/reflex/.templates/web/package.json b/reflex/.templates/web/package.json
index b1ab8b666..191e7a4e5 100644
--- a/reflex/.templates/web/package.json
+++ b/reflex/.templates/web/package.json
@@ -7,7 +7,6 @@
     "prod": "next start"
   },
   "dependencies": {
-    "@chakra-ui/icons": "^2.0.19",
     "@chakra-ui/react": "^2.6.0",
     "@chakra-ui/system": "^2.5.6",
     "@emotion/react": "^11.10.6",
@@ -15,32 +14,17 @@
     "axios": "^1.4.0",
     "chakra-react-select": "^4.6.0",
     "focus-visible": "^5.2.0",
-    "framer-motion": "^10.12.4",
-    "gridjs": "^6.0.6",
-    "gridjs-react": "^6.0.1",
     "json5": "^2.2.3",
     "next": "^13.3.1",
     "next-sitemap": "^4.1.8",
-    "plotly.js": "^2.22.0",
     "react": "^18.2.0",
-    "react-debounce-input": "^3.3.0",
     "react-dom": "^18.2.0",
-    "react-dropzone": "^14.2.3",
-    "react-markdown": "^8.0.7",
-    "react-player": "^2.12.0",
-    "react-plotly.js": "^2.6.0",
-    "react-syntax-highlighter": "^15.5.0",
-    "rehype-katex": "^6.0.3",
-    "rehype-raw": "^6.1.1",
-    "remark-gfm": "^3.0.1",
-    "remark-math": "^5.1.1",
     "socket.io-client": "^4.6.1",
-    "universal-cookie": "^4.0.4",
-    "victory": "^36.6.8"
+    "universal-cookie": "^4.0.4"
   },
   "devDependencies": {
     "autoprefixer": "^10.4.14",
     "postcss": "^8.4.24",
     "tailwindcss": "^3.3.2"
   }
-}
+}
\ No newline at end of file
diff --git a/reflex/app.py b/reflex/app.py
index 89fad454d..ef5b165d0 100644
--- a/reflex/app.py
+++ b/reflex/app.py
@@ -47,7 +47,7 @@ from reflex.route import (
     verify_route_validity,
 )
 from reflex.state import DefaultState, State, StateManager, StateUpdate
-from reflex.utils import console, format, types
+from reflex.utils import console, format, prerequisites, types
 
 # Define custom types.
 ComponentCallable = Callable[[], Component]
@@ -483,6 +483,27 @@ class App(Base):
 
             admin.mount_to(self.api)
 
+    def get_frontend_packages(self, imports: Dict[str, str]):
+        """Gets the frontend packages to be installed and filters out the unnecessary ones.
+
+        Args:
+            imports: A dictionary containing the imports used in the current page.
+
+        Example:
+            >>> get_frontend_packages({"react": "16.14.0", "react-dom": "16.14.0"})
+        """
+        page_imports = [
+            i
+            for i in imports
+            if i not in compiler.DEFAULT_IMPORTS.keys()
+            and i != "focus-visible/dist/focus-visible"
+            and "next" not in i
+            and not i.startswith("/")
+            and i != ""
+        ]
+        page_imports.extend(get_config().frontend_packages)
+        prerequisites.install_frontend_packages(page_imports)
+
     def compile(self):
         """Compile the app and output it to the pages folder."""
         if os.environ.get(constants.SKIP_COMPILE_ENV_VAR) == "yes":
@@ -493,12 +514,13 @@ class App(Base):
             MofNCompleteColumn(),
             TimeElapsedColumn(),
         )
-        task = progress.add_task("Compiling: ", total=len(self.pages))
 
-        # TODO: include all work done in progress indicator, not just self.pages
         for render, kwargs in DECORATED_PAGES:
             self.add_page(render, **kwargs)
 
+        task = progress.add_task("Compiling: ", total=len(self.pages))
+        # TODO: include all work done in progress indicator, not just self.pages
+
         # Get the env mode.
         config = get_config()
 
@@ -509,6 +531,7 @@ class App(Base):
         custom_components = set()
         # TODO Anecdotally, processes=2 works 10% faster (cpu_count=12)
         thread_pool = ThreadPool()
+        all_imports = {}
         with progress:
             for route, component in self.pages.items():
                 # TODO: this progress does not reflect actual threaded task completion
@@ -524,8 +547,12 @@ class App(Base):
                         ),
                     )
                 )
+                # add component.get_imports() to all_imports
+                all_imports.update(component.get_imports())
+
                 # Add the custom components from the page to the set.
                 custom_components |= component.get_custom_components()
+
         thread_pool.close()
         thread_pool.join()
 
@@ -537,7 +564,11 @@ class App(Base):
         # Compile the custom components.
         compile_results.append(compiler.compile_components(custom_components))
 
-        # Compile the root document with base styles and fonts.
+        # Iterate through all the custom components and add their imports to the all_imports
+        for component in custom_components:
+            all_imports.update(component.get_imports())
+
+        # Compile the root document with base styles and fonts
         compile_results.append(compiler.compile_document_root(self.stylesheets))
 
         # Compile the theme.
@@ -556,6 +587,9 @@ class App(Base):
         # Empty the .web pages directory
         compiler.purge_web_pages_dir()
 
+        # install frontend packages
+        self.get_frontend_packages(all_imports)
+
         # Write the pages at the end to trigger the NextJS hot reload only once.
         thread_pool = ThreadPool()
         for output_path, code in compile_results:
diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py
index f9a43b708..39d26cca5 100644
--- a/reflex/compiler/utils.py
+++ b/reflex/compiler/utils.py
@@ -25,7 +25,7 @@ from reflex.components.component import Component, ComponentStyle, CustomCompone
 from reflex.state import Cookie, LocalStorage, State
 from reflex.style import Style
 from reflex.utils import format, imports, path_ops
-from reflex.vars import ImportVar
+from reflex.vars import ImportVar, NoRenderImportVar
 
 # To re-export this function.
 merge_imports = imports.merge_imports
@@ -42,6 +42,9 @@ def compile_import_statement(fields: Set[ImportVar]) -> Tuple[str, Set[str]]:
         default: default library. When install "import def from library".
         rest: rest of libraries. When install "import {rest1, rest2} from library"
     """
+    # ignore the NoRenderImportVar fields during compilation
+    fields = {field for field in fields if not isinstance(field, NoRenderImportVar)}
+
     # Check for default imports.
     defaults = {field for field in fields if field.is_default}
     assert len(defaults) < 2
@@ -72,7 +75,8 @@ def validate_imports(imports: imports.ImportDict):
                 raise ValueError(
                     f"Can not compile, the tag {import_name} is used multiple time from {lib} and {used_tags[import_name]}"
                 )
-            used_tags[import_name] = lib
+            if import_name is not None:
+                used_tags[import_name] = lib
 
 
 def compile_imports(imports: imports.ImportDict) -> List[dict]:
@@ -87,6 +91,10 @@ def compile_imports(imports: imports.ImportDict) -> List[dict]:
     import_dicts = []
     for lib, fields in imports.items():
         default, rest = compile_import_statement(fields)
+        # prevent all NoRenderImportVar from being rendered on the page
+        if any({f for f in fields if isinstance(f, NoRenderImportVar)}):  # type: ignore
+            continue
+
         if not lib:
             assert not default, "No default field allowed for empty library."
             assert rest is not None and len(rest) > 0, "No fields to import."
@@ -94,6 +102,11 @@ def compile_imports(imports: imports.ImportDict) -> List[dict]:
                 import_dicts.append(get_import_dict(module))
             continue
 
+        # remove the version before rendering the package imports
+        lib, at, version = lib.rpartition("@")
+        if not lib:
+            lib = at + version
+
         import_dicts.append(get_import_dict(lib, default, rest))
     return import_dicts
 
diff --git a/reflex/components/component.py b/reflex/components/component.py
index 415fd3358..5888dfc2a 100644
--- a/reflex/components/component.py
+++ b/reflex/components/component.py
@@ -22,7 +22,7 @@ from reflex.event import (
 )
 from reflex.style import Style
 from reflex.utils import format, imports, types
-from reflex.vars import BaseVar, ImportVar, Var
+from reflex.vars import BaseVar, ImportVar, NoRenderImportVar, Var
 
 
 class Component(Base, ABC):
@@ -40,6 +40,9 @@ class Component(Base, ABC):
     # The library that the component is based on.
     library: Optional[str] = None
 
+    # List here the non-react dependency needed by `library`
+    lib_dependencies: List[str] = []
+
     # The tag to use when rendering the component.
     tag: Optional[str] = None
 
@@ -515,9 +518,12 @@ class Component(Base, ABC):
         return code
 
     def _get_imports(self) -> imports.ImportDict:
+        imports = {}
         if self.library is not None and self.tag is not None:
-            return {self.library: {self.import_var}}
-        return {}
+            imports[self.library] = {self.import_var}
+        for dep in self.lib_dependencies:
+            imports[dep] = {NoRenderImportVar()}  # type: ignore
+        return imports
 
     def get_imports(self) -> imports.ImportDict:
         """Get all the libraries and fields that are used by the component.
@@ -833,11 +839,26 @@ class NoSSRComponent(Component):
     """A dynamic component that is not rendered on the server."""
 
     def _get_imports(self):
-        return {"next/dynamic": {ImportVar(tag="dynamic", is_default=True)}}
+        imports = {"next/dynamic": {ImportVar(tag="dynamic", is_default=True)}}
+
+        for dep in [self.library, *self.lib_dependencies]:
+            imports[dep] = {NoRenderImportVar()}  # type: ignore
+
+        return imports
 
     def _get_custom_code(self) -> str:
         opts_fragment = ", { ssr: false });"
-        library_import = f"const {self.tag} = dynamic(() => import('{self.library}')"
+
+        # extract the correct import name from library name
+        if self.library is None:
+            raise ValueError("Undefined library for NoSSRComponent")
+
+        import_name_parts = [p for p in self.library.rpartition("@") if p != ""]
+        import_name = (
+            import_name_parts[0] if import_name_parts[0] != "@" else self.library
+        )
+
+        library_import = f"const {self.tag} = dynamic(() => import('{import_name}')"
         mod_import = (
             # https://nextjs.org/docs/pages/building-your-application/optimizing/lazy-loading#with-named-exports
             f".then((mod) => mod.{self.tag})"
diff --git a/reflex/components/datadisplay/code.py b/reflex/components/datadisplay/code.py
index 4453f75c7..27c44ac7c 100644
--- a/reflex/components/datadisplay/code.py
+++ b/reflex/components/datadisplay/code.py
@@ -19,7 +19,7 @@ PRISM_STYLES_PATH = "/styles/code/prism"
 class CodeBlock(Component):
     """A code block."""
 
-    library = "react-syntax-highlighter"
+    library = "react-syntax-highlighter@^15.5.0"
 
     tag = "Prism"
 
diff --git a/reflex/components/datadisplay/datatable.py b/reflex/components/datadisplay/datatable.py
index 2fe652873..13437a6b4 100644
--- a/reflex/components/datadisplay/datatable.py
+++ b/reflex/components/datadisplay/datatable.py
@@ -11,7 +11,9 @@ from reflex.vars import BaseVar, ComputedVar, ImportVar, Var
 class Gridjs(Component):
     """A component that wraps a nivo bar component."""
 
-    library = "gridjs-react"
+    library = "gridjs-react@^6.0.1"
+
+    lib_dependencies: List[str] = ["gridjs@^6.0.6"]
 
 
 class DataTable(Gridjs):
diff --git a/reflex/components/forms/debounce.py b/reflex/components/forms/debounce.py
index 6fde2c6f0..e5df504b2 100644
--- a/reflex/components/forms/debounce.py
+++ b/reflex/components/forms/debounce.py
@@ -16,7 +16,7 @@ class DebounceInput(Component):
     is experiencing high latency.
     """
 
-    library = "react-debounce-input"
+    library = "react-debounce-input@^3.3.0"
     tag = "DebounceInput"
 
     # Minimum input characters before triggering the on_change event
diff --git a/reflex/components/forms/upload.py b/reflex/components/forms/upload.py
index 344596618..a0e91d7ed 100644
--- a/reflex/components/forms/upload.py
+++ b/reflex/components/forms/upload.py
@@ -21,7 +21,7 @@ clear_selected_files = BaseVar(name="_e => setFiles((files) => [])", type_=Event
 class Upload(Component):
     """A file upload component."""
 
-    library = "react-dropzone"
+    library = "react-dropzone@^14.2.3"
 
     tag = "ReactDropzone"
 
diff --git a/reflex/components/graphing/plotly.py b/reflex/components/graphing/plotly.py
index 654cf6ee9..3a0d46965 100644
--- a/reflex/components/graphing/plotly.py
+++ b/reflex/components/graphing/plotly.py
@@ -1,6 +1,6 @@
 """Component for displaying a plotly graph."""
 
-from typing import Dict
+from typing import Dict, List
 
 from plotly.graph_objects import Figure
 
@@ -12,7 +12,9 @@ from reflex.vars import Var
 class PlotlyLib(NoSSRComponent):
     """A component that wraps a plotly lib."""
 
-    library = "react-plotly.js"
+    library = "react-plotly.js@^2.6.0"
+
+    lib_dependencies: List[str] = ["plotly.js@^2.22.0"]
 
 
 class Plotly(PlotlyLib):
diff --git a/reflex/components/graphing/victory.py b/reflex/components/graphing/victory.py
index 96e3c8653..a72f05d52 100644
--- a/reflex/components/graphing/victory.py
+++ b/reflex/components/graphing/victory.py
@@ -334,7 +334,7 @@ def data(graph: str, x: List, y: Optional[List] = None, **kwargs) -> List:
 class Victory(Component):
     """A component that wraps a victory lib."""
 
-    library = "victory"
+    library = "victory@^36.6.8"
 
     # The data to display.
     data: Var[List[Dict]]
diff --git a/reflex/components/libs/react_player.py b/reflex/components/libs/react_player.py
index fef1c583c..73e5066b3 100644
--- a/reflex/components/libs/react_player.py
+++ b/reflex/components/libs/react_player.py
@@ -11,7 +11,7 @@ class ReactPlayerComponent(NoSSRComponent):
     reference: https://github.com/cookpete/react-player.
     """
 
-    library = "react-player/lazy"
+    library = "react-player@^2.12.0"
 
     tag = "ReactPlayer"
 
diff --git a/reflex/components/media/icon.py b/reflex/components/media/icon.py
index 6f67d7c57..45527096e 100644
--- a/reflex/components/media/icon.py
+++ b/reflex/components/media/icon.py
@@ -7,7 +7,7 @@ from reflex.utils import format
 class ChakraIconComponent(Component):
     """A component that wraps a Chakra icon component."""
 
-    library = "@chakra-ui/icons"
+    library = "@chakra-ui/icons@^2.0.19"
 
 
 class Icon(ChakraIconComponent):
diff --git a/reflex/components/typography/markdown.py b/reflex/components/typography/markdown.py
index 743e46789..cedebffb0 100644
--- a/reflex/components/typography/markdown.py
+++ b/reflex/components/typography/markdown.py
@@ -32,7 +32,14 @@ components_by_tag: Dict[str, Callable] = {
 class Markdown(Component):
     """A markdown component."""
 
-    library = "react-markdown"
+    library = "react-markdown@^8.0.7"
+
+    lib_dependencies: List[str] = [
+        "rehype-katex@^6.0.3",
+        "remark-math@^5.1.1",
+        "rehype-raw@^6.1.1",
+        "remark-gfm@^3.0.1",
+    ]
 
     tag = "ReactMarkdown"
 
diff --git a/reflex/py.typed b/reflex/py.typed
deleted file mode 100644
index e69de29bb..000000000
diff --git a/reflex/reflex.py b/reflex/reflex.py
index daf929dd5..9f662c4e1 100644
--- a/reflex/reflex.py
+++ b/reflex/reflex.py
@@ -127,6 +127,9 @@ def run(
         frontend = True
         backend = True
 
+    if not frontend and backend:
+        _skip_compile()
+
     # Check that the app is initialized.
     prerequisites.check_initialized(frontend=frontend)
 
diff --git a/reflex/utils/build.py b/reflex/utils/build.py
index ee47d3f71..41f19b0f6 100644
--- a/reflex/utils/build.py
+++ b/reflex/utils/build.py
@@ -223,9 +223,6 @@ def setup_frontend(
         root: The root path of the project.
         disable_telemetry: Whether to disable the Next telemetry.
     """
-    # Install frontend packages.
-    prerequisites.install_frontend_packages()
-
     # Copy asset files to public folder.
     path_ops.cp(
         src=str(root / constants.APP_ASSETS_DIR),
diff --git a/reflex/utils/exec.py b/reflex/utils/exec.py
index 44f986004..f5f0f362a 100644
--- a/reflex/utils/exec.py
+++ b/reflex/utils/exec.py
@@ -2,11 +2,14 @@
 
 from __future__ import annotations
 
+import hashlib
+import json
 import os
 import platform
 import sys
 from pathlib import Path
 
+import psutil
 import uvicorn
 
 from reflex import constants
@@ -25,20 +28,78 @@ def start_watching_assets_folder(root):
     asset_watch.start()
 
 
+def detect_package_change(json_file_path: str) -> str:
+    """Calculates the SHA-256 hash of a JSON file and returns it as a hexadecimal string.
+
+    Args:
+        json_file_path (str): The path to the JSON file to be hashed.
+
+    Returns:
+        str: The SHA-256 hash of the JSON file as a hexadecimal string.
+
+    Example:
+        >>> detect_package_change("package.json")
+        'a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6q7r8s9t0u1v2w3x4y5z6a7b8c9d0e1f2'
+    """
+    with open(json_file_path, "r") as file:
+        json_data = json.load(file)
+
+    # Calculate the hash
+    json_string = json.dumps(json_data, sort_keys=True)
+    hash_object = hashlib.sha256(json_string.encode())
+    return hash_object.hexdigest()
+
+
+def kill(proc_pid: int):
+    """Kills a process and all its child processes.
+
+    Args:
+        proc_pid (int): The process ID of the process to be killed.
+
+    Example:
+        >>> kill(1234)
+    """
+    process = psutil.Process(proc_pid)
+    for proc in process.children(recursive=True):
+        proc.kill()
+    process.kill()
+
+
 def run_process_and_launch_url(run_command: list[str]):
     """Run the process and launch the URL.
 
     Args:
         run_command: The command to run.
     """
-    process = processes.new_process(
-        run_command, cwd=constants.WEB_DIR, shell=constants.IS_WINDOWS
-    )
+    json_file_path = os.path.join(constants.WEB_DIR, "package.json")
+    last_hash = detect_package_change(json_file_path)
+    process = None
+    first_run = True
 
-    for line in processes.stream_logs("Starting frontend", process):
-        if "ready started server on" in line:
-            url = line.split("url: ")[-1].strip()
-            console.print(f"App running at: [bold green]{url}")
+    while True:
+        if process is None:
+            process = processes.new_process(
+                run_command, cwd=constants.WEB_DIR, shell=constants.IS_WINDOWS
+            )
+        if process.stdout:
+            for line in processes.stream_logs("Starting frontend", process):
+                if "ready started server on" in line:
+                    if first_run:
+                        url = line.split("url: ")[-1].strip()
+                        console.print(f"App running at: [bold green]{url}")
+                        first_run = False
+                    else:
+                        console.print(f"New packages detected updating app...")
+                else:
+                    console.debug(line)
+                    new_hash = detect_package_change(json_file_path)
+                    if new_hash != last_hash:
+                        last_hash = new_hash
+                        kill(process.pid)
+                        process = None
+                        break  # for line in process.stdout
+        if process is not None:
+            break  # while True
 
 
 def run_frontend(root: Path, port: str):
diff --git a/reflex/utils/prerequisites.py b/reflex/utils/prerequisites.py
index 7ad105896..fefea00f3 100644
--- a/reflex/utils/prerequisites.py
+++ b/reflex/utils/prerequisites.py
@@ -14,7 +14,7 @@ import zipfile
 from fileinput import FileInput
 from pathlib import Path
 from types import ModuleType
-from typing import Optional
+from typing import List, Optional
 
 import httpx
 import typer
@@ -376,25 +376,32 @@ def install_bun():
     )
 
 
-def install_frontend_packages():
-    """Installs the base and custom frontend packages."""
+def install_frontend_packages(packages: List[str]):
+    """Installs the base and custom frontend packages.
+
+    Args:
+        packages (List[str]): A list of package names to be installed.
+
+    Example:
+        >>> install_frontend_packages(["react", "react-dom"])
+    """
     # Install the base packages.
     process = processes.new_process(
         [get_install_package_manager(), "install", "--loglevel", "silly"],
         cwd=constants.WEB_DIR,
         shell=constants.IS_WINDOWS,
     )
+
     processes.show_status("Installing base frontend packages", process)
 
-    # Install the app packages.
-    packages = get_config().frontend_packages
+    # Install the custom packages, if any.
     if len(packages) > 0:
         process = processes.new_process(
             [get_install_package_manager(), "add", *packages],
             cwd=constants.WEB_DIR,
             shell=constants.IS_WINDOWS,
         )
-        processes.show_status("Installing custom frontend packages", process)
+        processes.show_status("Installing frontend packages for components", process)
 
 
 def check_initialized(frontend: bool = True):
diff --git a/reflex/vars.py b/reflex/vars.py
index 14feff119..8a87454f9 100644
--- a/reflex/vars.py
+++ b/reflex/vars.py
@@ -1372,6 +1372,12 @@ class ImportVar(Base):
         return hash((self.tag, self.is_default, self.alias))
 
 
+class NoRenderImportVar(ImportVar):
+    """A import that doesn't need to be rendered."""
+
+    ...
+
+
 def get_local_storage(key: Optional[Union[Var, str]] = None) -> BaseVar:
     """Provide a base var as payload to get local storage item(s).