Run AppHarness selenium integration tests in CI (#1538)

.github/workflows/integration_app_harness.yml

@@ -0,0 +1,37 @@
+name: integration-app-harness
+
+on:
+  push:
+    branches: [ "main" ]
+    paths-ignore:
+      - '**/*.md'
+  pull_request:
+    branches: [ "main" ]
+    paths-ignore:
+      - '**/*.md'
+
+permissions:
+  contents: read
+
+jobs:
+  integration-app-harness:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: ./.github/actions/setup_build_env
+      with:
+          python-version: "3.11"
+          run-poetry-install: true
+          create-venv-at-path: .venv
+    - run: poetry run pip install pyvirtualdisplay pillow
+    - name: Run app harness tests
+      env:
+        SCREENSHOT_DIR: /tmp/screenshots
+      run: |
+        poetry run pytest integration
+    - uses: actions/upload-artifact@v3
+      name: Upload failed test screenshots
+      if: always()
+      with:
+        name: failed_test_screenshots
+        path: /tmp/screenshots

integration/conftest.py

@@ -0,0 +1,56 @@
+"""Shared conftest for all integration tests."""
+import os
+import re
+from pathlib import Path
+
+import pytest
+
+DISPLAY = None
+XVFB_DIMENSIONS = (800, 600)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def xvfb():
+    """Create virtual X display.
+
+    This function is a no-op unless GITHUB_ACTIONS is set in the environment.
+
+    Yields:
+        the pyvirtualdisplay object that the browser will be open on
+    """
+    if os.environ.get("GITHUB_ACTIONS"):
+        from pyvirtualdisplay.smartdisplay import (  # pyright: ignore [reportMissingImports]
+            SmartDisplay,
+        )
+
+        global DISPLAY
+        with SmartDisplay(visible=0, size=XVFB_DIMENSIONS) as DISPLAY:
+            yield DISPLAY
+        DISPLAY = None
+    else:
+        yield None
+
+
+def pytest_exception_interact(node, call, report):
+    """Take and upload screenshot when tests fail.
+
+    Args:
+        node: The pytest item that failed.
+        call: The pytest call describing when/where the test was invoked.
+        report: The pytest log report object.
+    """
+    screenshot_dir = os.environ.get("SCREENSHOT_DIR")
+    if DISPLAY is None or screenshot_dir is None:
+        return
+
+    screenshot_dir = Path(screenshot_dir)
+    screenshot_dir.mkdir(parents=True, exist_ok=True)
+    safe_filename = re.sub(
+        r"(?u)[^-\w.]",
+        "_",
+        str(node.nodeid).strip().replace(" ", "_"),
+    )
+
+    DISPLAY.waitgrab().save(
+        (Path(screenshot_dir) / safe_filename).with_suffix(".png"),
+    )

integration/test_input.py

@@ -96,21 +96,21 @@ async def test_fully_controlled_input(fully_controlled_input: AppHarness):
 
     # type more characters
     debounce_input.send_keys("getting testing done")
-    time.sleep(0.1)
+    time.sleep(0.2)
     assert debounce_input.get_attribute("value") == "getting testing done"
     assert backend_state.text == "getting testing done"
     assert fully_controlled_input.poll_for_value(value_input) == "getting testing done"
 
     # type into the on_change input
     on_change_input.send_keys("overwrite the state")
-    time.sleep(0.1)
+    time.sleep(0.2)
     assert debounce_input.get_attribute("value") == "overwrite the state"
     assert on_change_input.get_attribute("value") == "overwrite the state"
     assert backend_state.text == "overwrite the state"
     assert fully_controlled_input.poll_for_value(value_input) == "overwrite the state"
 
     clear_button.click()
-    time.sleep(0.1)
+    time.sleep(0.2)
     assert on_change_input.get_attribute("value") == ""
     # potential bug: clearing the on_change field doesn't itself trigger on_change
     # assert backend_state.text == ""

reflex/config.py

@@ -261,17 +261,23 @@ class Config(Base):
         return urllib.parse.urlsplit(event_url).path
 
 
-def get_config() -> Config:
+def get_config(reload: bool = False) -> Config:
     """Get the app config.
 
+    Args:
+        reload: Re-import the rxconfig module from disk
+
     Returns:
         The app config.
     """
     from reflex.config import Config
 
-    sys.path.append(os.getcwd())
+    sys.path.insert(0, os.getcwd())
     try:
-        return __import__(constants.CONFIG_MODULE).config
+        rxconfig = __import__(constants.CONFIG_MODULE)
+        if reload:
+            importlib.reload(rxconfig)
+        return rxconfig.config
 
     except ImportError:
         return Config(app_name="")  # type: ignore

reflex/testing.py

@@ -155,6 +155,8 @@ class AppHarness:
                 )
                 self.app_module_path.write_text(source_code)
         with chdir(self.app_path):
+            # ensure config is reloaded when testing different app
+            reflex.config.get_config(reload=True)
             self.app_module = reflex.utils.prerequisites.get_app()
         self.app_instance = self.app_module.app