diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 9395e8729..c8ab75adb 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -18,7 +18,6 @@ env: PYTHONIOENCODING: 'utf8' TELEMETRY_ENABLED: false NODE_OPTIONS: '--max_old_space_size=4096' - DATABASE_URL: ${{ secrets.DATABASE_URL }} PR_TITLE: ${{ github.event.pull_request.title }} jobs: @@ -62,17 +61,16 @@ jobs: run: | # Check that npm is home npm -v - poetry run bash scripts/benchmarks/benchmarks.sh ./reflex-web prod + poetry run bash benchmarks/lighthouse.sh ./reflex-web prod env: LHCI_GITHUB_APP_TOKEN: $ - name: Run Benchmarks # Only run if the database creds are available in this context. - if: ${{ env.DATABASE_URL }} - run: poetry run python scripts/benchmarks/lighthouse_score_upload.py "$GITHUB_SHA" ./integration/benchmarks/.lighthouseci + run: poetry run python benchmarks/benchmark_lighthouse.py "$GITHUB_SHA" ./integration/benchmarks/.lighthouseci env: GITHUB_SHA: ${{ github.sha }} - simple-apps-benchmarks: + simple-apps-benchmarks: # This app tests the compile times of various compoonents and pages if: github.event.pull_request.merged == true env: OUTPUT_FILE: benchmarks.json @@ -116,8 +114,6 @@ jobs: python-version: ${{ matrix.python-version }} run-poetry-install: true create-venv-at-path: .venv - - name: Install additional dependencies for DB access - run: poetry run uv pip install psycopg2-binary - name: Run benchmark tests env: APP_HARNESS_HEADLESS: 1 @@ -126,16 +122,14 @@ jobs: poetry run pytest -v benchmarks/ --benchmark-json=${{ env.OUTPUT_FILE }} -s - name: Upload benchmark results # Only run if the database creds are available in this context. - if: ${{ env.DATABASE_URL }} run: - poetry run python scripts/benchmarks/simple_app_benchmark_upload.py --os "${{ matrix.os }}" + poetry run python benchmarks/benchmark_compile_times.py --os "${{ matrix.os }}" --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}" - --benchmark-json "${{ env.OUTPUT_FILE }}" - --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}" - --event-type "${{ github.event_name }}" --actor "${{ github.actor }}" --pr-id "${{ github.event.pull_request.id }}" + --benchmark-json "${{ env.OUTPUT_FILE }}" --branch-name "${{ github.head_ref || github.ref_name }}" + --event-type "${{ github.event_name }}" --pr-id "${{ github.event.pull_request.id }}" - reflex-build-size: - if: github.event.pull_request.merged == true + reflex-dist-size: # This job is used to calculate the size of the Reflex distribution (wheel file) + if: github.event.pull_request.merged == true timeout-minutes: 30 strategy: # Prioritize getting more information out of the workflow (even if something fails) @@ -148,22 +142,19 @@ jobs: python-version: 3.11.5 run-poetry-install: true create-venv-at-path: .venv - - name: Install additional dependencies for DB access - run: poetry run uv pip install psycopg2-binary - name: Build reflex run: | poetry build - name: Upload benchmark results # Only run if the database creds are available in this context. - if: ${{ env.DATABASE_URL }} run: - poetry run python scripts/benchmarks/benchmark_reflex_size.py --os ubuntu-latest + poetry run python benchmarks/benchmark_package_size.py --os ubuntu-latest --python-version 3.11.5 --commit-sha "${{ github.sha }}" --pr-id "${{ github.event.pull_request.id }}" - --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}" - --measurement-type "reflex-build" --path ./dist + --branch-name "${{ github.head_ref || github.ref_name }}" + --path ./dist - reflex-plus-dependency-size: - if: github.event.pull_request.merged == true + reflex-venv-size: # This job calculates the total size of Reflex and its dependencies + if: github.event.pull_request.merged == true timeout-minutes: 30 strategy: # Prioritize getting more information out of the workflow (even if something fails) @@ -197,14 +188,10 @@ jobs: run: | poetry run pip install uv - - name: Install additional dependencies for DB access - run: poetry run uv pip install psycopg2-binary - - - if: ${{ env.DATABASE_URL }} - name: calculate and upload size + - name: calculate and upload size run: - poetry run python scripts/benchmarks/benchmark_reflex_size.py --os "${{ matrix.os }}" + poetry run python benchmarks/benchmark_package_size.py --os "${{ matrix.os }}" --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}" - --pr-id "${{ github.event.pull_request.id }}" --db-url "${{ env.DATABASE_URL }}" + --pr-id "${{ github.event.pull_request.id }}" --branch-name "${{ github.head_ref || github.ref_name }}" - --measurement-type "reflex-package" --path ./.venv + --path ./.venv \ No newline at end of file diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 0febb49e2..8fa787aba 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -30,7 +30,6 @@ env: PYTHONIOENCODING: 'utf8' TELEMETRY_ENABLED: false NODE_OPTIONS: '--max_old_space_size=4096' - DATABASE_URL: ${{ secrets.DATABASE_URL }} PR_TITLE: ${{ github.event.pull_request.title }} jobs: @@ -100,27 +99,25 @@ jobs: npm -v poetry run bash scripts/integration.sh ./reflex-examples/counter dev - name: Measure and upload .web size - if: ${{ env.DATABASE_URL}} run: - poetry run python scripts/benchmarks/benchmark_reflex_size.py --os "${{ matrix.os }}" + poetry run python benchmarks/benchmark_web_size.py --os "${{ matrix.os }}" --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}" - --pr-id "${{ github.event.pull_request.id }}" --db-url "${{ env.DATABASE_URL }}" + --pr-id "${{ github.event.pull_request.id }}" --branch-name "${{ github.head_ref || github.ref_name }}" - --measurement-type "counter-app-dot-web" --path ./reflex-examples/counter/.web + --path ./reflex-examples/counter/.web + --app-name "counter" - name: Install hyperfine run: cargo install hyperfine - name: Benchmark imports working-directory: ./reflex-examples/counter run: hyperfine --warmup 3 "export POETRY_VIRTUALENVS_PATH=../../.venv; poetry run python counter/counter.py" --show-output --export-json "${{ env.OUTPUT_FILE }}" --shell bash - name: Upload Benchmarks - if : ${{ env.DATABASE_URL }} run: - poetry run python scripts/benchmarks/benchmark_imports.py --os "${{ matrix.os }}" + poetry run python benchmarks/benchmark_imports.py --os "${{ matrix.os }}" --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}" --benchmark-json "./reflex-examples/counter/${{ env.OUTPUT_FILE }}" - --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}" - --event-type "${{ github.event_name }}" --actor "${{ github.actor }}" --pr-id "${{ github.event.pull_request.id }}" - + --branch-name "${{ github.head_ref || github.ref_name }}" --pr-id "${{ github.event.pull_request.id }}" + --app-name "counter" @@ -164,10 +161,8 @@ jobs: npm -v poetry run bash scripts/integration.sh ./reflex-web prod - name: Measure and upload .web size - if: ${{ env.DATABASE_URL}} run: - poetry run python scripts/benchmarks/benchmark_reflex_size.py --os "${{ matrix.os }}" + poetry run python benchmarks/benchmark_web_size.py --os "${{ matrix.os }}" --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}" - --pr-id "${{ github.event.pull_request.id }}" - --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}" - --measurement-type "reflex-web-dot-web" --path ./reflex-web/.web + --pr-id "${{ github.event.pull_request.id }}" --branch-name "${{ github.head_ref || github.ref_name }}" + --app-name "reflex-web" --path ./reflex-web/.web \ No newline at end of file diff --git a/scripts/benchmarks/simple_app_benchmark_upload.py b/benchmarks/benchmark_compile_times.py similarity index 71% rename from scripts/benchmarks/simple_app_benchmark_upload.py rename to benchmarks/benchmark_compile_times.py index 7edb73fab..2273bd5c8 100644 --- a/scripts/benchmarks/simple_app_benchmark_upload.py +++ b/benchmarks/benchmark_compile_times.py @@ -1,13 +1,12 @@ -"""Runs the benchmarks and inserts the results into the database.""" +"""Extracts the compile times from the JSON files in the specified directory and inserts them into the database.""" from __future__ import annotations import argparse import json import os -from datetime import datetime -import psycopg2 +from utils import send_data_to_posthog def extract_stats_from_json(json_file: str) -> list[dict]: @@ -51,7 +50,6 @@ def extract_stats_from_json(json_file: str) -> list[dict]: def insert_benchmarking_data( - db_connection_url: str, os_type_version: str, python_version: str, performance_data: list[dict], @@ -59,52 +57,33 @@ def insert_benchmarking_data( pr_title: str, branch_name: str, event_type: str, - actor: str, pr_id: str, ): """Insert the benchmarking data into the database. Args: - db_connection_url: The URL to connect to the database. os_type_version: The OS type and version to insert. python_version: The Python version to insert. performance_data: The performance data of reflex web to insert. commit_sha: The commit SHA to insert. pr_title: The PR title to insert. branch_name: The name of the branch. - event_type: Type of github event(push, pull request, etc) - actor: Username of the user that triggered the run. + event_type: Type of github event(push, pull request, etc). pr_id: Id of the PR. """ - # Serialize the JSON data - simple_app_performance_json = json.dumps(performance_data) + # Prepare the event data + properties = { + "os": os_type_version, + "python_version": python_version, + "distinct_id": commit_sha, + "pr_title": pr_title, + "branch_name": branch_name, + "event_type": event_type, + "performance": performance_data, + "pr_id": pr_id, + } - # Get the current timestamp - current_timestamp = datetime.now() - - # Connect to the database and insert the data - with psycopg2.connect(db_connection_url) as conn, conn.cursor() as cursor: - insert_query = """ - INSERT INTO simple_app_benchmarks (os, python_version, commit_sha, time, pr_title, branch_name, event_type, actor, performance, pr_id) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s); - """ - cursor.execute( - insert_query, - ( - os_type_version, - python_version, - commit_sha, - current_timestamp, - pr_title, - branch_name, - event_type, - actor, - simple_app_performance_json, - pr_id, - ), - ) - # Commit the transaction - conn.commit() + send_data_to_posthog("simple_app_benchmark", properties) def main(): @@ -124,11 +103,6 @@ def main(): "--benchmark-json", help="The JSON file containing the benchmark results.", ) - parser.add_argument( - "--db-url", - help="The URL to connect to the database.", - required=True, - ) parser.add_argument( "--pr-title", help="The PR title to insert into the database.", @@ -143,11 +117,6 @@ def main(): help="The github event type", required=True, ) - parser.add_argument( - "--actor", - help="Username of the user that triggered the run.", - required=True, - ) parser.add_argument( "--pr-id", help="ID of the PR.", @@ -162,7 +131,6 @@ def main(): cleaned_benchmark_results = extract_stats_from_json(args.benchmark_json) # Insert the data into the database insert_benchmarking_data( - db_connection_url=args.db_url, os_type_version=args.os, python_version=args.python_version, performance_data=cleaned_benchmark_results, @@ -170,7 +138,6 @@ def main(): pr_title=pr_title, branch_name=args.branch_name, event_type=args.event_type, - actor=args.actor, pr_id=args.pr_id, ) diff --git a/scripts/benchmarks/benchmark_imports.py b/benchmarks/benchmark_imports.py similarity index 63% rename from scripts/benchmarks/benchmark_imports.py rename to benchmarks/benchmark_imports.py index 6258434d6..4706c0cf6 100644 --- a/scripts/benchmarks/benchmark_imports.py +++ b/benchmarks/benchmark_imports.py @@ -1,13 +1,12 @@ -"""Runs the benchmarks and inserts the results into the database.""" +"""Extract and upload benchmarking data to PostHog.""" from __future__ import annotations import argparse import json import os -from datetime import datetime -import psycopg2 +from utils import send_data_to_posthog def extract_stats_from_json(json_file: str) -> dict: @@ -34,59 +33,39 @@ def extract_stats_from_json(json_file: str) -> dict: def insert_benchmarking_data( - db_connection_url: str, os_type_version: str, python_version: str, performance_data: dict, commit_sha: str, pr_title: str, branch_name: str, - event_type: str, - actor: str, pr_id: str, + app_name: str, ): """Insert the benchmarking data into the database. Args: - db_connection_url: The URL to connect to the database. os_type_version: The OS type and version to insert. python_version: The Python version to insert. performance_data: The imports performance data to insert. commit_sha: The commit SHA to insert. pr_title: The PR title to insert. branch_name: The name of the branch. - event_type: Type of github event(push, pull request, etc) - actor: Username of the user that triggered the run. pr_id: Id of the PR. + app_name: The name of the app being measured. """ - # Serialize the JSON data - simple_app_performance_json = json.dumps(performance_data) - # Get the current timestamp - current_timestamp = datetime.now() + properties = { + "os": os_type_version, + "python_version": python_version, + "distinct_id": commit_sha, + "pr_title": pr_title, + "branch_name": branch_name, + "pr_id": pr_id, + "performance": performance_data, + "app_name": app_name, + } - # Connect to the database and insert the data - with psycopg2.connect(db_connection_url) as conn, conn.cursor() as cursor: - insert_query = """ - INSERT INTO import_benchmarks (os, python_version, commit_sha, time, pr_title, branch_name, event_type, actor, performance, pr_id) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s); - """ - cursor.execute( - insert_query, - ( - os_type_version, - python_version, - commit_sha, - current_timestamp, - pr_title, - branch_name, - event_type, - actor, - simple_app_performance_json, - pr_id, - ), - ) - # Commit the transaction - conn.commit() + send_data_to_posthog("import_benchmark", properties) def main(): @@ -106,11 +85,6 @@ def main(): "--benchmark-json", help="The JSON file containing the benchmark results.", ) - parser.add_argument( - "--db-url", - help="The URL to connect to the database.", - required=True, - ) parser.add_argument( "--pr-title", help="The PR title to insert into the database.", @@ -121,13 +95,8 @@ def main(): required=True, ) parser.add_argument( - "--event-type", - help="The github event type", - required=True, - ) - parser.add_argument( - "--actor", - help="Username of the user that triggered the run.", + "--app-name", + help="The name of the app measured.", required=True, ) parser.add_argument( @@ -143,15 +112,13 @@ def main(): cleaned_benchmark_results = extract_stats_from_json(args.benchmark_json) # Insert the data into the database insert_benchmarking_data( - db_connection_url=args.db_url, os_type_version=args.os, python_version=args.python_version, performance_data=cleaned_benchmark_results, commit_sha=args.commit_sha, pr_title=pr_title, branch_name=args.branch_name, - event_type=args.event_type, - actor=args.actor, + app_name=args.app_name, pr_id=args.pr_id, ) diff --git a/scripts/benchmarks/lighthouse_score_upload.py b/benchmarks/benchmark_lighthouse.py similarity index 57% rename from scripts/benchmarks/lighthouse_score_upload.py rename to benchmarks/benchmark_lighthouse.py index 35f32f5d9..72f486b6f 100644 --- a/scripts/benchmarks/lighthouse_score_upload.py +++ b/benchmarks/benchmark_lighthouse.py @@ -1,52 +1,31 @@ -"""Runs the benchmarks and inserts the results into the database.""" +"""Extracts the Lighthouse scores from the JSON files in the specified directory and inserts them into the database.""" from __future__ import annotations import json import os import sys -from datetime import datetime -import psycopg2 +from utils import send_data_to_posthog def insert_benchmarking_data( - db_connection_url: str, lighthouse_data: dict, commit_sha: str, - pr_title: str, ): """Insert the benchmarking data into the database. Args: - db_connection_url: The URL to connect to the database. lighthouse_data: The Lighthouse data to insert. commit_sha: The commit SHA to insert. - pr_title: The PR title to insert. """ - # Serialize the JSON data - lighthouse_json = json.dumps(lighthouse_data) + properties = { + "distinct_id": commit_sha, + "lighthouse_data": lighthouse_data, + } - # Get the current timestamp - current_timestamp = datetime.now() - - # Connect to the database and insert the data - with psycopg2.connect(db_connection_url) as conn, conn.cursor() as cursor: - insert_query = """ - INSERT INTO benchmarks (lighthouse, commit_sha, pr_title, time) - VALUES (%s, %s, %s, %s); - """ - cursor.execute( - insert_query, - ( - lighthouse_json, - commit_sha, - pr_title, - current_timestamp, - ), - ) - # Commit the transaction - conn.commit() + # Send the data to PostHog + send_data_to_posthog("lighthouse_benchmark", properties) def get_lighthouse_scores(directory_path: str) -> dict: @@ -67,7 +46,7 @@ def get_lighthouse_scores(directory_path: str) -> dict: with open(file_path, "r") as file: data = json.load(file) # Extract scores and add them to the dictionary with the filename as key - scores[data["finalUrl"].replace("http://localhost:3000/", "")] = { + scores[data["finalUrl"].replace("http://localhost:3000/", "/")] = { "performance_score": data["categories"]["performance"]["score"], "accessibility_score": data["categories"]["accessibility"][ "score" @@ -76,11 +55,9 @@ def get_lighthouse_scores(directory_path: str) -> dict: "score" ], "seo_score": data["categories"]["seo"]["score"], - "pwa_score": data["categories"]["pwa"]["score"], } except Exception as e: - print(e) - return {"error": "Error parsing JSON files"} + return {"error": e} return scores @@ -91,18 +68,11 @@ def main(): commit_sha = sys.argv[1] json_dir = sys.argv[2] - # Get the PR title and database URL from the environment variables - pr_title = os.environ.get("PR_TITLE") - db_url = os.environ.get("DATABASE_URL") - - if db_url is None or pr_title is None: - sys.exit("Missing environment variables") - # Get the Lighthouse scores lighthouse_scores = get_lighthouse_scores(json_dir) # Insert the data into the database - insert_benchmarking_data(db_url, lighthouse_scores, commit_sha, pr_title) + insert_benchmarking_data(lighthouse_scores, commit_sha) if __name__ == "__main__": diff --git a/scripts/benchmarks/benchmark_reflex_size.py b/benchmarks/benchmark_package_size.py similarity index 51% rename from scripts/benchmarks/benchmark_reflex_size.py rename to benchmarks/benchmark_package_size.py index 3787007cc..8e2704355 100644 --- a/scripts/benchmarks/benchmark_reflex_size.py +++ b/benchmarks/benchmark_package_size.py @@ -1,53 +1,9 @@ -"""Checks the size of a specific directory and uploads result.""" +"""Checks the size of a specific directory and uploads result to Posthog.""" import argparse import os -import subprocess -from datetime import datetime -import psycopg2 - - -def get_directory_size(directory): - """Get the size of a directory in bytes. - - Args: - directory: The directory to check. - - Returns: - The size of the dir in bytes. - """ - total_size = 0 - for dirpath, _, filenames in os.walk(directory): - for f in filenames: - fp = os.path.join(dirpath, f) - total_size += os.path.getsize(fp) - return total_size - - -def get_python_version(venv_path, os_name): - """Get the python version of python in a virtual env. - - Args: - venv_path: Path to virtual environment. - os_name: Name of os. - - Returns: - The python version. - """ - python_executable = ( - os.path.join(venv_path, "bin", "python") - if "windows" not in os_name - else os.path.join(venv_path, "Scripts", "python.exe") - ) - try: - output = subprocess.check_output( - [python_executable, "--version"], stderr=subprocess.STDOUT - ) - python_version = output.decode("utf-8").strip().split()[1] - return ".".join(python_version.split(".")[:-1]) - except subprocess.CalledProcessError: - return None +from utils import get_directory_size, get_python_version, send_data_to_posthog def get_package_size(venv_path, os_name): @@ -64,6 +20,7 @@ def get_package_size(venv_path, os_name): ValueError: when venv does not exist or python version is None. """ python_version = get_python_version(venv_path, os_name) + print("Python version:", python_version) if python_version is None: raise ValueError("Error: Failed to determine Python version.") @@ -86,61 +43,45 @@ def get_package_size(venv_path, os_name): def insert_benchmarking_data( - db_connection_url: str, os_type_version: str, python_version: str, - measurement_type: str, commit_sha: str, pr_title: str, branch_name: str, pr_id: str, path: str, ): - """Insert the benchmarking data into the database. + """Insert the benchmarking data into PostHog. Args: - db_connection_url: The URL to connect to the database. os_type_version: The OS type and version to insert. python_version: The Python version to insert. - measurement_type: The type of metric to measure. commit_sha: The commit SHA to insert. pr_title: The PR title to insert. branch_name: The name of the branch. pr_id: The id of the PR. path: The path to the dir or file to check size. """ - if measurement_type == "reflex-package": - size = get_package_size(path, os_type_version) - else: + if "./dist" in path: size = get_directory_size(path) + else: + size = get_package_size(path, os_type_version) - # Get the current timestamp - current_timestamp = datetime.now() + # Prepare the event data + properties = { + "path": path, + "os": os_type_version, + "python_version": python_version, + "distinct_id": commit_sha, + "pr_title": pr_title, + "branch_name": branch_name, + "pr_id": pr_id, + "size_mb": round( + size / (1024 * 1024), 3 + ), # save size in MB and round to 3 places + } - # Connect to the database and insert the data - with psycopg2.connect(db_connection_url) as conn, conn.cursor() as cursor: - insert_query = """ - INSERT INTO size_benchmarks (os, python_version, commit_sha, created_at, pr_title, branch_name, pr_id, measurement_type, size) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s); - """ - cursor.execute( - insert_query, - ( - os_type_version, - python_version, - commit_sha, - current_timestamp, - pr_title, - branch_name, - pr_id, - measurement_type, - round( - size / (1024 * 1024), 3 - ), # save size in mb and round to 3 places. - ), - ) - # Commit the transaction - conn.commit() + send_data_to_posthog("package_size", properties) def main(): @@ -155,11 +96,6 @@ def main(): parser.add_argument( "--commit-sha", help="The commit SHA to insert into the database." ) - parser.add_argument( - "--db-url", - help="The URL to connect to the database.", - required=True, - ) parser.add_argument( "--pr-title", help="The PR title to insert into the database.", @@ -174,14 +110,9 @@ def main(): help="The pr id", required=True, ) - parser.add_argument( - "--measurement-type", - help="The type of metric to be checked.", - required=True, - ) parser.add_argument( "--path", - help="the current path to check size.", + help="The path to the vnenv.", required=True, ) args = parser.parse_args() @@ -191,10 +122,8 @@ def main(): # Insert the data into the database insert_benchmarking_data( - db_connection_url=args.db_url, os_type_version=args.os, python_version=args.python_version, - measurement_type=args.measurement_type, commit_sha=args.commit_sha, pr_title=pr_title, branch_name=args.branch_name, diff --git a/benchmarks/benchmark_web_size.py b/benchmarks/benchmark_web_size.py new file mode 100644 index 000000000..6c2f40bbc --- /dev/null +++ b/benchmarks/benchmark_web_size.py @@ -0,0 +1,105 @@ +"""Checks the size of a specific directory and uploads result to Posthog.""" + +import argparse +import os + +from utils import get_directory_size, send_data_to_posthog + + +def insert_benchmarking_data( + os_type_version: str, + python_version: str, + app_name: str, + commit_sha: str, + pr_title: str, + branch_name: str, + pr_id: str, + path: str, +): + """Insert the benchmarking data into PostHog. + + Args: + app_name: The name of the app being measured. + os_type_version: The OS type and version to insert. + python_version: The Python version to insert. + commit_sha: The commit SHA to insert. + pr_title: The PR title to insert. + branch_name: The name of the branch. + pr_id: The id of the PR. + path: The path to the dir or file to check size. + """ + size = get_directory_size(path) + + # Prepare the event data + properties = { + "app_name": app_name, + "os": os_type_version, + "python_version": python_version, + "distinct_id": commit_sha, + "pr_title": pr_title, + "branch_name": branch_name, + "pr_id": pr_id, + "size_mb": round( + size / (1024 * 1024), 3 + ), # save size in MB and round to 3 places + } + + send_data_to_posthog("web-size", properties) + + +def main(): + """Runs the benchmarks and inserts the results.""" + parser = argparse.ArgumentParser(description="Run benchmarks and process results.") + parser.add_argument( + "--os", help="The OS type and version to insert into the database." + ) + parser.add_argument( + "--python-version", help="The Python version to insert into the database." + ) + parser.add_argument( + "--commit-sha", help="The commit SHA to insert into the database." + ) + parser.add_argument( + "--pr-title", + help="The PR title to insert into the database.", + ) + parser.add_argument( + "--branch-name", + help="The current branch", + required=True, + ) + parser.add_argument( + "--app-name", + help="The name of the app measured.", + required=True, + ) + parser.add_argument( + "--pr-id", + help="The pr id", + required=True, + ) + parser.add_argument( + "--path", + help="The current path to app to check.", + required=True, + ) + args = parser.parse_args() + + # Get the PR title from env or the args. For the PR merge or push event, there is no PR title, leaving it empty. + pr_title = args.pr_title or os.getenv("PR_TITLE", "") + + # Insert the data into the database + insert_benchmarking_data( + app_name=args.app_name, + os_type_version=args.os, + python_version=args.python_version, + commit_sha=args.commit_sha, + pr_title=pr_title, + branch_name=args.branch_name, + pr_id=args.pr_id, + path=args.path, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmarks/benchmarks.sh b/benchmarks/lighthouse.sh similarity index 100% rename from scripts/benchmarks/benchmarks.sh rename to benchmarks/lighthouse.sh diff --git a/benchmarks/utils.py b/benchmarks/utils.py new file mode 100644 index 000000000..7b02c8cc8 --- /dev/null +++ b/benchmarks/utils.py @@ -0,0 +1,73 @@ +"""Utility functions for the benchmarks.""" + +import os +import subprocess + +import httpx +from httpx import HTTPError + + +def get_python_version(venv_path, os_name): + """Get the python version of python in a virtual env. + + Args: + venv_path: Path to virtual environment. + os_name: Name of os. + + Returns: + The python version. + """ + python_executable = ( + os.path.join(venv_path, "bin", "python") + if "windows" not in os_name + else os.path.join(venv_path, "Scripts", "python.exe") + ) + try: + output = subprocess.check_output( + [python_executable, "--version"], stderr=subprocess.STDOUT + ) + python_version = output.decode("utf-8").strip().split()[1] + return ".".join(python_version.split(".")[:-1]) + except subprocess.CalledProcessError: + return None + + +def get_directory_size(directory): + """Get the size of a directory in bytes. + + Args: + directory: The directory to check. + + Returns: + The size of the dir in bytes. + """ + total_size = 0 + for dirpath, _, filenames in os.walk(directory): + for f in filenames: + fp = os.path.join(dirpath, f) + total_size += os.path.getsize(fp) + return total_size + + +def send_data_to_posthog(event, properties): + """Send data to PostHog. + + Args: + event: The event to send. + properties: The properties to send. + + Raises: + HTTPError: When there is an error sending data to PostHog. + """ + event_data = { + "api_key": "phc_JoMo0fOyi0GQAooY3UyO9k0hebGkMyFJrrCw1Gt5SGb", + "event": event, + "properties": properties, + } + + with httpx.Client() as client: + response = client.post("https://app.posthog.com/capture/", json=event_data) + if response.status_code != 200: + raise HTTPError( + f"Error sending data to PostHog: {response.status_code} - {response.text}" + )