fix: split prod/dev requirements to shrink Heroku slug under 1GB

.github/workflows/linter.yml

@@ -43,7 +43,7 @@ jobs:
           python -m venv venv
           . ./venv/bin/activate
           pip install --upgrade pip setuptools
-          pip install -r requirements.txt
+          pip install -r requirements-dev.txt
       - name: OpenAPI guardrail
         run: make openapi-guardrail
       - name: Lint Code Base

Makefile

@@ -69,7 +69,7 @@ cover:
 install-deps-python:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	pip install --upgrade pip setuptools &&\
-	pip install -r requirements.txt
+	pip install -r requirements-dev.txt
 
 install-deps-typescript:
 	(cd application/frontend && yarn install)

application/cmd/cre_main.py

@@ -9,14 +9,12 @@
 import requests
 
 from collections import deque
-from typing import Any, Callable, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING
 import hashlib
 import json as _json
 from rq import Queue, job, exceptions
 from sqlalchemy import not_
 
-from application.utils.external_project_parsers.base_parser import BaseParser
-from application.utils.external_project_parsers.parsers import master_spreadsheet_parser
 from application import create_app  # type: ignore
 from application.config import CMDConfig
 from application.database import db
@@ -26,11 +24,12 @@
 from application.utils import spreadsheet as sheet_utils
 from application.utils import redis
 from application.utils import db_backend
-from alive_progress import alive_bar
-from application.prompt_client import prompt_client as prompt_client
 from application.utils import gap_analysis
 from application.utils import cres_csv_export
 
+if TYPE_CHECKING:
+    from application.prompt_client import prompt_client as prompt_client
+
 logging.basicConfig()
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -469,6 +468,8 @@ def _standard_structure_fingerprint(resource_name: str) -> str:
         return hashlib.sha256(payload.encode("utf-8")).hexdigest()
 
     conn = redis.connect()
+    from application.prompt_client import prompt_client as prompt_client
+
     ph = prompt_client.PromptHandler(database=collection)
     importing_name = standard_entries[0].name
     effective_calculate_gap_analysis = (
@@ -541,7 +542,7 @@ def _standard_structure_fingerprint(resource_name: str) -> str:
 def parse_standards_from_spreadsheeet(
     cre_file: List[Dict[str, Any]],
     cache_location: str,
-    prompt_handler: prompt_client.PromptHandler,
+    prompt_handler: "prompt_client.PromptHandler",
 ) -> None:
     """given a csv with standards, build a list of standards in the db"""
     if not cre_file:
@@ -568,6 +569,10 @@ def parse_standards_from_spreadsheeet(
         from application.utils import import_pipeline
 
         collection = db_connect(cache_location)
+        from application.utils.external_project_parsers.parsers import (
+            master_spreadsheet_parser,
+        )
+
         parse_result = master_spreadsheet_parser.MasterSpreadsheetParser.parse_rows(
             cre_file
         )
@@ -682,6 +687,8 @@ def download_gap_analysis_from_upstream(cache: str) -> None:
         pairs = [(sa, sb) for sa in standards for sb in standards if sa != sb]
 
         if os.environ.get("BENCHMARK_MODE") == "1":
+            from alive_progress import alive_bar
+
             with alive_bar(len(pairs), title="Fetching upstream Gap Analysis") as bar:
                 for sa, sb in pairs:
                     res = requests.get(
@@ -846,11 +853,15 @@ def backfill_gap_analysis_only(
                 jobs.append(j)
 
             if jobs:
+                from alive_progress import alive_bar
+
                 with alive_bar(
                     len(jobs), title=f"GA batch {i // batch_size + 1}"
                 ) as bar:
                     redis.wait_for_jobs(jobs, bar)
         else:
+            from alive_progress import alive_bar
+
             with alive_bar(len(batch), title=f"GA batch {i // batch_size + 1}") as bar:
                 for sa, sb in batch:
                     _compute_pair_direct(collection, sa, sb)
@@ -901,6 +912,8 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
         cache.delete_nodes(args.delete_resource)
 
     # individual resource importing
+    from application.utils.external_project_parsers.base_parser import BaseParser
+
     if args.zap_in:
         from application.utils.external_project_parsers.parsers import zap_alerts_parser
 
@@ -1014,6 +1027,8 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
 
 
 def ai_client_init(database: db.Node_collection):
+    from application.prompt_client import prompt_client as prompt_client
+
     return prompt_client.PromptHandler(database=database)
 
 
@@ -1053,6 +1068,8 @@ def prepare_for_review(cache: str) -> Tuple[str, str]:
 
 
 def generate_embeddings(db_url: str) -> None:
+    from application.prompt_client import prompt_client as prompt_client
+
     database = db_connect(path=db_url)
     prompt_client.PromptHandler(database, load_all_embeddings=True)
 
@@ -1109,6 +1126,8 @@ def run_librarian(
             "land W8); running in dry-run mode"
         )
 
+    from application.prompt_client import prompt_client as prompt_client
+
     cfg = load_config()
     database = db_connect(path=cache_file)
     ph = prompt_client.PromptHandler(database=database)
@@ -1207,6 +1226,8 @@ def run_librarian(
 
 def regenerate_embeddings(db_url: str) -> None:
     """Wipe all embedding rows, then rebuild (CRE + every node type) like ``--generate_embeddings``."""
+    from application.prompt_client import prompt_client as prompt_client
+
     database = db_connect(path=db_url)
     removed = database.delete_all_embeddings()
     logger.info("Removed %s embedding rows; rebuilding embeddings", removed)

application/prompt_client/prompt_client.py

@@ -2,15 +2,11 @@
 from application.defs import cre_defs
 from datetime import datetime
 from multiprocessing import Pool
-from nltk.corpus import stopwords
-from nltk.tokenize import word_tokenize
 from io import BytesIO
 from urllib.parse import urlparse
 
 from application.prompt_client import embed_alignment
 
-from playwright.sync_api import Error as PlaywrightError
-from playwright.sync_api import TimeoutError as PlaywrightTimeoutError, sync_playwright
 from scipy import sparse
 from sklearn.metrics.pairwise import cosine_similarity
 from typing import Dict, List, Any, Tuple, Optional
@@ -22,7 +18,6 @@
     from pypdf import PdfReader
 except ImportError:
     PdfReader = None  # type: ignore[misc, assignment]
-import nltk
 import numpy as np
 import os
 import json
@@ -235,6 +230,10 @@ def __init__(cls):
 
     # Function to get text content from a URL
     def get_content(self, url) -> Optional[str]:
+        # Playwright is for scrape/import embedding generation only — not chat.
+        from playwright.sync_api import Error as PlaywrightError
+        from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+
         for attempts in range(1, 10):
             if _is_likely_pdf_url(url):
                 text = _fetch_pdf_text_for_embeddings(url)
@@ -298,6 +297,9 @@ def get_content(self, url) -> Optional[str]:
 
     def get_html(self, url) -> Optional[str]:
         """Return raw HTML document string (for smart excerpt alignment). PDF URLs unsupported."""
+        from playwright.sync_api import Error as PlaywrightError
+        from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+
         for attempts in range(1, 10):
             if _is_likely_pdf_url(url):
                 return None
@@ -343,6 +345,8 @@ def _ensure_smart_embed_caches(self) -> None:
             ] = {}
 
     def clean_content(self, content):
+        from nltk.tokenize import word_tokenize  # lazy: scrape/import path only
+
         content = re.sub("\s+", " ", content.strip())
 
         # split into words
@@ -363,6 +367,9 @@ def with_ai_client(self, ai_client):
 
     def setup_playwright(self):
         # in case we want to run without connectivity to ai_client or playwright
+        import nltk
+        from playwright.sync_api import sync_playwright
+
         self.__playwright = sync_playwright().start()
         nltk.download("punkt")
         nltk.download("punkt_tab")

application/tests/spreadsheet_test.py

@@ -112,8 +112,8 @@ def test_prepare_spreadsheet(self) -> None:
 
         self.assertCountEqual(result, expected)
 
-    @mock.patch("application.utils.spreadsheet.gspread.service_account")
-    @mock.patch("application.utils.spreadsheet.gspread.oauth")
+    @mock.patch("gspread.service_account")
+    @mock.patch("gspread.oauth")
     def test_read_spreadsheet_iso_numbers(
         self, mock_oauth, mock_service_account
     ) -> None:
@@ -165,8 +165,8 @@ def test_read_spreadsheet_iso_numbers(
 
         self.assertEqual(result["ISO Numericise Test"], expected)
 
-    @mock.patch("application.utils.spreadsheet.gspread.service_account")
-    @mock.patch("application.utils.spreadsheet.gspread.oauth")
+    @mock.patch("gspread.service_account")
+    @mock.patch("gspread.oauth")
     def test_read_spreadsheet_empty_worksheet(
         self, mock_oauth, mock_service_account
     ) -> None:
@@ -191,8 +191,8 @@ def test_read_spreadsheet_empty_worksheet(
 
         self.assertEqual(result["Empty Sheet"], [])
 
-    @mock.patch("application.utils.spreadsheet.gspread.service_account")
-    @mock.patch("application.utils.spreadsheet.gspread.oauth")
+    @mock.patch("gspread.service_account")
+    @mock.patch("gspread.oauth")
     def test_read_spreadsheet_short_row_padded(
         self, mock_oauth, mock_service_account
     ) -> None:
@@ -221,15 +221,15 @@ def test_read_spreadsheet_short_row_padded(
         self.assertEqual(result["Short Row"], [{"Col A": "only-a", "Col B": ""}])
 
     def test_records_from_worksheet_values_duplicate_headers(self) -> None:
-        with self.assertRaises(gspread.exceptions.GSpreadException) as ctx:
+        with self.assertRaises(ValueError) as ctx:
             _records_from_worksheet_values(
                 [["Col A", "Col B", "Col A"], ["x", "y", "z"]]
             )
         self.assertIn("Duplicate worksheet headers", str(ctx.exception))
         self.assertIn("Col A", str(ctx.exception))
 
-    @mock.patch("application.utils.spreadsheet.gspread.service_account")
-    @mock.patch("application.utils.spreadsheet.gspread.oauth")
+    @mock.patch("gspread.service_account")
+    @mock.patch("gspread.oauth")
     def test_read_spreadsheet_duplicate_headers(
         self, mock_oauth, mock_service_account
     ) -> None:

application/tests/web_main_test.py

@@ -797,7 +797,9 @@ def test_gap_analysis_create_job_id(
             self.assertTrue(enqueue_call_mock.called)
             _, kwargs = enqueue_call_mock.call_args
             self.assertEqual("aaa->bbb", kwargs["description"])
-            self.assertEqual(cre_main.run_gap_pair_job, kwargs["func"])
+            self.assertEqual(
+                "application.cmd.cre_main.run_gap_pair_job", kwargs["func"]
+            )
             self.assertEqual("aaa", kwargs["kwargs"]["importing_name"])
             self.assertEqual("bbb", kwargs["kwargs"]["peer_name"])
             self.assertEqual(GAP_ANALYSIS_TIMEOUT, kwargs["timeout"])

application/utils/external_project_parsers/base_parser.py

@@ -1,10 +1,8 @@
 from application.utils.external_project_parsers import base_parser_defs
 from rq import Queue
 from application.utils import redis
-from application.prompt_client import prompt_client as prompt_client
 import logging
 import time
-from alive_progress import alive_bar
 from application.utils.external_project_parsers.parsers import *
 from application.utils import gap_analysis
 import os, json
@@ -22,6 +20,7 @@ def register_resource(
         db_connection_str: str,
     ):
         from application.cmd import cre_main
+        from application.prompt_client import prompt_client as prompt_client
 
         db = cre_main.db_connect(db_connection_str)
 
@@ -55,6 +54,8 @@ def call_importers(self, db_connection_str: str):
         somehow finds all the importers that have been registered (either reflection for implementing classes or an explicit method that registers all available importers)
         and schedules jobs to call those importers, monitors the jobs and alerts when done same as cre_main
         """
+        from alive_progress import alive_bar
+
         importers = []
         jobs = []
         conn = redis.connect()

application/utils/external_project_parsers/base_parser_defs.py

@@ -1,11 +1,13 @@
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, TYPE_CHECKING
 from dataclasses import dataclass
 from enum import Enum
 
 from application.defs import cre_defs as defs
-from application.prompt_client import prompt_client as prompt_client
 from application.database import db
 
+if TYPE_CHECKING:
+    from application.prompt_client import prompt_client as prompt_client
+
 # abstract class/interface that shows how to import a project that is not cre or its core resources
 
 
@@ -125,7 +127,7 @@ class ParserInterface(object):
 
     def parse(
         database: db.Node_collection,
-        prompt_client: Optional[prompt_client.PromptHandler],
+        prompt_client: Optional["prompt_client.PromptHandler"],
     ) -> ParseResult:
         """
         Parses the resources of a project,

application/utils/spreadsheet.py

@@ -5,7 +5,6 @@
 from copy import deepcopy
 from typing import Any, Dict, List, Optional, Set
 import os
-import gspread
 import yaml
 from application.database import db
 from application.defs import cre_defs as defs
@@ -38,9 +37,7 @@ def _records_from_worksheet_values(rows: List[List[Any]]) -> List[Dict[str, Any]
     headers = rows[0]
     duplicates = sorted(findDups(headers))
     if duplicates:
-        raise gspread.exceptions.GSpreadException(
-            f"Duplicate worksheet headers: {duplicates}"
-        )
+        raise ValueError(f"Duplicate worksheet headers: {duplicates}")
     records: List[Dict[str, Any]] = []
     for row in rows[1:]:
         padded = list(row) + [""] * max(0, len(headers) - len(row))
@@ -57,6 +54,8 @@ def read_spreadsheet(
 ) -> Dict[str, Any]:
     """given remote google spreadsheet url,
     reads each workbook into a collection of documents"""
+    import gspread  # lazy: Google Sheets import is CLI/local only
+
     result = {}
     try:
         if (
@@ -91,7 +90,7 @@ def read_spreadsheet(
         logger.error('Error opening spreadsheet "%s" : "%s"' % (alias, url))
         logger.error(ae)
         exit(1)
-    except gspread.exceptions.GSpreadException as gse:
+    except (gspread.exceptions.GSpreadException, ValueError) as gse:
         logger.error(
             "If this exception says you have a duplicate cell name, the duplicate is",
             findDups(wsh.row_values(1)),
@@ -302,6 +301,8 @@ def write_csv(docs: List[Dict[str, Any]]) -> io.StringIO:
 
 def write_spreadsheet(title: str, docs: List[Dict[str, Any]], emails: List[str]) -> str:
     """upload local array of flat yamls to url, share with email list"""
+    import gspread  # lazy: Google Sheets write is CLI/local only
+
     gc = gspread.oauth()  # oauth config,
     # TODO (northdpole): make this configurable
     sh = gc.create("0." + title)