diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index cf2d580e6..cc73668c2 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -43,7 +43,7 @@ jobs: python -m venv venv . ./venv/bin/activate pip install --upgrade pip setuptools - pip install -r requirements.txt + pip install -r requirements-dev.txt - name: OpenAPI guardrail run: make openapi-guardrail - name: Lint Code Base diff --git a/Makefile b/Makefile index ad76a3f7d..869c77f94 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ cover: install-deps-python: [ -d "./venv" ] && . ./venv/bin/activate &&\ pip install --upgrade pip setuptools &&\ - pip install -r requirements.txt + pip install -r requirements-dev.txt install-deps-typescript: (cd application/frontend && yarn install) diff --git a/application/cmd/cre_main.py b/application/cmd/cre_main.py index f42a4a5c9..002f10ac2 100644 --- a/application/cmd/cre_main.py +++ b/application/cmd/cre_main.py @@ -9,14 +9,12 @@ import requests from collections import deque -from typing import Any, Callable, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING import hashlib import json as _json from rq import Queue, job, exceptions from sqlalchemy import not_ -from application.utils.external_project_parsers.base_parser import BaseParser -from application.utils.external_project_parsers.parsers import master_spreadsheet_parser from application import create_app # type: ignore from application.config import CMDConfig from application.database import db @@ -26,11 +24,12 @@ from application.utils import spreadsheet as sheet_utils from application.utils import redis from application.utils import db_backend -from alive_progress import alive_bar -from application.prompt_client import prompt_client as prompt_client from application.utils import gap_analysis from application.utils import cres_csv_export +if TYPE_CHECKING: + from application.prompt_client import prompt_client as prompt_client + logging.basicConfig() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -469,6 +468,8 @@ def _standard_structure_fingerprint(resource_name: str) -> str: return hashlib.sha256(payload.encode("utf-8")).hexdigest() conn = redis.connect() + from application.prompt_client import prompt_client as prompt_client + ph = prompt_client.PromptHandler(database=collection) importing_name = standard_entries[0].name effective_calculate_gap_analysis = ( @@ -541,7 +542,7 @@ def _standard_structure_fingerprint(resource_name: str) -> str: def parse_standards_from_spreadsheeet( cre_file: List[Dict[str, Any]], cache_location: str, - prompt_handler: prompt_client.PromptHandler, + prompt_handler: "prompt_client.PromptHandler", ) -> None: """given a csv with standards, build a list of standards in the db""" if not cre_file: @@ -568,6 +569,10 @@ def parse_standards_from_spreadsheeet( from application.utils import import_pipeline collection = db_connect(cache_location) + from application.utils.external_project_parsers.parsers import ( + master_spreadsheet_parser, + ) + parse_result = master_spreadsheet_parser.MasterSpreadsheetParser.parse_rows( cre_file ) @@ -682,6 +687,8 @@ def download_gap_analysis_from_upstream(cache: str) -> None: pairs = [(sa, sb) for sa in standards for sb in standards if sa != sb] if os.environ.get("BENCHMARK_MODE") == "1": + from alive_progress import alive_bar + with alive_bar(len(pairs), title="Fetching upstream Gap Analysis") as bar: for sa, sb in pairs: res = requests.get( @@ -846,11 +853,15 @@ def backfill_gap_analysis_only( jobs.append(j) if jobs: + from alive_progress import alive_bar + with alive_bar( len(jobs), title=f"GA batch {i // batch_size + 1}" ) as bar: redis.wait_for_jobs(jobs, bar) else: + from alive_progress import alive_bar + with alive_bar(len(batch), title=f"GA batch {i // batch_size + 1}") as bar: for sa, sb in batch: _compute_pair_direct(collection, sa, sb) @@ -901,6 +912,8 @@ def run(args: argparse.Namespace) -> None: # pragma: no cover cache.delete_nodes(args.delete_resource) # individual resource importing + from application.utils.external_project_parsers.base_parser import BaseParser + if args.zap_in: from application.utils.external_project_parsers.parsers import zap_alerts_parser @@ -1014,6 +1027,8 @@ def run(args: argparse.Namespace) -> None: # pragma: no cover def ai_client_init(database: db.Node_collection): + from application.prompt_client import prompt_client as prompt_client + return prompt_client.PromptHandler(database=database) @@ -1053,6 +1068,8 @@ def prepare_for_review(cache: str) -> Tuple[str, str]: def generate_embeddings(db_url: str) -> None: + from application.prompt_client import prompt_client as prompt_client + database = db_connect(path=db_url) prompt_client.PromptHandler(database, load_all_embeddings=True) @@ -1109,6 +1126,8 @@ def run_librarian( "land W8); running in dry-run mode" ) + from application.prompt_client import prompt_client as prompt_client + cfg = load_config() database = db_connect(path=cache_file) ph = prompt_client.PromptHandler(database=database) @@ -1207,6 +1226,8 @@ def run_librarian( def regenerate_embeddings(db_url: str) -> None: """Wipe all embedding rows, then rebuild (CRE + every node type) like ``--generate_embeddings``.""" + from application.prompt_client import prompt_client as prompt_client + database = db_connect(path=db_url) removed = database.delete_all_embeddings() logger.info("Removed %s embedding rows; rebuilding embeddings", removed) diff --git a/application/prompt_client/prompt_client.py b/application/prompt_client/prompt_client.py index 112114034..1006d00a1 100644 --- a/application/prompt_client/prompt_client.py +++ b/application/prompt_client/prompt_client.py @@ -2,15 +2,11 @@ from application.defs import cre_defs from datetime import datetime from multiprocessing import Pool -from nltk.corpus import stopwords -from nltk.tokenize import word_tokenize from io import BytesIO from urllib.parse import urlparse from application.prompt_client import embed_alignment -from playwright.sync_api import Error as PlaywrightError -from playwright.sync_api import TimeoutError as PlaywrightTimeoutError, sync_playwright from scipy import sparse from sklearn.metrics.pairwise import cosine_similarity from typing import Dict, List, Any, Tuple, Optional @@ -22,7 +18,6 @@ from pypdf import PdfReader except ImportError: PdfReader = None # type: ignore[misc, assignment] -import nltk import numpy as np import os import json @@ -235,6 +230,10 @@ def __init__(cls): # Function to get text content from a URL def get_content(self, url) -> Optional[str]: + # Playwright is for scrape/import embedding generation only — not chat. + from playwright.sync_api import Error as PlaywrightError + from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + for attempts in range(1, 10): if _is_likely_pdf_url(url): text = _fetch_pdf_text_for_embeddings(url) @@ -298,6 +297,9 @@ def get_content(self, url) -> Optional[str]: def get_html(self, url) -> Optional[str]: """Return raw HTML document string (for smart excerpt alignment). PDF URLs unsupported.""" + from playwright.sync_api import Error as PlaywrightError + from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + for attempts in range(1, 10): if _is_likely_pdf_url(url): return None @@ -343,6 +345,8 @@ def _ensure_smart_embed_caches(self) -> None: ] = {} def clean_content(self, content): + from nltk.tokenize import word_tokenize # lazy: scrape/import path only + content = re.sub("\s+", " ", content.strip()) # split into words @@ -363,6 +367,9 @@ def with_ai_client(self, ai_client): def setup_playwright(self): # in case we want to run without connectivity to ai_client or playwright + import nltk + from playwright.sync_api import sync_playwright + self.__playwright = sync_playwright().start() nltk.download("punkt") nltk.download("punkt_tab") diff --git a/application/tests/spreadsheet_test.py b/application/tests/spreadsheet_test.py index a7b67569f..23210714f 100644 --- a/application/tests/spreadsheet_test.py +++ b/application/tests/spreadsheet_test.py @@ -112,8 +112,8 @@ def test_prepare_spreadsheet(self) -> None: self.assertCountEqual(result, expected) - @mock.patch("application.utils.spreadsheet.gspread.service_account") - @mock.patch("application.utils.spreadsheet.gspread.oauth") + @mock.patch("gspread.service_account") + @mock.patch("gspread.oauth") def test_read_spreadsheet_iso_numbers( self, mock_oauth, mock_service_account ) -> None: @@ -165,8 +165,8 @@ def test_read_spreadsheet_iso_numbers( self.assertEqual(result["ISO Numericise Test"], expected) - @mock.patch("application.utils.spreadsheet.gspread.service_account") - @mock.patch("application.utils.spreadsheet.gspread.oauth") + @mock.patch("gspread.service_account") + @mock.patch("gspread.oauth") def test_read_spreadsheet_empty_worksheet( self, mock_oauth, mock_service_account ) -> None: @@ -191,8 +191,8 @@ def test_read_spreadsheet_empty_worksheet( self.assertEqual(result["Empty Sheet"], []) - @mock.patch("application.utils.spreadsheet.gspread.service_account") - @mock.patch("application.utils.spreadsheet.gspread.oauth") + @mock.patch("gspread.service_account") + @mock.patch("gspread.oauth") def test_read_spreadsheet_short_row_padded( self, mock_oauth, mock_service_account ) -> None: @@ -221,15 +221,15 @@ def test_read_spreadsheet_short_row_padded( self.assertEqual(result["Short Row"], [{"Col A": "only-a", "Col B": ""}]) def test_records_from_worksheet_values_duplicate_headers(self) -> None: - with self.assertRaises(gspread.exceptions.GSpreadException) as ctx: + with self.assertRaises(ValueError) as ctx: _records_from_worksheet_values( [["Col A", "Col B", "Col A"], ["x", "y", "z"]] ) self.assertIn("Duplicate worksheet headers", str(ctx.exception)) self.assertIn("Col A", str(ctx.exception)) - @mock.patch("application.utils.spreadsheet.gspread.service_account") - @mock.patch("application.utils.spreadsheet.gspread.oauth") + @mock.patch("gspread.service_account") + @mock.patch("gspread.oauth") def test_read_spreadsheet_duplicate_headers( self, mock_oauth, mock_service_account ) -> None: diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index 0fab4fc74..f63ebe5bf 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -797,7 +797,9 @@ def test_gap_analysis_create_job_id( self.assertTrue(enqueue_call_mock.called) _, kwargs = enqueue_call_mock.call_args self.assertEqual("aaa->bbb", kwargs["description"]) - self.assertEqual(cre_main.run_gap_pair_job, kwargs["func"]) + self.assertEqual( + "application.cmd.cre_main.run_gap_pair_job", kwargs["func"] + ) self.assertEqual("aaa", kwargs["kwargs"]["importing_name"]) self.assertEqual("bbb", kwargs["kwargs"]["peer_name"]) self.assertEqual(GAP_ANALYSIS_TIMEOUT, kwargs["timeout"]) diff --git a/application/utils/external_project_parsers/base_parser.py b/application/utils/external_project_parsers/base_parser.py index 5bff50e63..49e7b4c2d 100644 --- a/application/utils/external_project_parsers/base_parser.py +++ b/application/utils/external_project_parsers/base_parser.py @@ -1,10 +1,8 @@ from application.utils.external_project_parsers import base_parser_defs from rq import Queue from application.utils import redis -from application.prompt_client import prompt_client as prompt_client import logging import time -from alive_progress import alive_bar from application.utils.external_project_parsers.parsers import * from application.utils import gap_analysis import os, json @@ -22,6 +20,7 @@ def register_resource( db_connection_str: str, ): from application.cmd import cre_main + from application.prompt_client import prompt_client as prompt_client db = cre_main.db_connect(db_connection_str) @@ -55,6 +54,8 @@ def call_importers(self, db_connection_str: str): somehow finds all the importers that have been registered (either reflection for implementing classes or an explicit method that registers all available importers) and schedules jobs to call those importers, monitors the jobs and alerts when done same as cre_main """ + from alive_progress import alive_bar + importers = [] jobs = [] conn = redis.connect() diff --git a/application/utils/external_project_parsers/base_parser_defs.py b/application/utils/external_project_parsers/base_parser_defs.py index 0cb79d6f9..4b6660794 100644 --- a/application/utils/external_project_parsers/base_parser_defs.py +++ b/application/utils/external_project_parsers/base_parser_defs.py @@ -1,11 +1,13 @@ -from typing import List, Dict, Optional +from typing import List, Dict, Optional, TYPE_CHECKING from dataclasses import dataclass from enum import Enum from application.defs import cre_defs as defs -from application.prompt_client import prompt_client as prompt_client from application.database import db +if TYPE_CHECKING: + from application.prompt_client import prompt_client as prompt_client + # abstract class/interface that shows how to import a project that is not cre or its core resources @@ -125,7 +127,7 @@ class ParserInterface(object): def parse( database: db.Node_collection, - prompt_client: Optional[prompt_client.PromptHandler], + prompt_client: Optional["prompt_client.PromptHandler"], ) -> ParseResult: """ Parses the resources of a project, diff --git a/application/utils/spreadsheet.py b/application/utils/spreadsheet.py index c490bc660..170ca38f5 100644 --- a/application/utils/spreadsheet.py +++ b/application/utils/spreadsheet.py @@ -5,7 +5,6 @@ from copy import deepcopy from typing import Any, Dict, List, Optional, Set import os -import gspread import yaml from application.database import db from application.defs import cre_defs as defs @@ -38,9 +37,7 @@ def _records_from_worksheet_values(rows: List[List[Any]]) -> List[Dict[str, Any] headers = rows[0] duplicates = sorted(findDups(headers)) if duplicates: - raise gspread.exceptions.GSpreadException( - f"Duplicate worksheet headers: {duplicates}" - ) + raise ValueError(f"Duplicate worksheet headers: {duplicates}") records: List[Dict[str, Any]] = [] for row in rows[1:]: padded = list(row) + [""] * max(0, len(headers) - len(row)) @@ -57,6 +54,8 @@ def read_spreadsheet( ) -> Dict[str, Any]: """given remote google spreadsheet url, reads each workbook into a collection of documents""" + import gspread # lazy: Google Sheets import is CLI/local only + result = {} try: if ( @@ -91,7 +90,7 @@ def read_spreadsheet( logger.error('Error opening spreadsheet "%s" : "%s"' % (alias, url)) logger.error(ae) exit(1) - except gspread.exceptions.GSpreadException as gse: + except (gspread.exceptions.GSpreadException, ValueError) as gse: logger.error( "If this exception says you have a duplicate cell name, the duplicate is", findDups(wsh.row_values(1)), @@ -302,6 +301,8 @@ def write_csv(docs: List[Dict[str, Any]]) -> io.StringIO: def write_spreadsheet(title: str, docs: List[Dict[str, Any]], emails: List[str]) -> str: """upload local array of flat yamls to url, share with email list""" + import gspread # lazy: Google Sheets write is CLI/local only + gc = gspread.oauth() # oauth config, # TODO (northdpole): make this configurable sh = gc.create("0." + title) diff --git a/application/web/web_main.py b/application/web/web_main.py index b460672b5..d795c453d 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -10,16 +10,13 @@ import pathlib import re import urllib.parse -from alive_progress import alive_bar from typing import Any -from application.utils import oscal_utils, redis from rq import job, exceptions from rq import Queue -from application.utils import oscal_utils, redis +from application.utils import redis from application.database import db -from application.cmd import cre_main from application.defs import cre_defs as defs from application.defs import cre_exceptions from application.feature_flags import ( @@ -31,7 +28,6 @@ from application.utils import spreadsheet as sheet_utils from application.utils import mdutils, redirectors, gap_analysis -from application.utils.external_project_parsers.parsers import myopencre_parser from application.web.openapi_registry import openapi_documented from enum import Enum from flask import json as flask_json @@ -179,6 +175,15 @@ def find_cre(creid: str = None, crename: str = None) -> Any: # refer return write_csv(docs=docs).getvalue().encode("utf-8") elif opt_format == SupportedFormats.OSCAL.value: + try: + from application.utils import oscal_utils + except ImportError: + abort( + 503, + "OSCAL export is unavailable on this deployment " + "(compliance-trestle not installed).", + ) + result = {"data": json.loads(oscal_utils.document_to_oscal(cre))} return jsonify(result) @@ -270,6 +275,15 @@ def find_node_by_name( return write_csv(docs=docs).getvalue().encode("utf-8") elif opt_format == SupportedFormats.OSCAL.value: + try: + from application.utils import oscal_utils + except ImportError: + abort( + 503, + "OSCAL export is unavailable on this deployment " + "(compliance-trestle not installed).", + ) + return jsonify(json.loads(oscal_utils.list_to_oscal(nodes))) # if opt_osib: @@ -307,6 +321,15 @@ def find_document_by_tag() -> Any: ) return write_csv(docs=docs).getvalue().encode("utf-8") elif opt_format == SupportedFormats.OSCAL.value: + try: + from application.utils import oscal_utils + except ImportError: + abort( + 503, + "OSCAL export is unavailable on this deployment " + "(compliance-trestle not installed).", + ) + return jsonify(json.loads(oscal_utils.list_to_oscal(documents))) return jsonify(result) @@ -395,7 +418,7 @@ def map_analysis() -> Any: j = q.enqueue_call( description=f"{standards[0]}->{standards[1]}", - func=cre_main.run_gap_pair_job, + func="application.cmd.cre_main.run_gap_pair_job", kwargs={ "importing_name": standards[0], "peer_name": standards[1], @@ -535,6 +558,8 @@ def ga_standards() -> Any: standards = list(database.standards()) if OPENCRE_STANDARD_NAME not in standards: standards.append(OPENCRE_STANDARD_NAME) + from application.cmd import cre_main + eligible = [ s for s in standards @@ -591,6 +616,15 @@ def text_search() -> Any: ) return write_csv(docs=docs).getvalue().encode("utf-8") elif opt_format == SupportedFormats.OSCAL.value: + try: + from application.utils import oscal_utils + except ImportError: + abort( + 503, + "OSCAL export is unavailable on this deployment " + "(compliance-trestle not installed).", + ) + return jsonify(json.loads(oscal_utils.list_to_oscal(documents))) res = [doc.todict() for doc in documents] @@ -648,6 +682,15 @@ def find_root_cres() -> Any: ) return write_csv(docs=docs).getvalue().encode("utf-8") elif opt_format == SupportedFormats.OSCAL.value: + try: + from application.utils import oscal_utils + except ImportError: + abort( + 503, + "OSCAL export is unavailable on this deployment " + "(compliance-trestle not installed).", + ) + return jsonify(json.loads(oscal_utils.list_to_oscal(documents))) return jsonify(result) @@ -1038,9 +1081,24 @@ def chat_cre() -> Any: database = db.Node_collection() # Lazy import to avoid loading heavy prompt/ML dependencies at web boot. - from application.prompt_client import llm_error_utils, prompt_client + try: + from application.prompt_client import llm_error_utils, prompt_client + + prompt = prompt_client.PromptHandler(database) + except ImportError: + abort( + 503, + "Chat is unavailable on this deployment (ML/LLM dependencies not installed).", + ) + except RuntimeError as exc: + # PromptHandler raises RuntimeError when litellm is missing. + if "litellm" not in str(exc).lower(): + raise + abort( + 503, + "Chat is unavailable on this deployment (ML/LLM dependencies not installed).", + ) - prompt = prompt_client.PromptHandler(database) try: response = prompt.generate_text(message.get("prompt")) except Exception as e: @@ -1294,6 +1352,8 @@ def import_from_cre_csv() -> Any: contents = file.read() csv_read = csv.DictReader(contents.decode("utf-8").splitlines()) try: + from application.utils.external_project_parsers.parsers import myopencre_parser + parse_result = myopencre_parser.parse_rows_to_documents(list(csv_read)) except cre_exceptions.DuplicateLinkException as dle: abort(500, f"error during parsing of the incoming CSV, err:{dle}") diff --git a/cre.py b/cre.py index 98b0198f2..b46467281 100644 --- a/cre.py +++ b/cre.py @@ -13,7 +13,6 @@ pass import click # type: ignore -import coverage # type: ignore from flask_migrate import Migrate # type: ignore from application import create_app, sqla # type: ignore @@ -33,6 +32,8 @@ def test(cover: bool, test_names: List[str]) -> None: COV = None if cover or os.environ.get("FLASK_COVERAGE"): + import coverage # type: ignore # lazy: not needed for web/worker boot + COV = coverage.coverage( branch=True, include="application/*", diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..fa57bfbec --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,86 @@ +# Local development, CI, importers, embeddings, and Librarian ML. +# Install with: pip install -r requirements-dev.txt +-r requirements.txt + +# ML / Librarian (pulls torch/CUDA; never install on Heroku) +sentence-transformers>=5.0.0,<6.0.0 + +# importer / embeddings scrape tooling (not needed for prod chat) +playwright +nltk +openai +vertexai +google-cloud-aiplatform +google-cloud-trace +grpcio +grpcio_status +proto-plus +protobuf +beautifulsoup4 +soupsieve +pypdf +alive-progress +gspread +GitPython +PyGithub +xmltodict +openpyxl +Pillow +Shapely +paramiko +PySnooper +untangle +python-frontmatter +python-slugify +google +orderedmultidict +pathable +pathspec +prance +jsonschema>=4.26.0 +openapi-schema-validator +openapi-spec-validator +pep517 +pbr +pip-autoremove +pluggy +prompt-toolkit +testresources +threadpoolctl +text-unidecode +toml +tomli +tqdm +smmap +sniffio +rfc3986 +ruamel.yaml +ruamel.yaml.clib +referencing>=0.37.0 +regex +requests-oauthlib +rsa +pyasn1 +pyasn1-modules +pycparser +pyee +pyparsing +pyrsistent +PyJWT +PyNaCl +packaging +platformdirs +nose +sqlalchemy-stubs +types-PyYAML +typing-inspect +pycodestyle +pyflakes + +# lint / test / typecheck +black==24.4.2 +coverage +mypy +pytest +pytest-base-url +pytest-playwright diff --git a/requirements.txt b/requirements.txt index 8edf363ed..ee5dbcf81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,124 +1,58 @@ -black==24.4.2 # do not touch, same version as superlinter -click -compliance-trestle -coverage -dacite +# Production / Heroku runtime dependencies. +# Local/CI/dev tooling and Librarian ML (torch) live in requirements-dev.txt. +# +# Keep torch / sentence-transformers OFF this file — they blow the 1000M slug. +# Chat (LiteLLM + prompt_client) and OSCAL export are supported on prod. + +# server +gunicorn Flask -Flask_Caching -flask_compress -Flask_Cors -Flask_Migrate +Flask-Caching +flask-compress +Flask-Cors +Flask-Migrate Flask-SQLAlchemy -setuptools -gitpython -google -google-api-core -google_auth_oauthlib -google-cloud-aiplatform -grpcio_status -grpcio -gspread -gunicorn -jsonschema>=4.26.0 -networkx -nltk -oauthlib -openai -playwright +SQLAlchemy psycopg2-binary -pygithub -python_markdown_maker -scikit_learn -scipy -semver setuptools -SQLAlchemy -compliance-trestle -nose -mypy -numpy +click +python-dotenv + +# queue / cache +redis +rq + +# graph / DB helpers (db.py imports these at boot) neo4j neomodel -openapi-schema-validator -openapi-spec-validator +networkx +PyYAML==6.0.1 + +# auth + HTTP +requests +oauthlib +google-auth +google-api-core +google_auth_oauthlib + +# OpenAPI / schemas used by web marshmallow apispec -openpyxl -orderedmultidict -packaging -paramiko -pathable -pathspec -pbr -pep517 -Pillow -pip-autoremove -platformdirs -playwright -pluggy -posthog -prance -prompt-toolkit -proto-plus -protobuf -psycopg2-binary -pyasn1 -pyasn1-modules -pycodestyle -pycparser pydantic>=2.4.0,<3 -pyee -pyflakes -PyGithub -PyJWT -PyNaCl -pyparsing -pypdf -pyrsistent -PySnooper -pytest -pytest-base-url -pytest-playwright -python-dateutil -python-dotenv -python-frontmatter +dacite +semver python-markdown-maker -python-slugify -PyYAML==6.0.1 -referencing>=0.37.0 -regex -requests -requests-oauthlib -rfc3986 -rsa -rq -redis -ruamel.yaml -ruamel.yaml.clib -scikit-learn -sentence-transformers>=5.0.0,<6.0.0 -Shapely -six -smmap -sniffio -soupsieve -SQLAlchemy -sqlalchemy-stubs -testresources -text-unidecode -threadpoolctl -toml -tomli -tqdm -types-PyYAML -typing-inspect -typing_extensions -untangle -urllib3 -vertexai -xmltodict -google-cloud-trace -alive-progress -beautifulsoup4 + +# chat (/rest/v1/completion) — embed prompt via LiteLLM, match with sklearn litellm +numpy +scipy +scikit-learn + +# OSCAL export (?format=oscal) +compliance-trestle +# optional analytics (env-gated import in web_main) +posthog + +typing_extensions diff --git a/scripts/build_labeled_dataset.py b/scripts/build_labeled_dataset.py index c978d9478..03b5f62af 100644 --- a/scripts/build_labeled_dataset.py +++ b/scripts/build_labeled_dataset.py @@ -39,7 +39,7 @@ try: from github import Auth, Github, GithubException except ImportError: - sys.exit("PyGithub not installed. Run: pip install -r requirements.txt") + sys.exit("PyGithub not installed. Run: pip install -r requirements-dev.txt") # --- Configuration -------------------------------------------------------- diff --git a/scripts/run-local.sh b/scripts/run-local.sh index 94631cbe9..345ac06ff 100755 --- a/scripts/run-local.sh +++ b/scripts/run-local.sh @@ -14,7 +14,7 @@ source "$VENV_DIR/bin/activate" if ! python -c "import flask" >/dev/null 2>&1; then echo "Installing Python dependencies" - pip install -r "$ROOT_DIR/requirements.txt" + pip install -r "$ROOT_DIR/requirements-dev.txt" fi export NO_LOGIN="${NO_LOGIN:-1}" diff --git a/scripts/sync_embeddings_table.py b/scripts/sync_embeddings_table.py index f61790a9b..13aaf359b 100644 --- a/scripts/sync_embeddings_table.py +++ b/scripts/sync_embeddings_table.py @@ -19,7 +19,7 @@ Foreign keys on ``embeddings`` reference ``cre`` and ``node``; every ``cre_id`` / ``node_id`` in the source rows must already exist on the target or inserts will fail. -Requires: psycopg2 (see requirements.txt). SQLite uses the stdlib ``sqlite3`` module. +Requires: psycopg2 (see requirements.txt / requirements-dev.txt). SQLite uses the stdlib ``sqlite3`` module. """ from __future__ import annotations diff --git a/scripts/update-cwe.sh b/scripts/update-cwe.sh index 2c6570f86..e501caf0e 100755 --- a/scripts/update-cwe.sh +++ b/scripts/update-cwe.sh @@ -17,7 +17,7 @@ source "$VENV_DIR/bin/activate" if ! python -c "import requests" >/dev/null 2>&1; then echo "Installing Python dependencies" - pip install -r "$ROOT_DIR/requirements.txt" + pip install -r "$ROOT_DIR/requirements-dev.txt" fi if [[ -f "$CACHE_FILE" ]]; then