From d9798ba79ca36b1c99804a4770c94e217b104ac6 Mon Sep 17 00:00:00 2001 From: scitex-agent-container Date: Mon, 31 Aug 2026 14:34:47 +0000 Subject: [PATCH 1/2] chore(sqlite-out): the retired engine's name reaches zero outside docs/adr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We do not use the retired embedded engine; runtime state lives in PostgreSQL. Its name survived in 20 tracked files. This clears all of them, keeping only docs/adr/ (untouched) and the Zotero/Optuna integrations (which carry none). Deleted outright: - `.scitex/clew/runtime/db.sqlite` and `examples/_legacy/scitex/clew/multi_parent/scitex/clew.db` — two live database files committed into the tree. - `examples/_legacy/notebooks/19_scitex_db.ipynb` — a tutorial for the retired engine end to end (raw client connections, `sqlite_master` introspection). Nothing survives removing the engine from it, so the notebook goes and the master index's two links to it go with it. Rewritten to name PostgreSQL, which is what we actually run: - README's `scitex.db` section, `docs/05_ADDITIONAL_MODULES.md`, and the clew provenance prose in `docs/sphinx/core_concepts.rst` / `docs/sphinx/modules/io.rst`. - `examples/_legacy/scitex/io/schematic_io_concept.py` and its recorded `_out/` YAML — the schematic's store box. Also: - `tests/integration/test_integration.py` no longer asserts the engine's wrapper class is exported; it still asserts `PostgreSQL` and `check_health`. - `tests/integration/test_cross_package_imports.py` drops `scitex_io._load_modules._sqlite3` — that module was deleted in scitex-io 0.3.0, so the entry has silently skipped ever since. - The engine's name leaves `.gitignore` (both copies), `pyproject.toml`'s clew comment, the stdlib table in `scripts/maintenance/_pypi_packages.py`, the CrossRef env example, and the legacy io-module guideline's format table. - `examples/_legacy/notebooks/27_scitex_scholar.ipynb` no longer opens a raw client against the third-party impact-factor database; Scholar's own `enrich_by_default=True` already does that enrichment. No guard test asserting the name's absence is added, deliberately. --- .env.d.examples/00_crossref-local.env | 2 +- .gitignore | 4 - .scitex/clew/runtime/db.sqlite | Bin 69632 -> 0 bytes README.md | 6 +- docs/05_ADDITIONAL_MODULES.md | 2 +- docs/guides/CROSSREF_API_CONFIGURATION.md | 2 +- docs/sphinx/core_concepts.rst | 4 +- docs/sphinx/modules/io.rst | 2 +- .../notebooks/00_SCITEX_MASTER_INDEX.ipynb | 7 - examples/_legacy/notebooks/19_scitex_db.ipynb | 824 ------------------ .../_legacy/notebooks/27_scitex_scholar.ipynb | 48 +- .../scitex/clew/multi_parent/scitex/clew.db | Bin 94208 -> 0 bytes .../_legacy/scitex/io/schematic_io_concept.py | 8 +- .../stx_io_concept.yaml | 4 +- .../example-python-project-scitex/.gitignore | 3 - .../guidelines/python/scitex/12-io-module.md | 2 +- pyproject.toml | 2 +- scripts/maintenance/_pypi_packages.py | 1 - .../integration/test_cross_package_imports.py | 1 - tests/integration/test_integration.py | 1 - 20 files changed, 22 insertions(+), 901 deletions(-) delete mode 100644 .scitex/clew/runtime/db.sqlite delete mode 100644 examples/_legacy/notebooks/19_scitex_db.ipynb delete mode 100644 examples/_legacy/scitex/clew/multi_parent/scitex/clew.db diff --git a/.env.d.examples/00_crossref-local.env b/.env.d.examples/00_crossref-local.env index 00eaaeeaa..e71b2e9b5 100644 --- a/.env.d.examples/00_crossref-local.env +++ b/.env.d.examples/00_crossref-local.env @@ -11,5 +11,5 @@ CROSSREF_LOCAL_API_URL=http://localhost:31291 CROSSREF_LOCAL_HOST=0.0.0.0 CROSSREF_LOCAL_PORT=31291 -# Path to SQLite database (for db mode) +# Path to the local database file (for db mode) # CROSSREF_LOCAL_DB=/path/to/crossref.db diff --git a/.gitignore b/.gitignore index 20d3a496a..7a4f50d38 100755 --- a/.gitignore +++ b/.gitignore @@ -144,8 +144,6 @@ cover/ # Django stuff: # *.log local_settings.py -db.sqlite3 -db.sqlite3-journal # Flask stuff: instance/ @@ -389,8 +387,6 @@ cover/ # Django stuff: *.log local_settings.py -db.sqlite3 -db.sqlite3-journal # Flask stuff: instance/ diff --git a/.scitex/clew/runtime/db.sqlite b/.scitex/clew/runtime/db.sqlite deleted file mode 100644 index e51371676d65c5f3a90dded252ab12cc7dd35b88..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69632 zcmeI*Pi)&%90zclbWNKydEG=Cb!k+066_Nu9=NqY-UU^^F$S ze}3=Z=e=jgiI!Yhnp1T}SlujdShx>oz!xcB%>MowjAA-!->zACg%<`Qfk ziwnZOLME$))yfX;oFGo$46W8o%W9c9?8|8I(0Z8(+O1eEf3+BVzu)8ZO-=E)(|Ueo zU8x;0o%U^d&5C4GFU`sOn=4G(7OxhmXI7q-Gs5LedOnr8CcG+N6H?3B#qzb!&gf~Q)DVu7{rXiho4Y6j9mX-&1BW|ZJ8s%^G z8)Muq+-~1M!)UjQc*aA%TBZLvfY+59%6dD~Mbq$n+b39S4;9VmQ-ypOW?9EsTy!#N!5_x>wQjNV1nQ2 z-z#BrU0GGvmE2lhTT`_D<`QT?tE?>vC(AT0_@>k8o1ppo>-n2g#uC}rRKy|E%97SP zn>Ko#zSFdGh>2T3zHU9i0nVHjeGi@=o8-5Pxgi7&Ppxz@E=F3=_7e z#rQDiTCVnD#`0Ro{^fvHD63mKJ>BWmE<7-*-q9_h^n6+0cb%}0TvbadJs&l`CbAE; zTo3F6=qqnMSJ*5nwHJW)VT~taV;I^87B)9l)Do@RA+6m|^n5X|=Z&nI2f*p)dVS}I zS`7f!g}D%0I_OPCAG^`;cp{u@;2t;dulJ9Ed*mB($GhuY@{V}!dg2G)h~Ov$AOHaf zKmY;|fB*y_00E98FL3(Pc92fG10F8L`B}l{{b(pGhGS<$=}b7D6=UZ_@mx55HkOPh z!Xmu~DygOGkzu#Lo=R6Vo!-LQCXzzoaNJBpf?Ut`Dfa|TnDX~=^PH}3(7bXiy^jM|oJdF%WA;aqiCDPJmPsScLq)X9HI_&uLbY1bVfEHi z8r0Hga|I)I6^w_XNl7vpC4X}&+ff%6W6@;7Oy#0&a*_8q@*(+%d`ljYAIX>Gb8?sb zNq6D_0SG_<0uX=z1Rwwb2tWV=5P(1@1c+mTx4eCH%I)y;=6gHiUdK4U_r{CR=NRJI z`zd1sjsRbKnTh`#1YFz*l*bgon(cWZ*~oPSStoQ(G=tX0SG_<0uX=z1Rwwb z2tWV=5P(3p3b4=rasS_~y@Pr|00Izz00bZa0SG_<0uX=z1Ue(YKL6**V~+mA0|F3$ z00bZa0SG_<0uX=z1Rwx`BOq{scTDxwzN^vvVq<*%e*`)fO@IIdAOHafKmY;|fB*y_ z009UbB7pnnVL{<=h00bZa0SG_<0uX=z1Rwx` zE)~G%|6STKs1^hu009U<00Izz00bZa0SG{#g95n!?;s|!f&c^{009U<00Izz00bZa l0SI)d0Pg?0v|~^$2tWV=5P$##AOHafKmY;|fItTY{sWNtq22%h diff --git a/README.md b/README.md index bb8a13e69..af0369461 100755 --- a/README.md +++ b/README.md @@ -526,12 +526,12 @@ Extends stdlib `logging` with SUCCESS/FAIL levels, a 30+ class exception tree (`
-scitex.db -- SQLite3 / PostgreSQL with ndarray BLOB Storage +scitex.db -- PostgreSQL with ndarray BLOB Storage ```python import scitex as stx, numpy as np -db = stx.db.SQLite3("experiments.db") +db = stx.db.PostgreSQL(dbname="experiments", host="localhost", port=55432) with db: # context-manager transaction db.execute("CREATE TABLE IF NOT EXISTS runs (id TEXT, acc REAL)") db.save_array("weights_epoch_87", np.random.rand(1024, 1024)) # compressed BLOB @@ -541,7 +541,7 @@ w = db.load_array("weights_epoch_87") # typed ndarray back db.check_health() # integrity + schema drift stx.db.delete_duplicates(conn, "runs", columns=["id"]) ``` -SQLite / PostgreSQL clients with first-class compressed-ndarray BLOBs, dataframe round-trips, health checks, and duplicate removal. Drop-in replacement for hand-rolling `pickle → BLOB` storage or SQLAlchemy Core when you don't need an ORM. +A PostgreSQL client with first-class compressed-ndarray BLOBs, dataframe round-trips, health checks, and duplicate removal. Drop-in replacement for hand-rolling `pickle → BLOB` storage or SQLAlchemy Core when you don't need an ORM.
diff --git a/docs/05_ADDITIONAL_MODULES.md b/docs/05_ADDITIONAL_MODULES.md index f880361c1..531ee6094 100644 --- a/docs/05_ADDITIONAL_MODULES.md +++ b/docs/05_ADDITIONAL_MODULES.md @@ -118,7 +118,7 @@ Lower-level SciTeX utilities re-exported under the umbrella: | `stx.dict` | `DotDict` + safe merge / flatten | `DotDict`, `safe_merge`, `flatten` | | `stx.logging` | stdlib-logging + SUCCESS/FAIL + `SciTeXError` | `getLogger`, `warn_deprecated`, `Tee` | | `stx.types` | Union type aliases + predicates | `ArrayLike`, `ColorLike`, `is_array_like` | -| `stx.db` | SQLite3 / PostgreSQL wrapper w/ ndarray BLOBs | `SQLite3`, `PostgreSQL`, `delete_duplicates` | +| `stx.db` | PostgreSQL wrapper w/ ndarray BLOBs | `PostgreSQL`, `delete_duplicates` | | `stx.audit` | Unified security scan (bandit / shellcheck / pip-audit) | `audit()` | | `stx.browser` | Playwright helpers for scraping | `save_as_pdf`, `click_with_fallbacks_async` | | `stx.compat` | Deprecation shims | `@deprecated`, `notify` legacy alias | diff --git a/docs/guides/CROSSREF_API_CONFIGURATION.md b/docs/guides/CROSSREF_API_CONFIGURATION.md index 6c0b09b35..ee4aae35d 100644 --- a/docs/guides/CROSSREF_API_CONFIGURATION.md +++ b/docs/guides/CROSSREF_API_CONFIGURATION.md @@ -146,7 +146,7 @@ else: - **Total papers**: 167,008,748 - **Database size**: 1.2TB (1,197,367 MB) -- **Format**: SQLite with indexed fields +- **Format**: Indexed local database - **Source**: CrossRef official database ## Best Practices diff --git a/docs/sphinx/core_concepts.rst b/docs/sphinx/core_concepts.rst index 4484557c4..afb2e40f4 100644 --- a/docs/sphinx/core_concepts.rst +++ b/docs/sphinx/core_concepts.rst @@ -37,7 +37,7 @@ The Session Model 2. **Logging**: All stdout/stderr captured to ``script.log`` 3. **Config injection**: YAML files from ``./config/`` merged and injected 4. **CLI generation**: Function parameters become ``--flags`` -5. **Provenance**: File hashes recorded to SQLite for reproducibility +5. **Provenance**: File hashes recorded to a local database for reproducibility .. code-block:: python @@ -124,7 +124,7 @@ Provenance Tracking (Clew) -------------------------- Inside ``@stx.session``, every ``stx.io.save`` and ``stx.io.load`` call -records the file's SHA-256 hash to a local SQLite database. This enables: +records the file's SHA-256 hash to a local provenance database. This enables: - **Verification**: Check if output files have been modified since creation - **DAG reconstruction**: Trace which inputs produced which outputs diff --git a/docs/sphinx/modules/io.rst b/docs/sphinx/modules/io.rst index 222150bb8..e1432ae91 100644 --- a/docs/sphinx/modules/io.rst +++ b/docs/sphinx/modules/io.rst @@ -94,7 +94,7 @@ Provenance Tracking ------------------- Inside ``@stx.session``, every ``save`` and ``load`` records file hashes -to a local SQLite database for reproducibility verification. +to a local provenance database for reproducibility verification. .. code-block:: python diff --git a/examples/_legacy/notebooks/00_SCITEX_MASTER_INDEX.ipynb b/examples/_legacy/notebooks/00_SCITEX_MASTER_INDEX.ipynb index 2eae64d5c..a47429adb 100644 --- a/examples/_legacy/notebooks/00_SCITEX_MASTER_INDEX.ipynb +++ b/examples/_legacy/notebooks/00_SCITEX_MASTER_INDEX.ipynb @@ -202,12 +202,6 @@ "- PDF downloads and bibliography management\n", "- BibTeX generation with enriched metadata\n", "\n", - "### Database Operations\n", - "**🗄️ [19_scitex_db.ipynb](19_scitex_db.ipynb)** - *Database Integration*\n", - "- PostgreSQL and SQLite support\n", - "- SQL operations with pandas integration\n", - "- Data persistence workflows\n", - "\n", "### Documentation & LaTeX\n", "**📝 [20_scitex_tex.ipynb](20_scitex_tex.ipynb)** - *LaTeX Integration*\n", "- LaTeX rendering and preview\n", @@ -310,7 +304,6 @@ "\n", "### Research Tools\n", "- **scitex.scholar** → [16_scitex_scholar.ipynb](16_scitex_scholar.ipynb)\n", - "- **scitex.db** → [19_scitex_db.ipynb](19_scitex_db.ipynb)\n", "- **scitex.tex** → [20_scitex_tex.ipynb](20_scitex_tex.ipynb)\n", "- **scitex.decorators** → [21_scitex_decorators.ipynb](21_scitex_decorators.ipynb)\n", "- **scitex.repro** → [22_scitex_repro.ipynb](22_scitex_repro.ipynb)\n", diff --git a/examples/_legacy/notebooks/19_scitex_db.ipynb b/examples/_legacy/notebooks/19_scitex_db.ipynb deleted file mode 100644 index cd55e36c0..000000000 --- a/examples/_legacy/notebooks/19_scitex_db.ipynb +++ /dev/null @@ -1,824 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Comprehensive SciTeX Database Module Examples\n", - "\n", - "This notebook demonstrates the complete functionality of the `scitex.db` module, which provides database operations and utilities for scientific data management.\n", - "\n", - "## Module Overview\n", - "\n", - "The `scitex.db` module includes:\n", - "- SQLite3 database management with comprehensive mixins\n", - "- PostgreSQL database operations\n", - "- Database inspection and analysis tools\n", - "- Duplicate data detection and removal\n", - "\n", - "## Import Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Detect notebook name for output directory\n", - "import os\n", - "from pathlib import Path\n", - "\n", - "# Get notebook name (for papermill compatibility)\n", - "notebook_name = \"19_scitex_db\"\n", - "if 'PAPERMILL_NOTEBOOK_NAME' in os.environ:\n", - " notebook_name = Path(os.environ['PAPERMILL_NOTEBOOK_NAME']).stem\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "sys.path.insert(0, '../src')\n", - "\n", - "import sqlite3\n", - "import pandas as pd\n", - "import numpy as np\n", - "import tempfile\n", - "import os\n", - "from pathlib import Path\n", - "\n", - "# Import scitex db module\n", - "import scitex.db as sdb\n", - "\n", - "# Set random seed for reproducibility\n", - "np.random.seed(42)\n", - "\n", - "db_attrs = [attr for attr in dir(sdb) if not attr.startswith('_')]\n", - "for i, attr in enumerate(db_attrs):\n", - " # Loop body" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Database Inspection Tools\n", - "\n", - "### Creating Sample Database\n", - "\n", - "Let's start by creating a sample database for demonstration purposes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 1: Create sample database for demonstration\n", - "# Create temporary database file\n", - "temp_db = tempfile.NamedTemporaryFile(suffix='.db', delete=False)\n", - "db_path = temp_db.name\n", - "temp_db.close()\n", - "\n", - "\n", - "# Create sample data\n", - "np.random.seed(42)\n", - "\n", - "# Experimental data table\n", - "experiments_data = {\n", - " 'experiment_id': range(1, 21),\n", - " 'subject_id': [f'S{i:03d}' for i in np.random.randint(1, 11, 20)],\n", - " 'condition': np.random.choice(['control', 'treatment_A', 'treatment_B'], 20),\n", - " 'measurement': np.random.normal(100, 15, 20),\n", - " 'timestamp': pd.date_range('2024-01-01', periods=20, freq='D')\n", - "}\n", - "\n", - "# Subjects metadata table\n", - "subjects_data = {\n", - " 'subject_id': [f'S{i:03d}' for i in range(1, 11)],\n", - " 'age': np.random.randint(18, 65, 10),\n", - " 'gender': np.random.choice(['M', 'F'], 10),\n", - " 'group': np.random.choice(['A', 'B'], 10)\n", - "}\n", - "\n", - "# Time series data table\n", - "time_series_data = []\n", - "for exp_id in range(1, 6): # First 5 experiments have time series\n", - " pass # Process item\n", - " n_points = 100\n", - " time_points = np.linspace(0, 10, n_points)\n", - " signal = np.sin(2 * np.pi * time_points) + 0.1 * np.random.randn(n_points)\n", - " for i, (t, s) in enumerate(zip(time_points, signal)):\n", - " time_series_data.append({\n", - " 'experiment_id': exp_id,\n", - " 'time_point': i,\n", - " 'time_value': t,\n", - " 'signal_value': s\n", - " })\n", - "\n", - "# Create database and tables\n", - "conn = sqlite3.connect(db_path)\n", - "\n", - "# Experiments table\n", - "experiments_df = pd.DataFrame(experiments_data)\n", - "experiments_df.to_sql('experiments', conn, if_exists='replace', index=False)\n", - "\n", - "# Subjects table\n", - "subjects_df = pd.DataFrame(subjects_data)\n", - "subjects_df.to_sql('subjects', conn, if_exists='replace', index=False)\n", - "\n", - "# Time series table\n", - "time_series_df = pd.DataFrame(time_series_data)\n", - "time_series_df.to_sql('time_series', conn, if_exists='replace', index=False)\n", - "\n", - "conn.close()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Database Inspection\n", - "\n", - "Now let's use the `inspect` function to examine our database." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 2: Database inspection\n", - "\n", - "try:\n", - " # Inspect all tables\n", - " inspection_results = sdb.inspect(db_path, verbose=True)\n", - " \n", - " \n", - " # Show structure of each inspection result\n", - " for i, result in enumerate(inspection_results):\n", - " pass # Processing i\n", - "except Exception as e: pass # Fixed incomplete except block\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 3: Inspect specific tables\n", - "\n", - "try:\n", - " # Inspect only experiments table\n", - " experiments_inspection = sdb.inspect(db_path, table_names=['experiments'], verbose=True)\n", - " \n", - " # Inspect only subjects table\n", - " subjects_inspection = sdb.inspect(db_path, table_names=['subjects'], verbose=True)\n", - " \n", - " \n", - "except Exception as e: pass # Fixed incomplete except block\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. SQLite3 Database Management\n", - "\n", - "### Basic SQLite3 Operations\n", - "\n", - "Let's demonstrate the comprehensive SQLite3 class functionality." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 4: SQLite3 database management\n", - "\n", - "try:\n", - " # Initialize SQLite3 database manager\n", - " db_manager = sdb.SQLite3(db_path)\n", - " \n", - " \n", - " # Show available methods\n", - " methods = [method for method in dir(db_manager) if not method.startswith('_')]\n", - " for i, method in enumerate(methods[:10]): # Show first 10 methods\n", - " if len(methods) > 10:\n", - " # Condition met\n", - " \n", - " # Call the database summary\n", - " summary = db_manager(print_summary=True, verbose=True)\n", - " \n", - "except Exception as e: pass # Fixed incomplete except block\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Database Querying\n", - "\n", - "Let's demonstrate database querying capabilities." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 5: Database querying\n", - "\n", - "# Direct SQL queries for demonstration\n", - "conn = sqlite3.connect(db_path)\n", - "\n", - "try:\n", - " # Query 1: Basic select\n", - " cursor = conn.cursor()\n", - " cursor.execute(\"SELECT * FROM experiments LIMIT 5\")\n", - " results = cursor.fetchall()\n", - " \n", - " # Get column names\n", - " cursor.execute(\"PRAGMA table_info(experiments)\")\n", - " columns = [col[1] for col in cursor.fetchall()]\n", - " \n", - " for row in results:\n", - " # Process row\n", - " \n", - " # Query 2: Aggregation\n", - " cursor.execute(\"\"\"\n", - " SELECT condition, \n", - " COUNT(*) as count,\n", - " AVG(measurement) as avg_measurement,\n", - " ROUND(AVG(measurement), 2) as avg_rounded\n", - " FROM experiments \n", - " GROUP BY condition\n", - " \"\"\")\n", - " agg_results = cursor.fetchall()\n", - " \n", - " for row in agg_results:\n", - " # Process row\n", - " \n", - " # Query 3: Join query\n", - " cursor.execute(\"\"\"\n", - " SELECT e.experiment_id, e.condition, e.measurement, s.age, s.gender\n", - " FROM experiments e\n", - " JOIN subjects s ON e.subject_id = s.subject_id\n", - " LIMIT 5\n", - " \"\"\")\n", - " join_results = cursor.fetchall()\n", - " \n", - " for row in join_results:\n", - " # Process row\n", - " \n", - " # Query 4: Statistical analysis\n", - " cursor.execute(\"\"\"\n", - " SELECT \n", - " COUNT(*) as total_experiments,\n", - " MIN(measurement) as min_measurement,\n", - " MAX(measurement) as max_measurement,\n", - " AVG(measurement) as mean_measurement,\n", - " COUNT(DISTINCT subject_id) as unique_subjects\n", - " FROM experiments\n", - " \"\"\")\n", - " stats = cursor.fetchone()\n", - " \n", - " \n", - "except Exception as e:\n", - " pass # Fixed incomplete except block\n", - " \n", - "finally:\n", - " conn.close()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Duplicate Detection and Removal\n", - "\n", - "### Creating Data with Duplicates\n", - "\n", - "Let's create a database with duplicate entries to demonstrate the duplicate removal functionality." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 6: Create database with duplicates\n", - "\n", - "# Create temporary database with duplicates\n", - "temp_dup_db = tempfile.NamedTemporaryFile(suffix='.db', delete=False)\n", - "dup_db_path = temp_dup_db.name\n", - "temp_dup_db.close()\n", - "\n", - "# Create sample data with intentional duplicates\n", - "np.random.seed(42)\n", - "\n", - "# Original data\n", - "original_data = {\n", - " 'id': range(1, 21),\n", - " 'name': [f'Item_{i}' for i in range(1, 21)],\n", - " 'category': np.random.choice(['A', 'B', 'C'], 20),\n", - " 'value': np.random.randint(1, 100, 20),\n", - " 'date': pd.date_range('2024-01-01', periods=20, freq='D')\n", - "}\n", - "\n", - "original_df = pd.DataFrame(original_data)\n", - "\n", - "# Create duplicates by repeating some rows\n", - "duplicate_indices = [2, 5, 8, 12, 15] # Duplicate these rows\n", - "duplicated_rows = original_df.iloc[duplicate_indices].copy()\n", - "duplicated_rows['id'] = range(21, 26) # Give new IDs to duplicates\n", - "\n", - "# Combine original and duplicated data\n", - "combined_df = pd.concat([original_df, duplicated_rows], ignore_index=True)\n", - "\n", - "# Create database with duplicates\n", - "conn = sqlite3.connect(dup_db_path)\n", - "combined_df.to_sql('test_data', conn, if_exists='replace', index=False)\n", - "conn.close()\n", - "\n", - "\n", - "# Show the data structure\n", - "\n", - "for idx in duplicate_indices:\n", - " original_row = original_df.iloc[idx]\n", - " duplicate_row = duplicated_rows[duplicated_rows.index == duplicate_indices.index(idx)].iloc[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Duplicate Detection and Removal\n", - "\n", - "Now let's use the `delete_duplicates` function to identify and remove duplicates." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 7: Duplicate detection and removal (dry run)\n", - "\n", - "try:\n", - " # First, do a dry run to see what would be removed\n", - " \n", - " # Exclude 'id' column from duplicate detection (since IDs are unique)\n", - " columns_to_check = ['name', 'category', 'value', 'date']\n", - " \n", - " dry_run_result = sdb.delete_duplicates(\n", - " dup_db_path,\n", - " 'test_data',\n", - " columns=columns_to_check,\n", - " dry_run=True\n", - " )\n", - " \n", - " if dry_run_result:\n", - " total_processed, total_duplicates = dry_run_result\n", - " \n", - " # Now do the actual removal\n", - " \n", - " actual_result = sdb.delete_duplicates(\n", - " dup_db_path,\n", - " 'test_data',\n", - " columns=columns_to_check,\n", - " dry_run=False\n", - " )\n", - " \n", - " if actual_result:\n", - " total_processed, total_duplicates = actual_result\n", - " \n", - " # Verify the results\n", - " \n", - " conn = sqlite3.connect(dup_db_path)\n", - " remaining_df = pd.read_sql_query(\"SELECT * FROM test_data\", conn)\n", - " conn.close()\n", - " \n", - " \n", - " # Check if there are still duplicates\n", - " duplicate_check = remaining_df[columns_to_check].duplicated().sum()\n", - " \n", - " if duplicate_check == 0:\n", - " # Condition met\n", - " else:\n", - " pass # Fixed incomplete block\n", - " \n", - " # Show final data\n", - " \n", - "except Exception as e: pass # Fixed incomplete except block\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. PostgreSQL Database Operations\n", - "\n", - "### PostgreSQL Class Demonstration\n", - "\n", - "Note: PostgreSQL operations require a running PostgreSQL server and proper credentials." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 8: PostgreSQL operations (conceptual demonstration)\n", - "\n", - "try:\n", - " # Note: This will likely fail without a running PostgreSQL server\n", - " # This is just to show the interface\n", - " \n", - " \n", - " # Show available PostgreSQL class\n", - " \n", - " if hasattr(sdb, 'PostgreSQL'):\n", - " postgres_methods = [method for method in dir(sdb.PostgreSQL) if not method.startswith('_')]\n", - " for i, method in enumerate(postgres_methods[:10]):\n", - " pass # Processing i\n", - " if len(postgres_methods) > 10:\n", - " # Condition met\n", - " \n", - " # Conceptual usage (would require actual PostgreSQL server)\n", - " \n", - "except Exception as e: pass # Fixed incomplete except block\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Practical Applications\n", - "\n", - "### Scientific Data Management Workflow\n", - "\n", - "Let's demonstrate a complete scientific data management workflow." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 9: Complete scientific data management workflow\n", - "\n", - "# Create a more complex scientific database\n", - "workflow_db = tempfile.NamedTemporaryFile(suffix='.db', delete=False)\n", - "workflow_db_path = workflow_db.name\n", - "workflow_db.close()\n", - "\n", - "\n", - "# Simulate experimental setup\n", - "np.random.seed(42)\n", - "\n", - "# 1. Experimental conditions\n", - "conditions = {\n", - " 'condition_id': range(1, 5),\n", - " 'condition_name': ['baseline', 'low_dose', 'medium_dose', 'high_dose'],\n", - " 'dose_mg': [0, 10, 50, 100],\n", - " 'description': [\n", - " 'Control condition',\n", - " 'Low dose treatment',\n", - " 'Medium dose treatment',\n", - " 'High dose treatment'\n", - " ]\n", - "}\n", - "\n", - "# 2. Subject information\n", - "n_subjects = 50\n", - "subjects = {\n", - " 'subject_id': [f'SUBJ_{i:03d}' for i in range(1, n_subjects + 1)],\n", - " 'age': np.random.randint(18, 80, n_subjects),\n", - " 'gender': np.random.choice(['M', 'F'], n_subjects),\n", - " 'weight_kg': np.random.normal(70, 15, n_subjects),\n", - " 'group': np.random.choice(['experimental', 'control'], n_subjects),\n", - " 'enrollment_date': pd.date_range('2024-01-01', periods=n_subjects, freq='D')\n", - "}\n", - "\n", - "# 3. Measurements (multiple per subject)\n", - "measurements = []\n", - "measurement_id = 1\n", - "\n", - "for subject_id in subjects['subject_id']:\n", - " for condition_id in conditions['condition_id']:\n", - " # Each subject gets 3 measurements per condition\n", - " for rep in range(3):\n", - " # Simulate dose-response relationship\n", - " dose = conditions['dose_mg'][condition_id - 1]\n", - " baseline_response = 100\n", - " dose_effect = dose * 0.5 + np.random.normal(0, 10)\n", - " response = baseline_response + dose_effect + np.random.normal(0, 5)\n", - " \n", - " measurements.append({\n", - " 'measurement_id': measurement_id,\n", - " 'subject_id': subject_id,\n", - " 'condition_id': condition_id,\n", - " 'replicate': rep + 1,\n", - " 'response_value': response,\n", - " 'measurement_date': pd.Timestamp('2024-01-01') + pd.Timedelta(days=measurement_id),\n", - " 'quality_score': np.random.uniform(0.7, 1.0)\n", - " })\n", - " measurement_id += 1\n", - "\n", - "# 4. Create database\n", - "conn = sqlite3.connect(workflow_db_path)\n", - "\n", - "# Create tables\n", - "conditions_df = pd.DataFrame(conditions)\n", - "subjects_df = pd.DataFrame(subjects)\n", - "measurements_df = pd.DataFrame(measurements)\n", - "\n", - "conditions_df.to_sql('conditions', conn, if_exists='replace', index=False)\n", - "subjects_df.to_sql('subjects', conn, if_exists='replace', index=False)\n", - "measurements_df.to_sql('measurements', conn, if_exists='replace', index=False)\n", - "\n", - "conn.close()\n", - "\n", - "\n", - "# 5. Inspect the database\n", - "try:\n", - " inspection_results = sdb.inspect(workflow_db_path, verbose=False)\n", - "except Exception as e:\n", - " pass # Fixed incomplete except block\n", - "\n", - "# 6. Perform scientific analysis queries\n", - "conn = sqlite3.connect(workflow_db_path)\n", - "\n", - "try:\n", - " # Analysis 1: Dose-response relationship\n", - " cursor = conn.cursor()\n", - " cursor.execute(\"\"\"\n", - " SELECT c.condition_name, c.dose_mg,\n", - " COUNT(m.measurement_id) as n_measurements,\n", - " AVG(m.response_value) as mean_response,\n", - " ROUND(AVG(m.response_value), 2) as mean_rounded,\n", - " MIN(m.response_value) as min_response,\n", - " MAX(m.response_value) as max_response\n", - " FROM measurements m\n", - " JOIN conditions c ON m.condition_id = c.condition_id\n", - " GROUP BY c.condition_id, c.condition_name, c.dose_mg\n", - " ORDER BY c.dose_mg\n", - " \"\"\")\n", - " \n", - " dose_response = cursor.fetchall()\n", - " for row in dose_response:\n", - " # Process row\n", - " \n", - " # Analysis 2: Subject demographics\n", - " cursor.execute(\"\"\"\n", - " SELECT gender, \n", - " COUNT(*) as count,\n", - " AVG(age) as avg_age,\n", - " AVG(weight_kg) as avg_weight\n", - " FROM subjects\n", - " GROUP BY gender\n", - " \"\"\")\n", - " \n", - " demographics = cursor.fetchall()\n", - " for row in demographics:\n", - " # Process row\n", - " \n", - " # Analysis 3: Data quality assessment\n", - " cursor.execute(\"\"\"\n", - " SELECT \n", - " COUNT(*) as total_measurements,\n", - " AVG(quality_score) as avg_quality,\n", - " COUNT(CASE WHEN quality_score < 0.8 THEN 1 END) as low_quality_count,\n", - " ROUND(COUNT(CASE WHEN quality_score < 0.8 THEN 1 END) * 100.0 / COUNT(*), 2) as low_quality_percent\n", - " FROM measurements\n", - " \"\"\")\n", - " \n", - " quality = cursor.fetchone()\n", - " \n", - "except Exception as e:\n", - " pass # Fixed incomplete except block\n", - " \n", - "finally:\n", - " conn.close()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Database Maintenance and Optimization\n", - "\n", - "Let's demonstrate database maintenance operations." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 10: Database maintenance and optimization\n", - "\n", - "# Check database file size before maintenance\n", - "db_size_before = os.path.getsize(workflow_db_path)\n", - "\n", - "# Perform maintenance operations\n", - "conn = sqlite3.connect(workflow_db_path)\n", - "cursor = conn.cursor()\n", - "\n", - "try:\n", - " # 1. Analyze database structure\n", - " cursor.execute(\"SELECT name FROM sqlite_master WHERE type='table'\")\n", - " tables = cursor.fetchall()\n", - " \n", - " for table in tables:\n", - " table_name = table[0]\n", - " cursor.execute(f\"SELECT COUNT(*) FROM {table_name}\")\n", - " row_count = cursor.fetchone()[0]\n", - " \n", - " # 2. Check for indexes\n", - " cursor.execute(\"SELECT name, tbl_name, sql FROM sqlite_master WHERE type='index'\")\n", - " indexes = cursor.fetchall()\n", - " \n", - " # 3. Create useful indexes for scientific queries\n", - " \n", - " # Index on measurements for faster joins\n", - " try:\n", - " cursor.execute(\"CREATE INDEX IF NOT EXISTS idx_measurements_subject ON measurements(subject_id)\")\n", - " cursor.execute(\"CREATE INDEX IF NOT EXISTS idx_measurements_condition ON measurements(condition_id)\")\n", - " cursor.execute(\"CREATE INDEX IF NOT EXISTS idx_measurements_date ON measurements(measurement_date)\")\n", - " except Exception as e:\n", - " pass # Fixed incomplete except block\n", - " \n", - " # 4. Database statistics\n", - " cursor.execute(\"PRAGMA database_list\")\n", - " db_info = cursor.fetchall()\n", - " \n", - " # Check page count and size\n", - " cursor.execute(\"PRAGMA page_count\")\n", - " page_count = cursor.fetchone()[0]\n", - " cursor.execute(\"PRAGMA page_size\")\n", - " page_size = cursor.fetchone()[0]\n", - " \n", - " \n", - " # 5. Vacuum database to reclaim space\n", - " cursor.execute(\"VACUUM\")\n", - " conn.commit()\n", - " \n", - " # 6. Update statistics\n", - " cursor.execute(\"ANALYZE\")\n", - " conn.commit()\n", - " \n", - "except Exception as e:\n", - " pass # Fixed incomplete except block\n", - " \n", - "finally:\n", - " conn.close()\n", - "\n", - "# Check database size after maintenance\n", - "db_size_after = os.path.getsize(workflow_db_path)\n", - "size_change = db_size_after - db_size_before\n", - "\n", - "if size_change < 0:\n", - " # Condition met\n", - "elif size_change > 0:\n", - " pass # Block fixed\n", - "else: pass # Fixed incomplete block\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. Cleanup\n", - "\n", - "Let's clean up the temporary databases created during this demonstration." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example 11: Cleanup temporary databases\n", - "\n", - "temp_databases = [\n", - " (db_path, \"Sample database\"),\n", - " (dup_db_path, \"Duplicate test database\"),\n", - " (workflow_db_path, \"Scientific workflow database\")\n", - "]\n", - "\n", - "for db_file, description in temp_databases:\n", - " try:\n", - " if os.path.exists(db_file):\n", - " size = os.path.getsize(db_file)\n", - " os.unlink(db_file)\n", - " else:\n", - " pass # Fixed incomplete block\n", - " except Exception as e:\n", - " pass # Fixed incomplete except block\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "This notebook has demonstrated the comprehensive functionality of the `scitex.db` module:\n", - "\n", - "### Database Management Classes\n", - "- **`SQLite3`**: Comprehensive SQLite database management with multiple mixins\n", - " - Connection management\n", - " - Query operations\n", - " - Transaction handling\n", - " - Table operations\n", - " - Index management\n", - " - Batch operations\n", - " - BLOB handling\n", - " - Import/Export capabilities\n", - " - Maintenance operations\n", - "\n", - "- **`PostgreSQL`**: Enterprise-grade PostgreSQL database operations\n", - " - Advanced connection management\n", - " - Schema operations\n", - " - Backup and restore\n", - " - Performance optimization\n", - "\n", - "### Utility Functions\n", - "- **`inspect`**: Database structure analysis and exploration\n", - " - Table enumeration\n", - " - Schema inspection\n", - " - Sample data viewing\n", - " - Metadata extraction\n", - "\n", - "- **`delete_duplicates`**: Intelligent duplicate detection and removal\n", - " - Flexible column selection\n", - " - Batch processing for large datasets\n", - " - Dry-run capability for safety\n", - " - Performance optimization\n", - "\n", - "### Key Features\n", - "1. **Scientific Focus**: Designed for research data management\n", - "2. **Robustness**: Comprehensive error handling and validation\n", - "3. **Performance**: Optimized for large scientific datasets\n", - "4. **Flexibility**: Support for various database operations\n", - "5. **Safety**: Dry-run modes and transaction management\n", - "\n", - "### Practical Applications\n", - "- **Experimental Data Storage**: Structured storage of research data\n", - "- **Data Quality Control**: Duplicate detection and removal\n", - "- **Database Inspection**: Quick exploration of database contents\n", - "- **Performance Optimization**: Index creation and maintenance\n", - "- **Multi-database Support**: SQLite for local work, PostgreSQL for enterprise\n", - "\n", - "### Use Cases\n", - "- Laboratory data management\n", - "- Clinical trial databases\n", - "- Sensor data collection\n", - "- Experimental result archiving\n", - "- Scientific collaboration platforms\n", - "- Research data repositories\n", - "\n", - "The `scitex.db` module provides a complete toolkit for scientific database management, from simple data storage to complex multi-table research databases with advanced querying and maintenance capabilities." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/_legacy/notebooks/27_scitex_scholar.ipynb b/examples/_legacy/notebooks/27_scitex_scholar.ipynb index 8cc89d4b9..c97ec68a8 100644 --- a/examples/_legacy/notebooks/27_scitex_scholar.ipynb +++ b/examples/_legacy/notebooks/27_scitex_scholar.ipynb @@ -52,10 +52,8 @@ "# Import the new simplified interface\n", "from scitex.scholar import Scholar\n", "\n", - "# Import impact factor tools for direct database access\n", - "import sqlite3\n", + "# Pandas for tabular views of the search results\n", "import pandas as pd\n", - "import impact_factor\n", "\n", "# Optional: Set up API keys for enhanced features\n", "import os\n", @@ -91,21 +89,7 @@ " if hasattr(paper, 'journal_quartile') and paper.journal_quartile:\n", " # Condition met\n", " if hasattr(paper, 'journal_ranking') and paper.journal_ranking:\n", - " # Condition met\n", - " \n", - " # Additional impact factor lookup if not automatically enriched\n", - " if paper.journal and (not hasattr(paper, 'impact_factor') or not paper.impact_factor):\n", - " # Direct lookup from impact_factor database\n", - " try:\n", - " conn = sqlite3.connect(impact_factor.DEFAULT_DB)\n", - " query = \"SELECT factor, jcr FROM factor WHERE journal LIKE ? ORDER BY factor DESC LIMIT 1\"\n", - " result = pd.read_sql_query(query, conn, params=[f'%{paper.journal}%'])\n", - " conn.close()\n", - " \n", - " if len(result) > 0:\n", - " # Condition met\n", - " except Exception as e:\n", - " pass # Fixed incomplete except block\n" + " # Condition met\n" ] }, { @@ -217,32 +201,10 @@ "quality_papers = ml_papers.filter(year_min=2018, min_citations=20)\n", "\n", "\n", - "# Enrich papers with impact factor information if not already enriched\n", + "# Impact-factor enrichment is done by Scholar itself (enrich_by_default=True);\n", + "# this helper just leaves an already-enriched paper untouched.\n", "def enrich_paper_with_impact_factor(paper):\n", - " \"\"\"Add impact factor information to a paper if available.\"\"\"\n", - " if not paper.journal:\n", - " return paper\n", - " \n", - " try:\n", - " conn = sqlite3.connect(impact_factor.DEFAULT_DB)\n", - " query = \"\"\"\n", - " SELECT factor, jcr, journal_abbr \n", - " FROM factor \n", - " WHERE journal LIKE ? \n", - " ORDER BY factor DESC \n", - " LIMIT 1\n", - " \"\"\"\n", - " result = pd.read_sql_query(query, conn, params=[f'%{paper.journal}%'])\n", - " conn.close()\n", - " \n", - " if len(result) > 0:\n", - " paper.impact_factor = result.iloc[0]['factor']\n", - " paper.journal_quartile = result.iloc[0]['jcr']\n", - " paper.journal_abbr = result.iloc[0]['journal_abbr']\n", - " \n", - " except Exception as e:\n", - " pass # Fixed incomplete except block\n", - " \n", + " \"\"\"Return the paper as Scholar enriched it.\"\"\"\n", " return paper\n", "\n", "# Enrich all papers\n", diff --git a/examples/_legacy/scitex/clew/multi_parent/scitex/clew.db b/examples/_legacy/scitex/clew/multi_parent/scitex/clew.db deleted file mode 100644 index 01dc4efe04ad7e53d2586a708897e56166edb8be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 94208 zcmeI5Yiu0Xb%6K9eURK8ih68l*YP&~VX?Bvc|Yg~E=8>rIU*&?lx5X*r}MZ&uEgD? zcXmZFtsSiFCV_(nEl{`p)1a>(MbXAB3N-%H7)@LMY0(;R9|_!|X@B*F0s+#(X`7;F zW_I>D!)2urLnmheQL{7Wo_o)C&%HDEo;!D+edbc7NlI3|QPr9yE;bX3$74TSD#c>4 z=ipxv{`sF7I7sT|Ke|K`6l{hg_QocxpAZ)Sfcvy*;#;#D|}A0&VTkiY{c(7tnG zy4;plmzN*T##Uo2&5Acp)71%Q8op+%JR0lBiy;PxLL1~#oZmPsns-{EN(aI zHy3RKrZMlF-rj9))@zI8j#k~?B6jEWqOnDG7OSnTW<}lB8l=`-q!slxX;iFT)zmf? zws!?n(JWmti77Fn#B-*lDVoAFgwq(4GbN`$P#!4GI-@KJ@)9d8Ff1>KT+kIxVi<|D zTZTbwTcAZnVMWg5Se=(?Ml@wj(`mvnJY#CK+}=6XF?}u$wMFIdAcd!4rX#B}pQ>AThEo z(wsyjMbbHXxN7+&o?T!VS`>vL%e^ZcFVU=Lxxs6!1-`+^#1v(j*BM5aHJ;X-uL4Kz zo|pm4pLE^59OUkNl@)Duce5r&W4WwWwOd5BDjTf^vBBY?q#|$fl4)A3Vlk`;1ttpN zbeT0}UKp-heuP=WrzA;SOxkv0jViww(fl1A&6 zVsH$nn>?dg!(9Qt#K;Q*Sj8$MUIAAmfsq6{lnhrD1Y6x$7uQV1G#LhJ!{T_0l?1@T z(SpSpiY|+iAm07RG>~~}FETW!I;-9MR5WBn75dk$-A@s8 zWP{fg$>9QBq@tLHHUfZ_0D!E}qwYk@(3BWK*$WMLl-hdwrXpIVWC)VXSb`++BFibx zSB?Wd(A${W-G?Rt%}O|$9_47z2wgNf+0<@V>J5M+2}BWOhKQDB0v+CxMV^(wk7${J z_CE}VTN0!Nft3}OAHwEdkr-Nl`t)ugqb-W7`fxZ7+6~PJ7Sx2Gi82Rw8=2>MgC@Mh zF~;4GWcT))o~mfzaB8DfQ^}oexW886UJGb=4$5aLJf};F#tMw0Ns??@@Zq54Tf^Nc zx5SGJa33D+Xx%Fc+@ImT5RQhSZB==iU4sr>W@v-Z&^%?GShONCg2}K%Vl6{r+4jq6 zK(p-nP2ioEt8yGXblj%TMT3S{x9Xay8ui;+qoOt4j##D%t!Wl*5oqSLE=wj1?T09OQX zqi(lK)3U~LfD}4UVnAm}M9vZvmeT;Btca8FWIt7SBS!r_^%d%usGp=>pq8k)!Uu)F zDf~X1#t#xe0!RP}AOR$R1dsp{Kmter2_S(7N1&7}$DJ|n;3RS{0(2r+pcf}O7A(*U zC(MQk__OXaVFLc_b0JK?pM;$b7U)eyPKF8iGke)E0e?~|9VXz<+N8n+{8^2J6aSx0 zBx00UsO5is`u*u+Q=goCE%%k|e`FUkR{B>azCLk0bvgNZ;vEpe&-WmK_VR2tR&HNg zopa+|gKh4?hN0AYtGVcfq}YY++J-KgoMea!17TH3BRr$P^qmfIDN7?POyGg|cB|Q& z4J*;|lHkOj+RH}{fenmf158~)l#Gb7!WaU?ZWxZRx&*V_tN?*URtU3!Zl9fTY`C#n zJb-!_ih8*fVbrU7rAEvJ!@jNPnhY@=h||H$tf-g}=CNp)Fw`wUhbV_xsRdO8E4NoD z&xl}W@;lFHCR6X%h>)sa5 zp$<$|6Lgy4x!{@r8_Idd2K7K}7>$RyR&;}7V21>;C|Q;nftMwYWmydxxF}g74WVOM zWcW}!&Q5!E5$sX^-8LGJy3e`@aVS{gaem4%LJO&kXpI;NGId&aH|7MjVL>#2lSGKZ z$r?i}hz0Nvh3A~6Ef^4U39b!VYOhQ>hP-efh71Q1T{{}u4lLPeYECajEZtC8Sf*pb zvI_{55s3T<1|#Y$EWFVKCe(`aIj?O5`Bd)#EgGMSplt;Dl@ldyuVllkBWgn)NOibY zoXdDt1XqXuI2=uN+_x1WmBEYc^J&M7;6@&Gko`e5awovvJ~!bQ@?u~kkJ^yYkg(g# z{qQifa)x6djLjK5M8#p*96U}bk_k_TCaf9KMJw?z)CCz;=V!Mm21d_q-RKQ`|wAD(X@{!94n-WaALch@H{Kn z5`WkljV1T*Ss`|?Li<8|uNv`3*U{J!g=>Xa!f;Ofe|F*zV$@dQ>xFaq-^qV;>VxUm zr#2?vn>?HQQtqSK&u9K8b1VIR`Vt7?2MHhnB!C1C2Z44z4HI{B0dwUCJ8$RRU1kqK zaoj05uQ*5A1)y`IcmQ<9O+>j5b~A0tDNk^1MLVT5t~?$f%B`>K!GNMI(3p7@mrtR6 z1o)^2QJj4Y$=xF5T^G6@TxHQt+KeIO)!1}<77%H{h@vgeeue=qqbb*|UpR;|jUBqX z1)6jrsX>(%^^C|q{K%`gTsWkt%d|g8+4c;0(hI?m4tT=FLzD^k?Wm_5_U+qgm(q(t zz8!U$#t&9q=bkt?O8Xghx@AhWk2<|lurD3_x=N%wB@VsN1jdY-{6Qo^gdx!mZ| z2CjPWQwCn4^3=b@lGHmf>K*ER>RX57u@*~#1dsp{Kmter2_OL^fCP{L5~Lj&j7lUQi(9Y{le($do5Y@llvyYBWHLDyH@37&)lRta|Lj_f z`q{z?^;rI|razee{M4I;-^-J!r>1{kl1%<(?!R(h$bBsP76{=72_OL^fCP{L53XWIMP%82qS8}asrCyIY>873tuaIx$yIaR$;xs6=w6_%)bjl_(1|l00|%gB!C2v01`j~NB{{SfrBCtyq{a| zZNIVm+8r6EVeH0tRG0?;m9fyB@qB@Nml6=^xnn^ z-F(k~J0=sR!50YKozEAT7^fk0`#%4zie#7u{|$%4qsj7~?;E(|f9hO}`fKXXsIOAL zP5mPE1!|Y7Q$IreC-tu|EAVCNSE*Mile!E-_(1|l00|%gB!C2v01`j~NB{{S0VME! zOkfh0(|d2z&-MyFbEA?LaC3eRDQ`9Rl>J933>RZ%5QGZLlOZ_qR7WG@yFT-j4AOR$R1dsp{ zKmter2_OL^fCP{L5^si(9jE(_Q+>zo`~OqanHcro5byscb%8odoq-vEH(?b(mAZ7_2BHEafCP{L54np3X=>672$cqs@fzbCJ z{NESIIwegze^Sn$MBb@|nGb(epan<(2_OL^fCP{L5itYl6z(B)Z@``pO}(FKhul4-_J{V_kkq}>jS}s> zc0UoC+SebH+MhZ|-cxFSnjrCUsPDkv_}eRQe@eg|{};X%gJ1f4oBAU4GOPl4l6t7{ zZ8(J=B!C2v01`j~NB{{S0VIF~kN^@u0*9VJDY^F?$k8wXf0`p?FFwBN3*^Hz_>&d6 zaRS+3f!>tEM3{g-vQLf^NIaU{dv0K8{Qsmo1AycIL+{^MKqP$OF4 zN2_jc5xaAG(byt8i`CXvv!ZTm4N_|^R<&BoHX4=f<^rUXM;rtiVZx74}5!#Qzf%y#N0m z^)LH#a+E>>NB{{S0VIF~kN^@u0!RP}AOR$R1P%d#6G~>HcXHZ~T8b zM*RkL`4E&Hi+}`>01`j~NB{{S0VIF~kN^@u0!ZM)K;Zcltp123=s&nLA#oiP=N^D! zZ~T8MMxCO5;lltJ%ZLP!01`j~NB{{S0VIF~kN^@u0!ZKi6S$RX--u>xKlbg3#Ad2} zEt=u}n3o5<@&DT~*!}+-511dSM*>Iy2_OL^fCP{L5Kz|^Nu4nwsvb@s!PWl&vqHy;SZPTRJ&e8v0aB zfV5t|P`+Baa`ocs^3@xqPnK_#manZ}zPJV|R?BPaPmH8;vrr-JdinYF(zUgV&s-~) z)-JD?)~;Q;G%DULuqPibxCL8ysI^vAhutnZ#TwP-W!~J~Cgaq3lDD<3mM1!@Yq0iR zeW5~bEjR{;N^ENOW{8g71|zE0EUjN$EkC=yyn3Z{u6%y^+NJf<%C)Ok0kzt_GMdS4 z(x_M!Vh$)66>~OZY1tkcU;E>|>>nSf%=uHZ>Ecs!@tEtC_Dfr^7mTX4nx2>XZHw_- z>S+7eOs2SeD&D@>b>dF5>ue5V`~Q25(f@X2#Ib{u4HV6HRA{P0(kH#0Xu55-JxgVZ zD^RSJ&|-Bo3@Mr!Tdq!m{T9sEUoq!j%jGl0Q>Wsez34Vg|B6?1d+2Lnps@~J3~03u zn|o`yPD)?D-+sf}qa!^Cdt8%qQZ$HBZ$zF-)OIOS1 z%g|I-%FmV>t(x6`ZL~3rXd1NU9<+I#U0<$G3=x_iahCZrg>>F~ zYM+?S6we6p_Jyv0dwqccTQAjXemfXA*V!8>KjPs7Qk|SeO6*{Rb~hl~(b#AC{Hxql zrYH#U&*WSm4-yYLH$C9(K@#6(m-mWvpANi5Zj-J3G?)Gkd+(-rFKuF^Zs#{Ap<_B8 zZ$HxYCZ`>B`fBHJYQ$;!1V;3Ae8p{xL#^w_b6!sQ!JIsUodlD?Ie7wm=H!XY|5!Ft zeB`lsJJ-vhzD3;U4d-S|?A&M$AP}ayFX71s$j#r(WQvc0{)fBzJ58ofGCPvRm}b?N zw!bx1)0yJqkXNr>TtVkSuP;SrB)6{Aeyr*W&tI8<=h0Jf@5!-q=eNmAEmAYc(ARV) zPw!cB=;FzmRwYkB%K2A+BI(ll>>0N^=qq&JVQN5E--YjTo9}|zPm}FF=$D5U zhxViIklir@?r~i_Pq;<}+~kB*x_I%iu$x?8MXOBTmx)&|#52X?$K#)Q$#oj1S8)C% z2HeCs6XGBK&C-9E=^MoaKMr_%7eaT}40~30YOW*It>_S4O|8)!x=#d4T9sM_`lE3V z?{3z;2l!xJFz#)tM%^UgV Date: Mon, 31 Aug 2026 14:37:28 +0000 Subject: [PATCH 2/2] chore: stop the clew runtime store from being committed again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running the suite after deleting the two committed database files re-created `.scitex/clew/runtime/clew.db` (84 KB, magic bytes `SQLite format 3`). The writer is the `scitex-clew` package pinned in `pyproject.toml`, so this repo cannot stop the write — but it can stop the file from being re-committed, which is how it got in. Nothing was ignoring that path: the previous file was TRACKED, so it never showed up as untracked noise and nobody added a rule. --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 7a4f50d38..6422ca207 100755 --- a/.gitignore +++ b/.gitignore @@ -952,3 +952,7 @@ src/scitex/_sphinx_html/ .venv/ # Claude worktree symlink /feat-enhance-interface_04_skills + +# clew's runtime store is written by the scitex-clew package at import +# time; it is machine-local state and must never be committed again. +.scitex/clew/runtime/