diff --git a/src/couchdb/.allowed_datafiles b/src/couchdb/.allowed_datafiles index 1654f8377..b5ef74124 100644 --- a/src/couchdb/.allowed_datafiles +++ b/src/couchdb/.allowed_datafiles @@ -13,4 +13,6 @@ src/couchdb/scenarios_data/shared/iot/hydraulic_pump_1.json src/couchdb/scenarios_data/shared/iot/metro_pump_1.json src/couchdb/scenarios_data/shared/iot/motor_01.json src/couchdb/scenarios_data/shared/work_order/workorders.csv +src/couchdb/scenarios_data/shared/tsfm/feature_catalog.json +src/couchdb/scenarios_data/shared/tsfm/model_catalog.json src/couchdb/scenarios_data/shared/iot/asset_profile_sample.json diff --git a/src/couchdb/collections.json b/src/couchdb/collections.json index db67d67f3..9541c50e6 100644 --- a/src/couchdb/collections.json +++ b/src/couchdb/collections.json @@ -143,5 +143,44 @@ ], "id_prefix": "fc", "indexes": [] + }, + "model_catalog": { + "format": "json", + "primary_key": [ + "model_id" + ], + "id_prefix": "model", + "indexes": [ + [ + "task_ids" + ], + [ + "status" + ], + [ + "model_family" + ], + [ + "domain" + ] + ] + }, + "feature_catalog": { + "format": "json", + "primary_key": [ + "feature_id" + ], + "id_prefix": "feature", + "indexes": [ + [ + "scenario_categories" + ], + [ + "kind" + ], + [ + "status" + ] + ] } } diff --git a/src/couchdb/scenarios_data/default/manifest.json b/src/couchdb/scenarios_data/default/manifest.json index 89d7f5ca3..8c6b39378 100644 --- a/src/couchdb/scenarios_data/default/manifest.json +++ b/src/couchdb/scenarios_data/default/manifest.json @@ -7,6 +7,8 @@ ], "asset": "shared/iot/asset_profile_sample.json", "vibration": "shared/iot/motor_01.json", + "model_catalog": "shared/tsfm/model_catalog.json", + "feature_catalog": "shared/tsfm/feature_catalog.json", "catalog": [ "shared/catalog/assets.csv", "shared/catalog/failure_modes.csv", diff --git a/src/couchdb/scenarios_data/shared/tsfm/feature_catalog.json b/src/couchdb/scenarios_data/shared/tsfm/feature_catalog.json new file mode 100644 index 000000000..eae06777c --- /dev/null +++ b/src/couchdb/scenarios_data/shared/tsfm/feature_catalog.json @@ -0,0 +1,2811 @@ +[ + { + "_id": "feature:efe_time_robust_norm_v1", + "feature_id": "efe_time_robust_norm_v1", + "name": "Invertible robust normalization", + "modality": "timeseries", + "interface": "fit_transform_inverse", + "class_name": "Transformation", + "invertible": true, + "code": "import numpy as np\nclass Transformation:\n \"\"\"Invertible robust per-channel normalization (median/IQR) for forecasting inputs.\"\"\"\n def fit(self, X, metadata):\n X = np.asarray(X, dtype=float)\n center = np.median(X, axis=0)\n q1, q3 = np.percentile(X, 25, axis=0), np.percentile(X, 75, axis=0)\n scale = np.where((q3 - q1) > 1e-8, (q3 - q1), 1.0)\n return {\"center\": center, \"scale\": scale}\n def transform(self, X, state):\n X = np.asarray(X, dtype=float)\n return (X - state[\"center\"]) / state[\"scale\"]\n def inverse_transform(self, Y, state):\n Y = np.asarray(Y, dtype=float)\n return Y * state[\"scale\"] + state[\"center\"]\n", + "scenario_types": [ + "Forecasting", + "Prognostics" + ], + "provenance": "evolved", + "method": "EFE-Time", + "parent_feature_id": null, + "generation": 0, + "target_task": "tsfm_forecasting", + "target_model": null, + "dataset": null, + "output_type": "normalized_series", + "metrics": [], + "columns_added": [], + "columns_dropped": [], + "validity": { + "entry_points": true, + "no_inplace": true, + "invertible_ok": true, + "leakage_checked": true + }, + "tags": [ + "timeseries", + "EFE-Time", + "Future State Prediction" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-20T00:00:00+00:00", + "kind": "transform", + "description": "Invertible per-channel robust normalization (median/IQR) for forecasting inputs." + }, + { + "_id": "feature:lag_rolling_features_v1", + "feature_id": "lag_rolling_features_v1", + "name": "Causal lag/rolling features", + "modality": "timeseries", + "interface": "fit_transform", + "class_name": "Transformation", + "invertible": false, + "code": "import numpy as np\nclass Transformation:\n \"\"\"Append causal rolling-mean channels (lag/rolling features) for forecasting.\"\"\"\n def fit(self, X, metadata):\n return {\"w\": int(metadata.get(\"window\", 8))}\n def transform(self, X, state):\n X = np.asarray(X, dtype=float); X = X.reshape(len(X), -1); w = state[\"w\"]\n cs = np.cumsum(X, axis=0); roll = np.copy(X)\n if len(X) > w:\n roll[w:] = (cs[w:] - cs[:-w]) / w\n roll[:w] = cs[:w] / np.arange(1, min(w, len(X)) + 1)[:, None]\n return np.concatenate([X, roll], axis=1)\n", + "scenario_types": [ + "Forecasting" + ], + "provenance": "handwritten", + "method": "handwritten", + "parent_feature_id": null, + "generation": 0, + "target_task": "tsfm_forecasting", + "target_model": null, + "dataset": null, + "output_type": "augmented_channels", + "metrics": [], + "columns_added": [], + "columns_dropped": [], + "validity": { + "entry_points": true, + "no_inplace": true, + "invertible_ok": null, + "leakage_checked": true + }, + "tags": [ + "timeseries", + "handwritten", + "Future State Prediction", + "Data Extraction" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-20T00:00:00+00:00", + "kind": "transform", + "description": "Appends causal lag/rolling-mean channels for forecasting." + }, + { + "_id": "feature:spectral_residual_v1", + "feature_id": "spectral_residual_v1", + "name": "Rolling-median residual z-score", + "modality": "timeseries", + "interface": "fit_transform", + "class_name": "Transformation", + "invertible": false, + "code": "import numpy as np\nclass Transformation:\n \"\"\"Robust residual + MAD z-score from a rolling-median baseline (anomaly features).\"\"\"\n def fit(self, X, metadata):\n return {\"w\": int(metadata.get(\"window\", 16))}\n def transform(self, X, state):\n X = np.asarray(X, dtype=float).reshape(len(X), -1); w = state[\"w\"]\n base = np.copy(X)\n for i in range(len(X)):\n base[i] = np.median(X[max(0, i - w + 1):i + 1], axis=0)\n resid = X - base\n mad = np.median(np.abs(resid - np.median(resid, axis=0)), axis=0) + 1e-8\n return resid / (1.4826 * mad)\n", + "scenario_types": [ + "AnomalyDetection" + ], + "provenance": "handwritten", + "method": "handwritten", + "parent_feature_id": null, + "generation": 0, + "target_task": "tsfm_anomaly_detection", + "target_model": null, + "dataset": null, + "output_type": "anomaly_features", + "metrics": [], + "columns_added": [], + "columns_dropped": [], + "validity": { + "entry_points": true, + "no_inplace": true, + "invertible_ok": null, + "leakage_checked": true + }, + "tags": [ + "timeseries", + "handwritten", + "Anomaly & Exception Detection" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-20T00:00:00+00:00", + "kind": "transform", + "description": "Robust residual + MAD z-score from a rolling-median baseline (anomaly features)." + }, + { + "_id": "feature:trend_slope_v1", + "feature_id": "trend_slope_v1", + "name": "Rolling local slope (degradation)", + "modality": "timeseries", + "interface": "fit_transform", + "class_name": "Transformation", + "invertible": false, + "code": "import numpy as np\nclass Transformation:\n \"\"\"Rolling local linear slope per channel (degradation / trend features).\"\"\"\n def fit(self, X, metadata):\n return {\"w\": int(metadata.get(\"window\", 24))}\n def transform(self, X, state):\n X = np.asarray(X, dtype=float).reshape(len(X), -1); w = state[\"w\"]; out = np.zeros_like(X)\n for i in range(len(X)):\n seg = X[max(0, i - w + 1):i + 1]; m = seg.shape[0]\n tt = np.arange(m) - (m - 1) / 2.0; d = (tt ** 2).sum() or 1.0\n out[i] = (tt[:, None] * seg).sum(axis=0) / d\n return out\n", + "scenario_types": [ + "Analytics" + ], + "provenance": "handwritten", + "method": "handwritten", + "parent_feature_id": null, + "generation": 0, + "target_task": "tsfm_forecasting_evaluation", + "target_model": null, + "dataset": null, + "output_type": "trend_features", + "metrics": [], + "columns_added": [], + "columns_dropped": [], + "validity": { + "entry_points": true, + "no_inplace": true, + "invertible_ok": null, + "leakage_checked": true + }, + "tags": [ + "timeseries", + "handwritten", + "Analysis & Inference" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-20T00:00:00+00:00", + "kind": "transform", + "description": "Rolling local linear slope per channel (degradation/trend features)." + }, + { + "_id": "feature:channel_select_v1", + "feature_id": "channel_select_v1", + "name": "Relevant-sensor channel selection", + "modality": "timeseries", + "interface": "fit_transform", + "class_name": "Transformation", + "invertible": false, + "code": "import numpy as np\nclass Transformation:\n \"\"\"Select the relevant sensor channels (e.g. PS1-PS6, FS1/FS2, VS1 from a pump scenario).\n Indices come from the scenario's relevant_sensors via metadata['channel_indices'].\"\"\"\n def fit(self, X, metadata):\n return {\"idx\": list(metadata.get(\"channel_indices\", []))}\n def transform(self, X, state):\n X = np.asarray(X, dtype=float).reshape(len(X), -1)\n idx = state[\"idx\"] or list(range(X.shape[1]))\n return X[:, idx]\n", + "scenario_types": [ + "AnomalyDetection", + "Forecasting" + ], + "provenance": "handwritten", + "method": "handwritten", + "parent_feature_id": null, + "generation": 0, + "target_task": "tsfm_anomaly_detection", + "target_model": null, + "dataset": null, + "output_type": "selected_channels", + "metrics": [], + "columns_added": [], + "columns_dropped": [], + "validity": { + "entry_points": true, + "no_inplace": true, + "invertible_ok": null, + "leakage_checked": true + }, + "tags": [ + "timeseries", + "handwritten", + "Data Extraction", + "Anomaly & Exception Detection", + "Future State Prediction" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-20T00:00:00+00:00", + "kind": "transform", + "description": "Selects the most relevant sensor channels (dimensionality reduction)." + }, + { + "_id": "feature:condition_index_v1", + "feature_id": "condition_index_v1", + "name": "Weighted condition/health index", + "modality": "timeseries", + "interface": "fit_transform", + "class_name": "Transformation", + "invertible": false, + "code": "import numpy as np\nclass Transformation:\n \"\"\"Aggregate normalized channels into a single condition/health index (decision support).\"\"\"\n def fit(self, X, metadata):\n X = np.asarray(X, dtype=float)\n return {\"mean\": X.mean(axis=0), \"std\": X.std(axis=0) + 1e-8,\n \"w\": np.asarray(metadata.get(\"weights\", []), dtype=float)}\n def transform(self, X, state):\n X = np.asarray(X, dtype=float).reshape(len(X), -1)\n z = (X - state[\"mean\"]) / state[\"std\"]\n w = state[\"w\"] if state[\"w\"].size == z.shape[1] else np.ones(z.shape[1]) / z.shape[1]\n return (z * w).sum(axis=1, keepdims=True)\n", + "scenario_types": [ + "DecisionSupport", + "Prognostics" + ], + "provenance": "handwritten", + "method": "handwritten", + "parent_feature_id": null, + "generation": 0, + "target_task": "tsfm_forecasting_evaluation", + "target_model": null, + "dataset": null, + "output_type": "condition_index", + "metrics": [], + "columns_added": [], + "columns_dropped": [], + "validity": { + "entry_points": true, + "no_inplace": true, + "invertible_ok": null, + "leakage_checked": true + }, + "tags": [ + "timeseries", + "handwritten", + "Recommendation & Optimization", + "Analysis & Inference" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-20T00:00:00+00:00", + "kind": "transform", + "description": "Weighted condition/health index" + }, + { + "feature_id": "mean", + "name": "mean", + "description": "Average value of the window.", + "kind": "extractor", + "extractor_name": "mean", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "mean" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:std", + "feature_id": "std", + "name": "FLOps extractor: std", + "description": "Standard deviation (spread).", + "kind": "extractor", + "extractor_name": "std", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "std" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:min", + "feature_id": "min", + "name": "FLOps extractor: min", + "description": "Minimum value.", + "kind": "extractor", + "extractor_name": "min", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "min" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:max", + "feature_id": "max", + "name": "FLOps extractor: max", + "description": "Maximum value.", + "kind": "extractor", + "extractor_name": "max", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "max" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:range", + "feature_id": "range", + "name": "FLOps extractor: range", + "description": "Max minus min (peak-to-peak spread).", + "kind": "extractor", + "extractor_name": "range", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "range" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:q25", + "feature_id": "q25", + "name": "FLOps extractor: q25", + "description": "25th percentile (lower quartile).", + "kind": "extractor", + "extractor_name": "q25", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "q25" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:q75", + "feature_id": "q75", + "name": "FLOps extractor: q75", + "description": "75th percentile (upper quartile).", + "kind": "extractor", + "extractor_name": "q75", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "q75" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:kurtosis", + "feature_id": "kurtosis", + "name": "FLOps extractor: kurtosis", + "description": "Tailedness/peakedness of the distribution (excess kurtosis).", + "kind": "extractor", + "extractor_name": "kurtosis", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "kurtosis" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:skew", + "feature_id": "skew", + "name": "FLOps extractor: skew", + "description": "Asymmetry of the distribution.", + "kind": "extractor", + "extractor_name": "skew", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "skew" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:slope", + "feature_id": "slope", + "name": "FLOps extractor: slope", + "description": "Linear trend slope over the window.", + "kind": "extractor", + "extractor_name": "slope", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "slope" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:autocorr1", + "feature_id": "autocorr1", + "name": "FLOps extractor: autocorr1", + "description": "Lag-1 autocorrelation (short-term persistence).", + "kind": "extractor", + "extractor_name": "autocorr1", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "autocorr1" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:energy", + "feature_id": "energy", + "name": "FLOps extractor: energy", + "description": "Mean squared value (average power).", + "kind": "extractor", + "extractor_name": "energy", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "energy" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:abs_diff_mean", + "feature_id": "abs_diff_mean", + "name": "FLOps extractor: abs_diff_mean", + "description": "Mean absolute first difference (roughness).", + "kind": "extractor", + "extractor_name": "abs_diff_mean", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "abs_diff_mean" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:spectral_centroid", + "feature_id": "spectral_centroid", + "name": "FLOps extractor: spectral_centroid", + "description": "Center of mass of the frequency spectrum (brightness).", + "kind": "extractor", + "extractor_name": "spectral_centroid", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "spectral_centroid" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:dominant_freq_power", + "feature_id": "dominant_freq_power", + "name": "FLOps extractor: dominant_freq_power", + "description": "Power of the strongest non-DC frequency.", + "kind": "extractor", + "extractor_name": "dominant_freq_power", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "dominant_freq_power" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:var", + "feature_id": "var", + "name": "FLOps extractor: var", + "description": "Variance of the window.", + "kind": "extractor", + "extractor_name": "var", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "var" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:median", + "feature_id": "median", + "name": "FLOps extractor: median", + "description": "Median value (robust center).", + "kind": "extractor", + "extractor_name": "median", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "median" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:q05", + "feature_id": "q05", + "name": "FLOps extractor: q05", + "description": "5th percentile.", + "kind": "extractor", + "extractor_name": "q05", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "q05" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:q10", + "feature_id": "q10", + "name": "FLOps extractor: q10", + "description": "10th percentile.", + "kind": "extractor", + "extractor_name": "q10", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "q10" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:q90", + "feature_id": "q90", + "name": "FLOps extractor: q90", + "description": "90th percentile.", + "kind": "extractor", + "extractor_name": "q90", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "q90" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:q95", + "feature_id": "q95", + "name": "FLOps extractor: q95", + "description": "95th percentile.", + "kind": "extractor", + "extractor_name": "q95", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "q95" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:iqr", + "feature_id": "iqr", + "name": "FLOps extractor: iqr", + "description": "Inter-quartile range (robust spread).", + "kind": "extractor", + "extractor_name": "iqr", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "iqr" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:mad", + "feature_id": "mad", + "name": "FLOps extractor: mad", + "description": "Median absolute deviation (robust spread).", + "kind": "extractor", + "extractor_name": "mad", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "mad" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:mean_abs", + "feature_id": "mean_abs", + "name": "FLOps extractor: mean_abs", + "description": "Mean of absolute values.", + "kind": "extractor", + "extractor_name": "mean_abs", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "mean_abs" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:sum_abs", + "feature_id": "sum_abs", + "name": "FLOps extractor: sum_abs", + "description": "Sum of absolute values.", + "kind": "extractor", + "extractor_name": "sum_abs", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "sum_abs" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:rms", + "feature_id": "rms", + "name": "FLOps extractor: rms", + "description": "Root-mean-square amplitude.", + "kind": "extractor", + "extractor_name": "rms", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "rms" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:abs_energy", + "feature_id": "abs_energy", + "name": "FLOps extractor: abs_energy", + "description": "Sum of squared values (total energy).", + "kind": "extractor", + "extractor_name": "abs_energy", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "abs_energy" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:cv", + "feature_id": "cv", + "name": "FLOps extractor: cv", + "description": "Coefficient of variation (std/|mean|).", + "kind": "extractor", + "extractor_name": "cv", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "cv" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:std_to_mean", + "feature_id": "std_to_mean", + "name": "FLOps extractor: std_to_mean", + "description": "Std divided by mean (relative dispersion).", + "kind": "extractor", + "extractor_name": "std_to_mean", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "std_to_mean" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:count_above_mean", + "feature_id": "count_above_mean", + "name": "FLOps extractor: count_above_mean", + "description": "Number of points above the mean.", + "kind": "extractor", + "extractor_name": "count_above_mean", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "count_above_mean" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:count_below_mean", + "feature_id": "count_below_mean", + "name": "FLOps extractor: count_below_mean", + "description": "Number of points below the mean.", + "kind": "extractor", + "extractor_name": "count_below_mean", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "count_below_mean" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:ratio_above_mean", + "feature_id": "ratio_above_mean", + "name": "FLOps extractor: ratio_above_mean", + "description": "Fraction of points above the mean.", + "kind": "extractor", + "extractor_name": "ratio_above_mean", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "ratio_above_mean" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:count_above_2std", + "feature_id": "count_above_2std", + "name": "FLOps extractor: count_above_2std", + "description": "Number of points beyond 2 std (outliers).", + "kind": "extractor", + "extractor_name": "count_above_2std", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "count_above_2std" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:abs_max", + "feature_id": "abs_max", + "name": "FLOps extractor: abs_max", + "description": "Maximum absolute value.", + "kind": "extractor", + "extractor_name": "abs_max", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "abs_max" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:abs_min", + "feature_id": "abs_min", + "name": "FLOps extractor: abs_min", + "description": "Minimum absolute value.", + "kind": "extractor", + "extractor_name": "abs_min", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "abs_min" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:intercept", + "feature_id": "intercept", + "name": "FLOps extractor: intercept", + "description": "Mean level (regression intercept proxy).", + "kind": "extractor", + "extractor_name": "intercept", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "intercept" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:autocorr2", + "feature_id": "autocorr2", + "name": "FLOps extractor: autocorr2", + "description": "Lag-2 autocorrelation.", + "kind": "extractor", + "extractor_name": "autocorr2", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "autocorr2" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:autocorr3", + "feature_id": "autocorr3", + "name": "FLOps extractor: autocorr3", + "description": "Lag-3 autocorrelation.", + "kind": "extractor", + "extractor_name": "autocorr3", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "autocorr3" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:autocorr5", + "feature_id": "autocorr5", + "name": "FLOps extractor: autocorr5", + "description": "Lag-5 autocorrelation.", + "kind": "extractor", + "extractor_name": "autocorr5", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "autocorr5" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:autocorr10", + "feature_id": "autocorr10", + "name": "FLOps extractor: autocorr10", + "description": "Lag-10 autocorrelation (longer memory).", + "kind": "extractor", + "extractor_name": "autocorr10", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "autocorr10" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:mean_diff", + "feature_id": "mean_diff", + "name": "FLOps extractor: mean_diff", + "description": "Mean of first differences (average step).", + "kind": "extractor", + "extractor_name": "mean_diff", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "mean_diff" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:mean_2nd_diff", + "feature_id": "mean_2nd_diff", + "name": "FLOps extractor: mean_2nd_diff", + "description": "Mean of second differences (curvature).", + "kind": "extractor", + "extractor_name": "mean_2nd_diff", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "mean_2nd_diff" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:abs_2nd_diff_mean", + "feature_id": "abs_2nd_diff_mean", + "name": "FLOps extractor: abs_2nd_diff_mean", + "description": "Mean absolute second difference.", + "kind": "extractor", + "extractor_name": "abs_2nd_diff_mean", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "abs_2nd_diff_mean" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:std_diff", + "feature_id": "std_diff", + "name": "FLOps extractor: std_diff", + "description": "Std of first differences (volatility of change).", + "kind": "extractor", + "extractor_name": "std_diff", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "std_diff" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:num_zero_crossings", + "feature_id": "num_zero_crossings", + "name": "FLOps extractor: num_zero_crossings", + "description": "Count of sign changes around zero.", + "kind": "extractor", + "extractor_name": "num_zero_crossings", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "num_zero_crossings" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:num_mean_crossings", + "feature_id": "num_mean_crossings", + "name": "FLOps extractor: num_mean_crossings", + "description": "Count of crossings of the mean level.", + "kind": "extractor", + "extractor_name": "num_mean_crossings", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "num_mean_crossings" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:longest_above_mean", + "feature_id": "longest_above_mean", + "name": "FLOps extractor: longest_above_mean", + "description": "Longest run of points above the mean.", + "kind": "extractor", + "extractor_name": "longest_above_mean", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "longest_above_mean" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:longest_below_mean", + "feature_id": "longest_below_mean", + "name": "FLOps extractor: longest_below_mean", + "description": "Longest run of points below the mean.", + "kind": "extractor", + "extractor_name": "longest_below_mean", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "longest_below_mean" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:first_loc_max", + "feature_id": "first_loc_max", + "name": "FLOps extractor: first_loc_max", + "description": "Relative position (0-1) of the first maximum.", + "kind": "extractor", + "extractor_name": "first_loc_max", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "first_loc_max" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:last_loc_max", + "feature_id": "last_loc_max", + "name": "FLOps extractor: last_loc_max", + "description": "Relative position of the last maximum.", + "kind": "extractor", + "extractor_name": "last_loc_max", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "last_loc_max" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:first_loc_min", + "feature_id": "first_loc_min", + "name": "FLOps extractor: first_loc_min", + "description": "Relative position of the first minimum.", + "kind": "extractor", + "extractor_name": "first_loc_min", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "first_loc_min" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:mean_change", + "feature_id": "mean_change", + "name": "FLOps extractor: mean_change", + "description": "Average per-step change ((last-first)/n).", + "kind": "extractor", + "extractor_name": "mean_change", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "mean_change" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:cid_ce", + "feature_id": "cid_ce", + "name": "FLOps extractor: cid_ce", + "description": "Complexity-invariant distance estimate (curve complexity).", + "kind": "extractor", + "extractor_name": "cid_ce", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "cid_ce" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:c3_lag1", + "feature_id": "c3_lag1", + "name": "FLOps extractor: c3_lag1", + "description": "Non-linearity measure C3 at lag 1.", + "kind": "extractor", + "extractor_name": "c3_lag1", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "c3_lag1" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:c3_lag2", + "feature_id": "c3_lag2", + "name": "FLOps extractor: c3_lag2", + "description": "Non-linearity measure C3 at lag 2.", + "kind": "extractor", + "extractor_name": "c3_lag2", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "c3_lag2" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:time_reversal_asym", + "feature_id": "time_reversal_asym", + "name": "FLOps extractor: time_reversal_asym", + "description": "Time-reversal asymmetry statistic (lag 1).", + "kind": "extractor", + "extractor_name": "time_reversal_asym", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "time_reversal_asym" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:distinct_ratio", + "feature_id": "distinct_ratio", + "name": "FLOps extractor: distinct_ratio", + "description": "Ratio of distinct values to length.", + "kind": "extractor", + "extractor_name": "distinct_ratio", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "distinct_ratio" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:binned_entropy", + "feature_id": "binned_entropy", + "name": "FLOps extractor: binned_entropy", + "description": "Shannon entropy of a 10-bin value histogram.", + "kind": "extractor", + "extractor_name": "binned_entropy", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "binned_entropy" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:perm_entropy", + "feature_id": "perm_entropy", + "name": "FLOps extractor: perm_entropy", + "description": "Permutation entropy (ordinal complexity, order 3).", + "kind": "extractor", + "extractor_name": "perm_entropy", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "perm_entropy" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:spectral_entropy", + "feature_id": "spectral_entropy", + "name": "FLOps extractor: spectral_entropy", + "description": "Entropy of the normalized power spectrum.", + "kind": "extractor", + "extractor_name": "spectral_entropy", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "spectral_entropy" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:hjorth_mobility", + "feature_id": "hjorth_mobility", + "name": "FLOps extractor: hjorth_mobility", + "description": "Hjorth mobility (mean frequency).", + "kind": "extractor", + "extractor_name": "hjorth_mobility", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "hjorth_mobility" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:hjorth_complexity", + "feature_id": "hjorth_complexity", + "name": "FLOps extractor: hjorth_complexity", + "description": "Hjorth complexity (spectral bandwidth/shape change).", + "kind": "extractor", + "extractor_name": "hjorth_complexity", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "hjorth_complexity" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:spectral_rolloff", + "feature_id": "spectral_rolloff", + "name": "FLOps extractor: spectral_rolloff", + "description": "Frequency below which 85% of spectral energy lies.", + "kind": "extractor", + "extractor_name": "spectral_rolloff", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "spectral_rolloff" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:spectral_flatness", + "feature_id": "spectral_flatness", + "name": "FLOps extractor: spectral_flatness", + "description": "Geometric/arithmetic mean of spectrum (tonal vs noisy).", + "kind": "extractor", + "extractor_name": "spectral_flatness", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "spectral_flatness" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:dc_power", + "feature_id": "dc_power", + "name": "FLOps extractor: dc_power", + "description": "Magnitude of the DC component (|mean|).", + "kind": "extractor", + "extractor_name": "dc_power", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "dc_power" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:total_spectral_energy", + "feature_id": "total_spectral_energy", + "name": "FLOps extractor: total_spectral_energy", + "description": "Total power across the spectrum.", + "kind": "extractor", + "extractor_name": "total_spectral_energy", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "total_spectral_energy" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:band_low", + "feature_id": "band_low", + "name": "FLOps extractor: band_low", + "description": "Spectral-energy fraction in the low band (0-1/3).", + "kind": "extractor", + "extractor_name": "band_low", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "band_low" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:band_mid", + "feature_id": "band_mid", + "name": "FLOps extractor: band_mid", + "description": "Spectral-energy fraction in the mid band (1/3-2/3).", + "kind": "extractor", + "extractor_name": "band_mid", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "band_mid" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:band_high", + "feature_id": "band_high", + "name": "FLOps extractor: band_high", + "description": "Spectral-energy fraction in the high band (2/3-1).", + "kind": "extractor", + "extractor_name": "band_high", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "band_high" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:dominant_freq", + "feature_id": "dominant_freq", + "name": "FLOps extractor: dominant_freq", + "description": "Index of the dominant non-DC frequency.", + "kind": "extractor", + "extractor_name": "dominant_freq", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "dominant_freq" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:peak_to_peak", + "feature_id": "peak_to_peak", + "name": "FLOps extractor: peak_to_peak", + "description": "Max minus min amplitude.", + "kind": "extractor", + "extractor_name": "peak_to_peak", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "peak_to_peak" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:crest_factor", + "feature_id": "crest_factor", + "name": "FLOps extractor: crest_factor", + "description": "Peak/RMS — impulsiveness (vibration health).", + "kind": "extractor", + "extractor_name": "crest_factor", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "crest_factor" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:shape_factor", + "feature_id": "shape_factor", + "name": "FLOps extractor: shape_factor", + "description": "RMS/mean-absolute — waveform shape (vibration).", + "kind": "extractor", + "extractor_name": "shape_factor", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "shape_factor" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:impulse_factor", + "feature_id": "impulse_factor", + "name": "FLOps extractor: impulse_factor", + "description": "Peak/mean-absolute — impulsiveness (bearing faults).", + "kind": "extractor", + "extractor_name": "impulse_factor", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "impulse_factor" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:clearance_factor", + "feature_id": "clearance_factor", + "name": "FLOps extractor: clearance_factor", + "description": "Peak/(mean sqrt|x|)^2 — early bearing wear (vibration).", + "kind": "extractor", + "extractor_name": "clearance_factor", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "clearance_factor" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:margin_factor", + "feature_id": "margin_factor", + "name": "FLOps extractor: margin_factor", + "description": "Peak-to-peak / RMS.", + "kind": "extractor", + "extractor_name": "margin_factor", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "margin_factor" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:form_factor", + "feature_id": "form_factor", + "name": "FLOps extractor: form_factor", + "description": "RMS / |mean|.", + "kind": "extractor", + "extractor_name": "form_factor", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "form_factor" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:linear_trend_r", + "feature_id": "linear_trend_r", + "name": "FLOps extractor: linear_trend_r", + "description": "Pearson correlation of value vs time (trend strength).", + "kind": "extractor", + "extractor_name": "linear_trend_r", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "linear_trend_r" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:half_mean_diff", + "feature_id": "half_mean_diff", + "name": "FLOps extractor: half_mean_diff", + "description": "Mean of 2nd half minus 1st half (drift).", + "kind": "extractor", + "extractor_name": "half_mean_diff", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "half_mean_diff" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:half_std_ratio", + "feature_id": "half_std_ratio", + "name": "FLOps extractor: half_std_ratio", + "description": "Std of 2nd half / 1st half (variance shift).", + "kind": "extractor", + "extractor_name": "half_std_ratio", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "half_std_ratio" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:energy_ratio_first_half", + "feature_id": "energy_ratio_first_half", + "name": "FLOps extractor: energy_ratio_first_half", + "description": "Fraction of total energy in the first half.", + "kind": "extractor", + "extractor_name": "energy_ratio_first_half", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "energy_ratio_first_half" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:trend_strength", + "feature_id": "trend_strength", + "name": "FLOps extractor: trend_strength", + "description": "Normalized trend magnitude (|slope|*n/std).", + "kind": "extractor", + "extractor_name": "trend_strength", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "trend_strength" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:cumsum_argmax_ratio", + "feature_id": "cumsum_argmax_ratio", + "name": "FLOps extractor: cumsum_argmax_ratio", + "description": "Relative position of the cumulative-sum peak.", + "kind": "extractor", + "extractor_name": "cumsum_argmax_ratio", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "cumsum_argmax_ratio" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:cumsum_max", + "feature_id": "cumsum_max", + "name": "FLOps extractor: cumsum_max", + "description": "Maximum of the mean-centered cumulative sum.", + "kind": "extractor", + "extractor_name": "cumsum_max", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "cumsum_max" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:q33", + "feature_id": "q33", + "name": "FLOps extractor: q33", + "description": "33rd percentile.", + "kind": "extractor", + "extractor_name": "q33", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "q33" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:q66", + "feature_id": "q66", + "name": "FLOps extractor: q66", + "description": "66th percentile.", + "kind": "extractor", + "extractor_name": "q66", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "q66" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:range_to_std", + "feature_id": "range_to_std", + "name": "FLOps extractor: range_to_std", + "description": "Range divided by std.", + "kind": "extractor", + "extractor_name": "range_to_std", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "range_to_std" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:mean_square", + "feature_id": "mean_square", + "name": "FLOps extractor: mean_square", + "description": "Mean of squared values.", + "kind": "extractor", + "extractor_name": "mean_square", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "mean_square" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:abs_sum_changes", + "feature_id": "abs_sum_changes", + "name": "FLOps extractor: abs_sum_changes", + "description": "Sum of absolute first differences (total variation).", + "kind": "extractor", + "extractor_name": "abs_sum_changes", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "abs_sum_changes" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:max_diff", + "feature_id": "max_diff", + "name": "FLOps extractor: max_diff", + "description": "Largest single-step increase.", + "kind": "extractor", + "extractor_name": "max_diff", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "max_diff" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:min_diff", + "feature_id": "min_diff", + "name": "FLOps extractor: min_diff", + "description": "Largest single-step decrease.", + "kind": "extractor", + "extractor_name": "min_diff", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "min_diff" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:var_diff", + "feature_id": "var_diff", + "name": "FLOps extractor: var_diff", + "description": "Variance of first differences.", + "kind": "extractor", + "extractor_name": "var_diff", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "var_diff" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:positive_diff_ratio", + "feature_id": "positive_diff_ratio", + "name": "FLOps extractor: positive_diff_ratio", + "description": "Fraction of upward steps.", + "kind": "extractor", + "extractor_name": "positive_diff_ratio", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "positive_diff_ratio" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:autocorr4", + "feature_id": "autocorr4", + "name": "FLOps extractor: autocorr4", + "description": "Lag-4 autocorrelation.", + "kind": "extractor", + "extractor_name": "autocorr4", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "autocorr4" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:autocorr7", + "feature_id": "autocorr7", + "name": "FLOps extractor: autocorr7", + "description": "Lag-7 autocorrelation.", + "kind": "extractor", + "extractor_name": "autocorr7", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "autocorr7" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:num_peaks", + "feature_id": "num_peaks", + "name": "FLOps extractor: num_peaks", + "description": "Count of local maxima.", + "kind": "extractor", + "extractor_name": "num_peaks", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "num_peaks" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:num_valleys", + "feature_id": "num_valleys", + "name": "FLOps extractor: num_valleys", + "description": "Count of local minima.", + "kind": "extractor", + "extractor_name": "num_valleys", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "num_valleys" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:longest_above_2std", + "feature_id": "longest_above_2std", + "name": "FLOps extractor: longest_above_2std", + "description": "Longest run beyond 2 std (sustained anomaly).", + "kind": "extractor", + "extractor_name": "longest_above_2std", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "longest_above_2std" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:zero_crossing_rate", + "feature_id": "zero_crossing_rate", + "name": "FLOps extractor: zero_crossing_rate", + "description": "Mean-crossings per sample.", + "kind": "extractor", + "extractor_name": "zero_crossing_rate", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "zero_crossing_rate" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:skew_abs", + "feature_id": "skew_abs", + "name": "FLOps extractor: skew_abs", + "description": "Absolute skewness.", + "kind": "extractor", + "extractor_name": "skew_abs", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "skew_abs" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:p2p_to_std", + "feature_id": "p2p_to_std", + "name": "FLOps extractor: p2p_to_std", + "description": "Peak-to-peak / std.", + "kind": "extractor", + "extractor_name": "p2p_to_std", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "p2p_to_std" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:band_q1", + "feature_id": "band_q1", + "name": "FLOps extractor: band_q1", + "description": "Spectral-energy fraction, quartile band 0-25%.", + "kind": "extractor", + "extractor_name": "band_q1", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "band_q1" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:band_q2", + "feature_id": "band_q2", + "name": "FLOps extractor: band_q2", + "description": "Spectral-energy fraction, quartile band 25-50%.", + "kind": "extractor", + "extractor_name": "band_q2", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "band_q2" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:band_q3", + "feature_id": "band_q3", + "name": "FLOps extractor: band_q3", + "description": "Spectral-energy fraction, quartile band 50-75%.", + "kind": "extractor", + "extractor_name": "band_q3", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "band_q3" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:band_q4", + "feature_id": "band_q4", + "name": "FLOps extractor: band_q4", + "description": "Spectral-energy fraction, quartile band 75-100%.", + "kind": "extractor", + "extractor_name": "band_q4", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "band_q4" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:mean_psd", + "feature_id": "mean_psd", + "name": "FLOps extractor: mean_psd", + "description": "Mean power spectral density.", + "kind": "extractor", + "extractor_name": "mean_psd", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "mean_psd" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:peak_psd_ratio", + "feature_id": "peak_psd_ratio", + "name": "FLOps extractor: peak_psd_ratio", + "description": "Peak PSD / total PSD (spectral peakedness).", + "kind": "extractor", + "extractor_name": "peak_psd_ratio", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "peak_psd_ratio" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:diff_entropy", + "feature_id": "diff_entropy", + "name": "FLOps extractor: diff_entropy", + "description": "Binned entropy of first differences.", + "kind": "extractor", + "extractor_name": "diff_entropy", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "diff_entropy" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:quarter_mean_diff", + "feature_id": "quarter_mean_diff", + "name": "FLOps extractor: quarter_mean_diff", + "description": "Mean of last quarter minus first quarter (drift).", + "kind": "extractor", + "extractor_name": "quarter_mean_diff", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "quarter_mean_diff" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-22T03:27:01+00:00" + }, + { + "_id": "feature:longest_constant_run", + "feature_id": "longest_constant_run", + "name": "FLOps extractor: longest_constant_run", + "description": "Longest run of (near-)identical consecutive values (flatline length).", + "kind": "extractor", + "extractor_name": "longest_constant_run", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "pattern", + "cessation", + "longest_constant_run" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-23T00:00:00+00:00" + }, + { + "_id": "feature:flatline_fraction", + "feature_id": "flatline_fraction", + "name": "FLOps extractor: flatline_fraction", + "description": "Longest constant run as a fraction of length (cessation/quiet indicator).", + "kind": "extractor", + "extractor_name": "flatline_fraction", + "modality": "timeseries", + "interface": "extract", + "output_type": "scalar", + "provenance": "library", + "method": "FLOps", + "metrics": [], + "tags": [ + "flops", + "extractor", + "pattern", + "cessation", + "flatline_fraction" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-23T00:00:00+00:00" + } +] diff --git a/src/couchdb/scenarios_data/shared/tsfm/model_catalog.json b/src/couchdb/scenarios_data/shared/tsfm/model_catalog.json new file mode 100644 index 000000000..5757c0e3d --- /dev/null +++ b/src/couchdb/scenarios_data/shared/tsfm/model_catalog.json @@ -0,0 +1,1561 @@ +[ + { + "model_id": "ttm_96_28", + "model_checkpoint": "ttm_96_28", + "model_family": "TinyTimeMixer", + "provenance": "pretrained", + "base_model_id": null, + "task_ids": [ + "tsfm_forecasting", + "tsfm_forecasting_finetune", + "tsfm_forecasting_evaluation" + ], + "context_length": 96, + "prediction_length": 28, + "domain": "general", + "frequency": "any", + "source": "local_artifact", + "artifact_path": "artifacts/tsfm_models/ttm_96_28", + "hf_repo": "ibm-granite/granite-timeseries-ttm-r2", + "description": "Pretrained TinyTimeMixer forecasting model, context length 96, prediction horizon 28. General-purpose zero-shot multivariate forecaster; good for short-context, short-horizon tasks.", + "trained_on": [ + "pretraining-corpus" + ], + "metrics": [], + "tags": [ + "ttm", + "forecasting", + "zero-shot", + "short-context" + ], + "status": "active", + "version": "r2", + "created_by": "seed", + "created_at": "2026-06-20T00:00:00+00:00", + "_id": "model:ttm_96_28", + "sktime_class": "sktime.forecasting.ttm.TinyTimeMixerForecaster", + "params": { + "model_path": "ibm-granite/granite-timeseries-ttm-r2" + } + }, + { + "model_id": "ttm_512_96", + "model_checkpoint": "ttm_512_96", + "model_family": "TinyTimeMixer", + "provenance": "pretrained", + "base_model_id": null, + "task_ids": [ + "tsfm_forecasting", + "tsfm_forecasting_finetune", + "tsfm_forecasting_evaluation" + ], + "context_length": 512, + "prediction_length": 96, + "domain": "general", + "frequency": "any", + "source": "local_artifact", + "artifact_path": "artifacts/tsfm_models/ttm_512_96", + "hf_repo": "ibm-granite/granite-timeseries-ttm-r2", + "description": "Pretrained TinyTimeMixer forecasting model, context length 512, prediction horizon 96. General-purpose zero-shot multivariate forecaster; prefer when longer context improves accuracy and a longer horizon is needed.", + "trained_on": [ + "pretraining-corpus" + ], + "metrics": [], + "tags": [ + "ttm", + "forecasting", + "zero-shot", + "long-context" + ], + "status": "active", + "version": "r2", + "created_by": "seed", + "created_at": "2026-06-20T00:00:00+00:00", + "_id": "model:ttm_512_96", + "sktime_class": "sktime.forecasting.ttm.TinyTimeMixerForecaster", + "params": { + "model_path": "ibm-granite/granite-timeseries-ttm-r2" + } + }, + { + "model_id": "ttm_energy_96_28", + "model_checkpoint": "ttm_96_28", + "model_family": "TinyTimeMixer", + "provenance": "finetuned", + "base_model_id": "ttm_96_28", + "task_ids": [ + "tsfm_forecasting", + "tsfm_forecasting_evaluation" + ], + "context_length": 96, + "prediction_length": 28, + "domain": "energy", + "frequency": "any", + "source": "local_artifact", + "artifact_path": "artifacts/tsfm_models/ttm_96_28", + "hf_repo": null, + "description": "TinyTimeMixer fine-tuned on energy-domain data, context length 96, prediction horizon 28. Prefer over the general model for energy/HVAC assets.", + "trained_on": [ + "energy" + ], + "metrics": [], + "tags": [ + "ttm", + "forecasting", + "energy", + "finetuned", + "short-context" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-20T00:00:00+00:00", + "_id": "model:ttm_energy_96_28", + "sktime_class": "sktime.forecasting.ttm.TinyTimeMixerForecaster", + "params": { + "model_path": "artifacts/tsfm_models/ttm_96_28" + } + }, + { + "model_id": "ttm_energy_512_96", + "model_checkpoint": "ttm_512_96", + "model_family": "TinyTimeMixer", + "provenance": "finetuned", + "base_model_id": "ttm_512_96", + "task_ids": [ + "tsfm_forecasting", + "tsfm_forecasting_evaluation" + ], + "context_length": 512, + "prediction_length": 96, + "domain": "energy", + "frequency": "any", + "source": "local_artifact", + "artifact_path": "artifacts/tsfm_models/ttm_512_96", + "hf_repo": null, + "description": "TinyTimeMixer fine-tuned on energy-domain data, context length 512, prediction horizon 96. Prefer over the general long-context model for energy/HVAC assets needing longer horizons.", + "trained_on": [ + "energy" + ], + "metrics": [], + "tags": [ + "ttm", + "forecasting", + "energy", + "finetuned", + "long-context" + ], + "status": "active", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-20T00:00:00+00:00", + "_id": "model:ttm_energy_512_96", + "sktime_class": "sktime.forecasting.ttm.TinyTimeMixerForecaster", + "params": { + "model_path": "artifacts/tsfm_models/ttm_512_96" + } + }, + { + "model_id": "ttm_chiller6_512_96_ft", + "model_checkpoint": "artifacts/output/tuned_models/ttm_chiller6_512_96_ft", + "model_family": "TinyTimeMixer", + "provenance": "finetuned", + "base_model_id": "ttm_512_96", + "task_ids": [ + "tsfm_forecasting", + "tsfm_forecasting_evaluation" + ], + "context_length": 512, + "prediction_length": 96, + "domain": "hvac", + "frequency": "1min", + "source": "local_artifact", + "artifact_path": "artifacts/output/tuned_models/ttm_chiller6_512_96_ft", + "hf_repo": null, + "description": "Example agent-registered model: TinyTimeMixer fine-tuned on Chiller-6 condenser/efficiency channels. Demonstrates how a model produced by the finetune tool is written back into the catalog at runtime.", + "trained_on": [ + "chiller_6" + ], + "metrics": [ + { + "dataset": "chiller_6_holdout", + "metric": "mae", + "value": 0.041, + "forecast_horizon": 96 + }, + { + "dataset": "chiller_6_holdout", + "metric": "mse", + "value": 0.0036, + "forecast_horizon": 96 + } + ], + "tags": [ + "ttm", + "forecasting", + "hvac", + "chiller", + "finetuned" + ], + "status": "active", + "version": "1", + "created_by": "agent.tsfm.finetune", + "created_at": "2026-06-20T12:00:00+00:00", + "_id": "model:ttm_chiller6_512_96_ft", + "sktime_class": "sktime.forecasting.ttm.TinyTimeMixerForecaster", + "params": { + "model_path": "artifacts/output/tuned_models/ttm_chiller6_512_96_ft" + } + }, + { + "_id": "model:ibm-granite__granite-timeseries-ttm-r1", + "model_id": "ibm-granite__granite-timeseries-ttm-r1", + "sktime_class": "sktime.forecasting.ttm.TinyTimeMixerForecaster", + "params": { + "model_path": "ibm-granite/granite-timeseries-ttm-r1" + }, + "description": "granite-timeseries-ttm-r1: timeseries foundation model for forecasting context length 512, prediction horizon 96. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "tinytimemixer", + "hf_repo": "ibm-granite/granite-timeseries-ttm-r1", + "domain": "general", + "context_length": 512, + "prediction_length": 96, + "tags": [ + "tinytimemixer", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:ibm-granite__granite-timeseries-ttm-r2", + "model_id": "ibm-granite__granite-timeseries-ttm-r2", + "sktime_class": "sktime.forecasting.ttm.TinyTimeMixerForecaster", + "params": { + "model_path": "ibm-granite/granite-timeseries-ttm-r2" + }, + "description": "granite-timeseries-ttm-r2: timeseries foundation model for forecasting context length 512, prediction horizon 96. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "tinytimemixer", + "hf_repo": "ibm-granite/granite-timeseries-ttm-r2", + "domain": "general", + "context_length": 512, + "prediction_length": 96, + "tags": [ + "tinytimemixer", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:ibm-granite__granite-timeseries-tspulse-r1", + "model_id": "ibm-granite__granite-timeseries-tspulse-r1", + "sktime_class": "sktime.detection.tspulse.TSPulseAnomalyDetector", + "params": { + "model_path": "ibm-granite/granite-timeseries-tspulse-r1" + }, + "description": "granite-timeseries-tspulse-r1: timeseries foundation model for anomaly_detection, imputation, embedding, classification context length 512. Use for zero-shot multivariate anomaly detection on industrial sensor data.", + "task_ids": [ + "tsfm_anomaly_detection" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "tspulse", + "hf_repo": "ibm-granite/granite-timeseries-tspulse-r1", + "domain": "general", + "context_length": 512, + "prediction_length": null, + "tags": [ + "tspulse", + "timeseries", + "anomaly_detection", + "imputation", + "embedding", + "classification", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:ibm-granite__granite-timeseries-patchtst", + "model_id": "ibm-granite__granite-timeseries-patchtst", + "sktime_class": "sktime.forecasting.patch_tst.PatchTSTForecaster", + "params": { + "model_path": "ibm-granite/granite-timeseries-patchtst" + }, + "description": "granite-timeseries-patchtst: timeseries foundation model for forecasting context length 512, prediction horizon 96. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "patch_tsfm", + "hf_repo": "ibm-granite/granite-timeseries-patchtst", + "domain": "general", + "context_length": 512, + "prediction_length": 96, + "tags": [ + "patch_tsfm", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:ibm-granite__granite-timeseries-patchtsmixer", + "model_id": "ibm-granite__granite-timeseries-patchtsmixer", + "sktime_class": "sktime.forecasting.patch_tsmixer.PatchTSMixerForecaster", + "params": { + "model_path": "ibm-granite/granite-timeseries-patchtsmixer" + }, + "description": "granite-timeseries-patchtsmixer: timeseries foundation model for forecasting context length 512, prediction horizon 96. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "patch_tsfm", + "hf_repo": "ibm-granite/granite-timeseries-patchtsmixer", + "domain": "general", + "context_length": 512, + "prediction_length": 96, + "tags": [ + "patch_tsfm", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:amazon__chronos-t5-tiny", + "model_id": "amazon__chronos-t5-tiny", + "sktime_class": "sktime.forecasting.chronos.ChronosForecaster", + "params": { + "model_path": "amazon/chronos-t5-tiny" + }, + "description": "chronos-t5-tiny: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "chronos", + "hf_repo": "amazon/chronos-t5-tiny", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "chronos", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:amazon__chronos-t5-mini", + "model_id": "amazon__chronos-t5-mini", + "sktime_class": "sktime.forecasting.chronos.ChronosForecaster", + "params": { + "model_path": "amazon/chronos-t5-mini" + }, + "description": "chronos-t5-mini: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "chronos", + "hf_repo": "amazon/chronos-t5-mini", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "chronos", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:amazon__chronos-t5-small", + "model_id": "amazon__chronos-t5-small", + "sktime_class": "sktime.forecasting.chronos.ChronosForecaster", + "params": { + "model_path": "amazon/chronos-t5-small" + }, + "description": "chronos-t5-small: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "chronos", + "hf_repo": "amazon/chronos-t5-small", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "chronos", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:amazon__chronos-t5-base", + "model_id": "amazon__chronos-t5-base", + "sktime_class": "sktime.forecasting.chronos.ChronosForecaster", + "params": { + "model_path": "amazon/chronos-t5-base" + }, + "description": "chronos-t5-base: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "chronos", + "hf_repo": "amazon/chronos-t5-base", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "chronos", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:amazon__chronos-t5-large", + "model_id": "amazon__chronos-t5-large", + "sktime_class": "sktime.forecasting.chronos.ChronosForecaster", + "params": { + "model_path": "amazon/chronos-t5-large" + }, + "description": "chronos-t5-large: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "chronos", + "hf_repo": "amazon/chronos-t5-large", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "chronos", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:amazon__chronos-bolt-tiny", + "model_id": "amazon__chronos-bolt-tiny", + "sktime_class": "sktime.forecasting.chronos.ChronosForecaster", + "params": { + "model_path": "amazon/chronos-bolt-tiny" + }, + "description": "chronos-bolt-tiny: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "chronos", + "hf_repo": "amazon/chronos-bolt-tiny", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "chronos", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:amazon__chronos-bolt-mini", + "model_id": "amazon__chronos-bolt-mini", + "sktime_class": "sktime.forecasting.chronos.ChronosForecaster", + "params": { + "model_path": "amazon/chronos-bolt-mini" + }, + "description": "chronos-bolt-mini: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "chronos", + "hf_repo": "amazon/chronos-bolt-mini", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "chronos", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:amazon__chronos-bolt-small", + "model_id": "amazon__chronos-bolt-small", + "sktime_class": "sktime.forecasting.chronos.ChronosForecaster", + "params": { + "model_path": "amazon/chronos-bolt-small" + }, + "description": "chronos-bolt-small: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "chronos", + "hf_repo": "amazon/chronos-bolt-small", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "chronos", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:amazon__chronos-bolt-base", + "model_id": "amazon__chronos-bolt-base", + "sktime_class": "sktime.forecasting.chronos.ChronosForecaster", + "params": { + "model_path": "amazon/chronos-bolt-base" + }, + "description": "chronos-bolt-base: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "chronos", + "hf_repo": "amazon/chronos-bolt-base", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "chronos", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:google__timesfm-1.0-200m", + "model_id": "google__timesfm-1.0-200m", + "sktime_class": "sktime.forecasting.timesfm.TimesFMForecaster", + "params": { + "repo_id": "google/timesfm-1.0-200m" + }, + "description": "timesfm-1.0-200m: timeseries foundation model for forecasting context length 512, prediction horizon 128. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "timesfm", + "hf_repo": "google/timesfm-1.0-200m", + "domain": "general", + "context_length": 512, + "prediction_length": 128, + "tags": [ + "timesfm", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:google__timesfm-2.0-500m-pytorch", + "model_id": "google__timesfm-2.0-500m-pytorch", + "sktime_class": "sktime.forecasting.timesfm.TimesFMForecaster", + "params": { + "repo_id": "google/timesfm-2.0-500m-pytorch" + }, + "description": "timesfm-2.0-500m-pytorch: timeseries foundation model for forecasting context length 2048, prediction horizon 128. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "timesfm", + "hf_repo": "google/timesfm-2.0-500m-pytorch", + "domain": "general", + "context_length": 2048, + "prediction_length": 128, + "tags": [ + "timesfm", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:Salesforce__moirai-1.0-R-small", + "model_id": "Salesforce__moirai-1.0-R-small", + "sktime_class": "sktime.forecasting.moirai.MOIRAIForecaster", + "params": { + "checkpoint_path": "Salesforce/moirai-1.0-R-small" + }, + "description": "moirai-1.0-R-small: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "moirai", + "hf_repo": "Salesforce/moirai-1.0-R-small", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "moirai", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:Salesforce__moirai-1.0-R-base", + "model_id": "Salesforce__moirai-1.0-R-base", + "sktime_class": "sktime.forecasting.moirai.MOIRAIForecaster", + "params": { + "checkpoint_path": "Salesforce/moirai-1.0-R-base" + }, + "description": "moirai-1.0-R-base: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "moirai", + "hf_repo": "Salesforce/moirai-1.0-R-base", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "moirai", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:Salesforce__moirai-1.0-R-large", + "model_id": "Salesforce__moirai-1.0-R-large", + "sktime_class": "sktime.forecasting.moirai.MOIRAIForecaster", + "params": { + "checkpoint_path": "Salesforce/moirai-1.0-R-large" + }, + "description": "moirai-1.0-R-large: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "moirai", + "hf_repo": "Salesforce/moirai-1.0-R-large", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "moirai", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:Salesforce__moirai-1.1-R-small", + "model_id": "Salesforce__moirai-1.1-R-small", + "sktime_class": "sktime.forecasting.moirai.MOIRAIForecaster", + "params": { + "checkpoint_path": "Salesforce/moirai-1.1-R-small" + }, + "description": "moirai-1.1-R-small: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "moirai", + "hf_repo": "Salesforce/moirai-1.1-R-small", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "moirai", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:Salesforce__moirai-1.1-R-base", + "model_id": "Salesforce__moirai-1.1-R-base", + "sktime_class": "sktime.forecasting.moirai.MOIRAIForecaster", + "params": { + "checkpoint_path": "Salesforce/moirai-1.1-R-base" + }, + "description": "moirai-1.1-R-base: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "moirai", + "hf_repo": "Salesforce/moirai-1.1-R-base", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "moirai", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:Salesforce__moirai-1.1-R-large", + "model_id": "Salesforce__moirai-1.1-R-large", + "sktime_class": "sktime.forecasting.moirai.MOIRAIForecaster", + "params": { + "checkpoint_path": "Salesforce/moirai-1.1-R-large" + }, + "description": "moirai-1.1-R-large: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "moirai", + "hf_repo": "Salesforce/moirai-1.1-R-large", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "moirai", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:Salesforce__moirai-moe-1.0-R-small", + "model_id": "Salesforce__moirai-moe-1.0-R-small", + "sktime_class": "sktime.forecasting.moirai.MOIRAIForecaster", + "params": { + "checkpoint_path": "Salesforce/moirai-moe-1.0-R-small" + }, + "description": "moirai-moe-1.0-R-small: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "moirai", + "hf_repo": "Salesforce/moirai-moe-1.0-R-small", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "moirai", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:Salesforce__moirai-moe-1.0-R-base", + "model_id": "Salesforce__moirai-moe-1.0-R-base", + "sktime_class": "sktime.forecasting.moirai.MOIRAIForecaster", + "params": { + "checkpoint_path": "Salesforce/moirai-moe-1.0-R-base" + }, + "description": "moirai-moe-1.0-R-base: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "moirai", + "hf_repo": "Salesforce/moirai-moe-1.0-R-base", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "moirai", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:AutonLab__MOMENT-1-large", + "model_id": "AutonLab__MOMENT-1-large", + "sktime_class": "sktime.forecasting.momentfm.MomentFMForecaster", + "params": { + "pretrained_model_name_or_path": "AutonLab/MOMENT-1-large" + }, + "description": "MOMENT-1-large: timeseries foundation model for forecasting, anomaly_detection, imputation, classification context length 512. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "moment", + "hf_repo": "AutonLab/MOMENT-1-large", + "domain": "general", + "context_length": 512, + "prediction_length": null, + "tags": [ + "moment", + "timeseries", + "forecasting", + "anomaly_detection", + "imputation", + "classification", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:AutonLab__MOMENT-1-base", + "model_id": "AutonLab__MOMENT-1-base", + "sktime_class": "sktime.forecasting.momentfm.MomentFMForecaster", + "params": { + "pretrained_model_name_or_path": "AutonLab/MOMENT-1-base" + }, + "description": "MOMENT-1-base: timeseries foundation model for forecasting, anomaly_detection, imputation, classification context length 512. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "moment", + "hf_repo": "AutonLab/MOMENT-1-base", + "domain": "general", + "context_length": 512, + "prediction_length": null, + "tags": [ + "moment", + "timeseries", + "forecasting", + "anomaly_detection", + "imputation", + "classification", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:AutonLab__MOMENT-1-small", + "model_id": "AutonLab__MOMENT-1-small", + "sktime_class": "sktime.forecasting.momentfm.MomentFMForecaster", + "params": { + "pretrained_model_name_or_path": "AutonLab/MOMENT-1-small" + }, + "description": "MOMENT-1-small: timeseries foundation model for forecasting, anomaly_detection, imputation, classification context length 512. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "moment", + "hf_repo": "AutonLab/MOMENT-1-small", + "domain": "general", + "context_length": 512, + "prediction_length": null, + "tags": [ + "moment", + "timeseries", + "forecasting", + "anomaly_detection", + "imputation", + "classification", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:Maple728__TimeMoE-50M", + "model_id": "Maple728__TimeMoE-50M", + "sktime_class": "sktime.forecasting.timemoe.TimeMoEForecaster", + "params": { + "model_path": "Maple728/TimeMoE-50M" + }, + "description": "TimeMoE-50M: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "timemoe", + "hf_repo": "Maple728/TimeMoE-50M", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "timemoe", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:Maple728__TimeMoE-200M", + "model_id": "Maple728__TimeMoE-200M", + "sktime_class": "sktime.forecasting.timemoe.TimeMoEForecaster", + "params": { + "model_path": "Maple728/TimeMoE-200M" + }, + "description": "TimeMoE-200M: timeseries foundation model for forecasting. Use for zero-shot multivariate forecasting on industrial sensor data.", + "task_ids": [ + "tsfm_forecasting" + ], + "modality": "timeseries", + "provenance": "pretrained", + "training_regime": "zero_shot", + "model_family": "timemoe", + "hf_repo": "Maple728/TimeMoE-200M", + "domain": "general", + "context_length": null, + "prediction_length": null, + "tags": [ + "timemoe", + "timeseries", + "forecasting", + "migrated", + "foundation" + ], + "status": "active", + "version": "1", + "created_by": "migrated_curated", + "source": "granite-tsfm-curated-catalog", + "framework": "sktime" + }, + { + "_id": "model:akits_deepad", + "model_id": "akits_deepad", + "model_checkpoint": "anomalykits://pipeline/DeepAD", + "model_family": "anomalykits", + "framework": "anomalykits", + "pipeline_type": "DeepAD", + "provenance": "toolkit", + "base_model_id": null, + "modality": "timeseries", + "task_ids": [ + "tsfm_anomaly_detection" + ], + "usage_modes": [ + "unsupervised", + "semi_supervised", + "train_test" + ], + "output_type": "anomaly_score_label_contribution", + "context_length": null, + "prediction_length": null, + "domain": "general", + "frequency": "any", + "source": "toolkit", + "artifact_path": null, + "hf_repo": null, + "execution_engines": [ + "serverless", + "spark", + "gpu" + ], + "thresholding": [ + "static", + "dynamic" + ], + "selection_metrics": [ + "EM", + "AL", + "recall" + ], + "description": "AnomalyKiTS DeepAD: ensemble of time-series forecasting models for anomaly detection; emits score + label(+1/-1) + per-variable contribution.", + "metrics": [], + "tags": [ + "anomalykits", + "anomaly_detection", + "forecasting-ensemble", + "deepad" + ], + "status": "deprecated", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-21T00:00:00+00:00" + }, + { + "_id": "model:akits_relationshipad", + "model_id": "akits_relationshipad", + "model_checkpoint": "anomalykits://pipeline/RelationshipAD", + "model_family": "anomalykits", + "framework": "anomalykits", + "pipeline_type": "RelationshipAD", + "provenance": "toolkit", + "base_model_id": null, + "modality": "timeseries", + "task_ids": [ + "tsfm_anomaly_detection" + ], + "usage_modes": [ + "unsupervised", + "semi_supervised" + ], + "output_type": "anomaly_score_label_contribution", + "context_length": null, + "prediction_length": null, + "domain": "general", + "frequency": "any", + "source": "toolkit", + "artifact_path": null, + "hf_repo": null, + "execution_engines": [ + "serverless", + "spark", + "gpu" + ], + "thresholding": [ + "static", + "dynamic" + ], + "selection_metrics": [ + "EM", + "AL", + "recall" + ], + "description": "AnomalyKiTS RelationshipAD: pairwise inter-variable relationship modeling (e.g. DAGMM/graphical) for multivariate anomaly; contribution per variable pair.", + "metrics": [], + "tags": [ + "anomalykits", + "anomaly_detection", + "multivariate", + "relationship" + ], + "status": "deprecated", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-21T00:00:00+00:00" + }, + { + "_id": "model:akits_reconstructad", + "model_id": "akits_reconstructad", + "model_checkpoint": "anomalykits://pipeline/ReconstructAD", + "model_family": "anomalykits", + "framework": "anomalykits", + "pipeline_type": "ReconstructAD", + "provenance": "toolkit", + "base_model_id": null, + "modality": "timeseries", + "task_ids": [ + "tsfm_anomaly_detection" + ], + "usage_modes": [ + "unsupervised", + "semi_supervised" + ], + "output_type": "anomaly_score_label_contribution", + "context_length": null, + "prediction_length": null, + "domain": "general", + "frequency": "any", + "source": "toolkit", + "artifact_path": null, + "hf_repo": null, + "execution_engines": [ + "serverless", + "gpu" + ], + "thresholding": [ + "static", + "dynamic" + ], + "selection_metrics": [ + "EM", + "AL", + "recall" + ], + "description": "AnomalyKiTS ReconstructAD: reconstruction-error anomaly (autoencoder / GAN style); contribution = per-channel reconstruction error.", + "metrics": [], + "tags": [ + "anomalykits", + "anomaly_detection", + "reconstruction", + "autoencoder" + ], + "status": "deprecated", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-21T00:00:00+00:00" + }, + { + "_id": "model:akits_windowad", + "model_id": "akits_windowad", + "model_checkpoint": "anomalykits://pipeline/WindowAD", + "model_family": "anomalykits", + "framework": "anomalykits", + "pipeline_type": "WindowAD", + "provenance": "toolkit", + "base_model_id": null, + "modality": "timeseries", + "task_ids": [ + "tsfm_anomaly_detection" + ], + "usage_modes": [ + "unsupervised", + "semi_supervised", + "batch" + ], + "output_type": "anomaly_score_label_contribution", + "context_length": null, + "prediction_length": null, + "domain": "general", + "frequency": "any", + "source": "toolkit", + "artifact_path": null, + "hf_repo": null, + "execution_engines": [ + "serverless", + "spark" + ], + "thresholding": [ + "static", + "dynamic" + ], + "selection_metrics": [ + "EM", + "AL", + "recall" + ], + "description": "AnomalyKiTS WindowAD: sliding-window outlier detection over engineered features; light, CPU-friendly; static otsu thresholding default.", + "metrics": [], + "tags": [ + "anomalykits", + "anomaly_detection", + "window", + "outlier" + ], + "status": "deprecated", + "version": "1", + "created_by": "seed", + "created_at": "2026-06-21T00:00:00+00:00" + }, + { + "_id": "model:naive_persistence", + "model_id": "naive_persistence", + "scitype": "forecaster", + "sktime_class": "sktime.forecasting.naive.NaiveForecaster", + "params": { + "strategy": "last" + }, + "task_ids": [ + "tsfm_forecasting" + ], + "family": "baseline", + "provenance": "toolkit", + "description": "Persistence / zero-model baseline (repeat last).", + "status": "active" + }, + { + "_id": "model:autoarima", + "model_id": "autoarima", + "scitype": "forecaster", + "sktime_class": "sktime.forecasting.arima.AutoARIMA", + "params": { + "suppress_warnings": true + }, + "task_ids": [ + "tsfm_forecasting" + ], + "family": "statistical", + "provenance": "toolkit", + "description": "Classical statistical forecaster.", + "status": "active" + }, + { + "_id": "model:ttm", + "model_id": "ttm", + "scitype": "forecaster", + "sktime_class": "sktime.forecasting.ttm.TinyTimeMixerForecaster", + "params": {}, + "task_ids": [ + "tsfm_forecasting" + ], + "family": "tsfm", + "provenance": "pretrained", + "soft_deps": [ + "torch", + "transformers", + "tsfm_public" + ], + "description": "IBM Granite TinyTimeMixer foundation forecaster (zero-shot).", + "status": "active" + }, + { + "_id": "model:chronos", + "model_id": "chronos", + "scitype": "forecaster", + "sktime_class": "sktime.forecasting.chronos.ChronosForecaster", + "params": { + "model_path": "amazon/chronos-t5-small" + }, + "task_ids": [ + "tsfm_forecasting" + ], + "family": "tsfm", + "provenance": "pretrained", + "soft_deps": [ + "torch", + "transformers" + ], + "param_hints": { + "model_path": "HF repo id, e.g. amazon/chronos-t5-{tiny,small,base}" + }, + "description": "Amazon Chronos foundation forecaster.", + "status": "active" + }, + { + "_id": "model:conformal_wrapper", + "model_id": "conformal_wrapper", + "scitype": "forecaster", + "sktime_class": "sktime.forecasting.conformal.ConformalIntervals", + "params": { + "forecaster": { + "_target_": "sktime.forecasting.naive.NaiveForecaster", + "params": { + "strategy": "last" + } + } + }, + "task_ids": [ + "tsfm_forecasting" + ], + "family": "probabilistic", + "provenance": "toolkit", + "wraps": "any forecaster", + "outputs": [ + "point", + "prediction_interval", + "quantiles" + ], + "description": "Conformal prediction intervals around a base forecaster.", + "status": "active" + }, + { + "_id": "model:pyod_iforest", + "model_id": "pyod_iforest", + "scitype": "detector", + "sktime_class": "sktime.detection.adapters._pyod.PyODDetector", + "params": { + "estimator": { + "_target_": "pyod.models.iforest.IForest", + "params": { + "n_estimators": 100, + "contamination": 0.1, + "random_state": 42 + } + } + }, + "task_ids": [ + "tsfm_anomaly_detection" + ], + "family": "sklearn_ad", + "provenance": "toolkit", + "training_regime": "fit_on_series", + "soft_deps": [ + "pyod" + ], + "param_hints": { + "estimator.params.n_estimators": "number of isolation trees; higher = more stable, slower (default 100)", + "estimator.params.contamination": "expected outlier fraction in (0, 0.5]; sets the decision threshold (default 0.1)", + "estimator.params.random_state": "seed for reproducible splits" + }, + "description": "PyOD IsolationForest anomaly detection on a univariate time series via the sktime PyOD adapter. Fits on the series' own history (fit_on_series). Requires `pip install pyod`.", + "status": "active" + }, + { + "_id": "model:sublof", + "model_id": "sublof", + "scitype": "detector", + "sktime_class": "sktime.detection.lof.SubLOF", + "params": { + "n_neighbors": 20, + "window_size": 50, + "novelty": true + }, + "task_ids": [ + "tsfm_anomaly_detection" + ], + "family": "sklearn_ad", + "provenance": "toolkit", + "description": "Subsequence Local Outlier Factor (sklearn LOF over windows).", + "status": "active" + }, + { + "_id": "model:tspulse_ad", + "model_id": "tspulse_ad", + "scitype": "detector", + "sktime_class": "sktime.detection.tspulse.TSPulseAnomalyDetector", + "params": {}, + "task_ids": [ + "tsfm_anomaly_detection" + ], + "family": "tsfm", + "provenance": "pretrained", + "soft_deps": [ + "torch", + "granite-tsfm" + ], + "description": "IBM Granite TSPulse zero-shot anomaly detector.", + "status": "active" + }, + { + "_id": "model:tspulse_clf", + "model_id": "tspulse_clf", + "scitype": "classifier", + "sktime_class": "sktime.classification.foundation_models.tspulse.TSPulseClassifier", + "params": {}, + "task_ids": [ + "tsfm_classification" + ], + "family": "tsfm", + "provenance": "pretrained", + "soft_deps": [ + "torch", + "granite-tsfm" + ], + "description": "IBM Granite TSPulse classifier (representation-based).", + "status": "active" + }, + { + "_id": "model:tskmeans", + "model_id": "tskmeans", + "scitype": "clusterer", + "sktime_class": "sktime.clustering.k_means.TimeSeriesKMeans", + "params": { + "n_clusters": 3 + }, + "task_ids": [ + "tsfm_clustering" + ], + "family": "classical", + "provenance": "toolkit", + "description": "Time-series k-means (DTW-capable) clustering.", + "status": "active" + } +] \ No newline at end of file diff --git a/src/servers/tsfm/README.md b/src/servers/tsfm/README.md new file mode 100644 index 000000000..20af649cb --- /dev/null +++ b/src/servers/tsfm/README.md @@ -0,0 +1,103 @@ +# TSFM MCP Server + +A time-series-AI server for AssetOpsBench, built on **sktime** as the execution substrate. The +agent **discovers components, reads evidence, composes a recipe, and runs it** — forecasting, +anomaly detection, regression/classification/clustering, imputation, and evaluation — over the +standard AssetOpsBench MCP contract. Models and features are **catalog data (cards), not tools** +(the HuggingGPT principle); bulk data crosses the boundary as **file pointers**. + +Substrate is sktime end-to-end (TTM/Chronos/MOIRAI/Moment/TimesFM via sktime's foundation +adapters; TSPulse / SubLOF / PyOD for anomaly; classical forecasters and tabular estimators). +There is **no legacy / `tsfm_public`-specific path** — foundation models resolve through sktime +and only need `tsfm_public`+`torch` at run time. + +## Quick start + +```bash +# tests (in-memory store, no CouchDB / torch needed) +TSFM_STORE=memory python -m pytest src/servers/tsfm --import-mode=importlib -q + +# production: load the catalogs into CouchDB (standard AssetOpsBench loader), then serve +tsfm-mcp-server # entry point → tsfm.main:main, FastMCP over stdio, reads CouchDB +``` + +## House contract + +Entry point `tsfm-mcp-server` → `tsfm.main:main`, a module-level `mcp = FastMCP("tsfm")` served +over stdio, matching the sibling AssetOpsBench servers. Every tool: + +- is decorated `@mcp.tool(...)` and returns a **typed Pydantic result** (`Union[XResult, ErrorResult]`); +- **validates inputs** → returns `ErrorResult(error=…)` rather than throwing across the wire; +- takes bulk data as a **file pointer** (`dataset_path` in, `results_file`/`evidence_file` out). + +## Layered package + +``` +core/ store (MemoryStore|CouchStore), schemas (ModelCard/FeatureCard), tasks, glossary, results_models +substrate/ resolver — sktime as the execution substrate (training_regime, is_foundation, resolve) +stores/ model_store, feature_store (lifecycle + lineage + search), results +reasoning/ profile (evidence), param_space, feature_selection (FLOps, 109 extractors), patterns (pattern evidence), dataquality +engine/ composition (run_recipe forecast+anomaly, run_tabular_recipe, discover_components), plan (recipe DAG), feature_runner (EFE), evolve (AlphaEvolve) +eval/ gifteval — GIFT-Eval scoring + leaderboard +io/ refs (file pointers: load/write series, materialize IoT), window +main.py the MCP tool surface · config.py env knobs · bootstrap.py seed loader +``` + +## The surface (36 tools) + +- **Discover / evidence:** `list_tasks`, `discover_components`, `describe_candidates`, + `find_models`, `find_features`, `get_component`, `profile_series`, `select_features` (FLOps), + `characterize_series` (pattern evidence), `data_quality`. +- **Compose & run:** `run_recipe` (forecasting **and** anomaly — anomaly via + `recipe.task=tsfm_anomaly_detection`, `method=detector|conformal`), `run_tabular_recipe`, + `run_plan` (recipe DAG), `evaluate` (GIFT-Eval). +- **Catalog write-back / lifecycle:** `register_model`/`register_feature`, `register_finetuned`, + `list_models`/`search_models`, `update_/deprecate_/new_*_version`, lineage, `list_extractors`. +- **Results:** `get_result`, `list_results`, `get_run`, `list_runs`. +- **Evolve (AlphaEvolve):** `evolve_ask`, `evolve_tell`, `evolve_best`. + +See `docs/TOOLS.md` for the full as-built table. + +## Data & catalog model + +- **Bulk data is a file pointer.** IoT/vibration data lives in CouchDB; the IoT server + materialises it to a local CSV; TSFM takes `dataset_path` and returns a `results_file`. No bulk + arrays cross the MCP boundary. +- **Catalog = two CouchDB collections** — `model_catalog` (48 cards) and `feature_catalog` + (6 transforms + 109 FLOps extractors). They load like every other AssetOpsBench collection from + `src/couchdb/scenarios_data/shared/tsfm/{model,feature}_catalog.json`; `bootstrap.load_seeds` + reads from there (override with `$TSFM_SEEDS_DIR`). See `docs/COUCHDB_LOADING.md`. +- **Stores:** `CouchStore` is the default/production backend; `MemoryStore` is a **test double** + only (`TSFM_STORE=memory`). `make_store()` picks by env. + +## Workflows + +- **Forecasting** — `run_recipe(dataset_path, …, recipe={estimator:{model_id:"ttm_96_28"}, fh:[…]})` + → transforms + single/ensemble + optional conformal → `results_file`. Zero-shot is the default; + fine-tune is opt-in. See `docs/FORECASTING_WORKFLOW.md`. +- **Anomaly detection** — same tool, `recipe.task=tsfm_anomaly_detection`: `method=detector` + (TSPulse zero-shot, SubLOF, PyOD) or `method=conformal` (forecast + sktime `ConformalIntervals` + → flag out-of-band points). +- **Pattern evidence** — `characterize_series` describes a signal's *shape* (per channel-group + state + rate over changepoint phases, plus cross-channel relation) as structured, **fault-free** + evidence for an LLM to reason over. Generic (any signals; grouping is opt-in). See + `docs/PATTERN_EVIDENCE.md`. + +The principle throughout: the **server supplies evidence; the agent decides** the recipe, the +parameters, and the diagnosis. Results hand off downstream (FMSR/WO/Spot) — no alerts here. + +## Config + +`TSFM_STORE=couch|memory` (default `couch`) · `TSFM_SEEDS_DIR` (override catalog seed location) · +`COUCHDB_URL` / credentials (couch backend) · `LOG_LEVEL`. + +Apple-Silicon note: TSPulse/torch may hit a Metal (MPS) `gatherND` assertion at predict time; run +foundation models on CPU (e.g. disable MPS) until the upstream PyTorch fix lands. + +## Status + +sktime-native surface (36 tools), conformal AD, FLOps/EFE feature layers, AlphaEvolve loop, and +the pattern-evidence engine are **done and tested** — `121 passed / 5 skipped` (TSPulse skips +cleanly without Python ≥3.11 + `granite-tsfm`). Foundation-model inference (TTM/TSPulse/…) is +env-gated on `tsfm_public`+`torch`. Docs live in `docs/` (STRUCTURE, TOOLS, FORECASTING_WORKFLOW, +PATTERN_EVIDENCE, COUCHDB_LOADING, GLOSSARY, …). diff --git a/src/servers/tsfm/__init__.py b/src/servers/tsfm/__init__.py index e69de29bb..13c84dbc9 100644 --- a/src/servers/tsfm/__init__.py +++ b/src/servers/tsfm/__init__.py @@ -0,0 +1,22 @@ +"""TSFM MCP server — time-series AI tasks on the sktime substrate. + +Layered package: + core/ store, schemas, tasks — the data model + task contracts + substrate/ resolver — sktime as the execution substrate + stores/ model_store, feature_store, — catalog = pointer index (models/features as data) + results + reasoning/ profile, param_space, — evidence the agent reasons from + feature_selection (FLOps) + engine/ composition, plan, — the recipe engine (the pipeline is composed) + feature_runner, evolve + eval/ gifteval — GIFT-Eval scoring + leaderboard + io/ refs (file pointers), window — data I/O + main.py the MCP tool surface — config.py the env knobs + +Design: models & features are DATA (catalog cards), not tools; the agent composes recipes and +reasons every parameter; the server provides evidence. See docs/STRUCTURE.md. +""" + +__all__ = ["core", "substrate", "stores", "reasoning", "engine", "eval", "io", + "bootstrap", "config"] +__version__ = "0.2.0" diff --git a/src/servers/tsfm/anomaly.py b/src/servers/tsfm/anomaly.py deleted file mode 100644 index c9eb7cadc..000000000 --- a/src/servers/tsfm/anomaly.py +++ /dev/null @@ -1,585 +0,0 @@ -"""Conformal anomaly detection for time-series forecasting outputs.""" - -from __future__ import annotations - -import json -import os -import pickle - -import numpy as np -import pandas as pd - -from .io import _read_ts_data - - -# ── Conformal anomaly detection ─────────────────────────────────────────────── - -_NONCONFORMITY_SCORES = ["absolute_error"] - - -def _absolute_error(y, y_pred): - assert y.shape == y_pred.shape, ( - f"y and y_pred shapes do not match: {y.shape} vs {y_pred.shape}" - ) - error = np.abs(y - y_pred) - if len(error.shape) > 1: - error = np.mean(error, axis=-1) - return error - - -def _nonconformity_score_functions( - y_pred, y_gt, X=None, nonconformity_score="absolute_error" -): - assert nonconformity_score in _NONCONFORMITY_SCORES - if nonconformity_score == "absolute_error": - return _absolute_error(y_gt, y_pred) - - -def _conformal_set(y_pred, score_threshold, nonconformity_score="absolute_error"): - if nonconformity_score == "absolute_error": - return {"y_low": y_pred - score_threshold, "y_high": y_pred + score_threshold} - - -def _weighted_conformal_quantile( - scores, weights, alpha=0.05, conformal_correction=False, max_score=np.inf -): - if weights is None: - weights = np.ones_like(scores) - assert np.max(weights) <= 1 - assert np.min(weights) >= 0 - assert weights.shape[0] == scores.shape[0] - if conformal_correction: - weights = np.append(weights, np.array([1])) - scores = np.append(scores, np.array([max_score])) - weights = np.array(weights) / np.sum(weights) - sorted_indices = np.argsort(scores) - sorted_scores = scores[sorted_indices] - sorted_weights = weights[sorted_indices] - cumulative_weights = np.cumsum(sorted_weights) - quantile_index = np.searchsorted(cumulative_weights, 1 - alpha) - return sorted_scores[quantile_index] - - -def _weighted_conformal_alpha( - scores, weights, score_observed, conformal_correction=False, max_score=np.inf -): - if weights is None: - weights = np.ones_like(score_observed) - if conformal_correction: - weights = np.append(weights, np.array([1])) - scores = np.append(scores, np.array([max_score])) - weights = np.array(weights) / np.sum(weights) - sorted_indices = np.argsort(scores) - sorted_scores = scores[sorted_indices] - sorted_weights = weights[sorted_indices] - return np.sum(sorted_weights[sorted_scores > score_observed]) - - -class _TSADWeightedConformalWrapper: - def __init__( - self, - nonconformity_score="absolute_error", - false_alarm=0.05, - weighting="uniform", - weighting_params=None, - threshold_function="weighting", - window_size=None, - online_adaptive=False, - ): - if weighting_params is None: - weighting_params = {} - self.nonconformity_score = nonconformity_score - assert self.nonconformity_score in _NONCONFORMITY_SCORES - self.nonconformity_score_func = _nonconformity_score_functions - self.quantile = 1 - false_alarm - self.false_alarm = false_alarm - self.weighting = weighting - self.weighting_params = weighting_params - self.window_size = window_size - self.online_size = 1 - self.online = online_adaptive - self.threshold_function = threshold_function - self.cal_scores: list = [] - self.weights: list = [] - self.cal_X: list = [] - self.cal_timestamps: list = [] - self.score_threshold = None - - def fit(self, y_cal_pred, y_cal_gt, X_cal=None, cal_timestamps=None): - if self.window_size is None: - self.window_size = y_cal_pred.shape[0] - self.cal_scores = _nonconformity_score_functions( - y_cal_pred, y_cal_gt, X=X_cal, nonconformity_score=self.nonconformity_score - ) - self.cal_scores = self.cal_scores[-self.window_size :] - if X_cal is not None: - self.cal_X = X_cal[-self.window_size :] - if cal_timestamps is not None: - self.cal_timestamps = cal_timestamps[-self.window_size :] - if self.weighting in ("uniform", "exponential_decay"): - cal_weights = self.get_weights() - self.weights.append(cal_weights) - if self.threshold_function == "weighting": - self.score_threshold = self._score_threshold_func( - cal_weights, false_alarm=self.false_alarm - ) - critical_efficient_size = np.ceil(1 / self.false_alarm) - assert np.sum(cal_weights) >= critical_efficient_size, ( - f" The effective size is too small for the desired false alarm of {self.false_alarm}, " - f"the calibration set should be larger than {critical_efficient_size}" - ) - - def get_weights(self, y_pred=None, X=None, timestamps=None, false_alarm=None): - if false_alarm is None: - false_alarm = self.false_alarm - if self.weighting in ("uniform", "exponential_decay"): - if self.weights: - return self.weights[-1] - if self.weighting == "uniform": - return np.ones(self.window_size) - if self.weighting == "exponential_decay": - decay_param = self.weighting_params.get("decay_param", 0.99) - return decay_param ** (self.window_size - np.arange(self.window_size)) - - def _score_threshold_func( - self, - cal_weights, - cal_scores=None, - y_pred=None, - X=None, - timestamps=None, - false_alarm=None, - ): - if cal_scores is None: - cal_scores = self.cal_scores - if false_alarm is None: - false_alarm = self.false_alarm - score_threshold = [] - if self.threshold_function == "weighting": - if len(cal_weights.shape) == 1: - score_threshold = _weighted_conformal_quantile( - np.append(cal_scores, np.array([np.infty]), axis=0), - np.append(cal_weights, np.array([1]), axis=0), - alpha=false_alarm, - ) - else: - for i in range(cal_weights.shape[0]): - st_i = _weighted_conformal_quantile( - np.append(cal_scores, np.array([np.infty]), axis=0), - np.append(cal_weights[i, :], np.array([1]), axis=0), - alpha=false_alarm, - ) - score_threshold.append(st_i) - score_threshold = np.array(score_threshold) - return score_threshold - - def predict_batch( - self, y_pred, y_gt=None, X=None, timestamps=None, false_alarm=None, update=None - ): - if false_alarm is None: - false_alarm = self.false_alarm - if update is None: - update = self.online - cal_weights = self.get_weights( - y_pred, X=X, timestamps=timestamps, false_alarm=false_alarm - ) - if ( - false_alarm == self.false_alarm - and self.weighting in ("uniform",) - and self.threshold_function in ("weighting",) - ): - score_threshold = self.score_threshold - else: - score_threshold = self._score_threshold_func( - cal_weights, - y_pred=y_pred, - X=X, - timestamps=timestamps, - false_alarm=false_alarm, - ) - prediction_interval = _conformal_set( - y_pred, score_threshold, nonconformity_score=self.nonconformity_score - ) - output: dict = {} - if y_gt is not None: - test_scores = _nonconformity_score_functions( - y_pred, y_gt, X=X, nonconformity_score=self.nonconformity_score - ) - test_outliers = np.array(test_scores > score_threshold).astype("int") - test_ad_scores = [ - _weighted_conformal_alpha( - np.append(self.cal_scores, np.array([np.infty]), axis=0), - np.append(cal_weights, np.array([1]), axis=0), - score, - ) - for score in test_scores - ] - if update: - self.update(test_scores, X=X, timestamps=timestamps) - output["outliers"] = test_outliers - output["outliers_scores"] = np.array(test_ad_scores).flatten() - output["prediction_interval"] = prediction_interval - return output - - def predict( - self, y_pred, y_gt=None, X=None, timestamps=None, false_alarm=None, update=None - ): - if false_alarm is None: - false_alarm = self.false_alarm - if update is None: - update = self.online - n_samples = y_pred.shape[0] - n_batches = int(np.ceil(n_samples / self.online_size)) - if y_gt is not None and update: - output = None - for ix_b in range(n_batches): - ix_ini = int(ix_b * self.online_size) - ix_end = min( - int(ix_b * self.online_size + self.online_size), y_pred.shape[0] - ) - y_pred_b = y_pred[ix_ini:ix_end] - y_gt_b = y_gt[ix_ini:ix_end] - X_b = X[ix_ini:ix_end] if X is not None else None - ts_b = timestamps[ix_ini:ix_end] if timestamps is not None else None - output_b = self.predict_batch( - y_pred_b, - y_gt=y_gt_b, - X=X_b, - timestamps=ts_b, - false_alarm=false_alarm, - update=update, - ) - if output is None: - output = output_b.copy() - else: - for k in output_b: - if k == "prediction_interval": - for k2 in output_b[k]: - output[k][k2] = np.append( - output[k][k2], np.array(output_b[k][k2]), axis=0 - ) - else: - output[k] = np.append( - output[k], np.array(output_b[k]), axis=0 - ) - else: - output = self.predict_batch( - y_pred, - y_gt=y_gt, - X=X, - timestamps=timestamps, - false_alarm=false_alarm, - update=update, - ) - return output - - def update(self, scores, X=None, timestamps=None): - self.cal_scores = np.append(self.cal_scores, scores, axis=0) - self.cal_scores = self.cal_scores[-self.window_size :] - if timestamps is not None: - self.cal_timestamps.extend(timestamps) - self.cal_timestamps = self.cal_timestamps[-self.window_size :] - if X is not None: - self.cal_X = np.append(self.cal_X, X, axis=0) - self.cal_X = self.cal_X[-self.window_size :] - if self.weighting == "uniform": - cal_weights = self.get_weights() - if self.threshold_function == "weighting": - self.score_threshold = self._score_threshold_func( - cal_weights, false_alarm=self.false_alarm - ) - - -# ── TSAD data alignment ─────────────────────────────────────────────────────── - - -def _get_tsfm_dataloaders( - df_dataframe, model_config, dataset_config_dictionary, scaling=False -): - from tsfm_public.toolkit.dataset import ForecastDFDataset - from tsfm_public.toolkit.time_series_preprocessor import TimeSeriesPreprocessor - from tsfm_public.toolkit.util import select_by_index - - forecast_horizon = model_config["prediction_length"] - context_length = model_config["context_length"] - assert context_length <= len(df_dataframe), ( - " length of dataframe needs to be >= context length" - ) - - column_specifiers = dataset_config_dictionary["column_specifiers"] - id_columns = dataset_config_dictionary.get("id_columns", []) - - data = select_by_index( - df_dataframe, id_columns=[], start_index=0, end_index=len(df_dataframe) - ) - - tsp = TimeSeriesPreprocessor( - **column_specifiers, scaling=scaling, encode_categorical=False - ) - tsp = tsp.train(data) - - dataset_inference = ForecastDFDataset( - tsp.preprocess(data), - **column_specifiers, - context_length=context_length, - prediction_length=forecast_horizon, - id_columns=id_columns, - ) - return dataset_inference - - -def _tsfm_dataloader_to_array( - dataset_calibration, ix_target_features, x_context_window=-1 -): - y_gt = [] - X = [] - timestamp_id_value_dic: dict = {} - for i in range(len(dataset_calibration)): - try: - y_gt.append( - dataset_calibration[i]["future_values"][:, ix_target_features] - .detach() - .numpy() - ) - except Exception as ex: - raise ValueError( - f"At least one of the target_columns is not in the input file: {ix_target_features}" - ) from ex - X.append(dataset_calibration[i]["past_values"].detach().numpy()) - if "timestamp" in dataset_calibration[i]: - timestamp_id_value_dic.setdefault("timestamp", []).append( - dataset_calibration[i]["timestamp"] - ) - if "id" in dataset_calibration[i]: - timestamp_id_value_dic.setdefault("id", []).extend( - list(dataset_calibration[i]["id"]) - ) - y_gt_arr = np.array(y_gt) - if len(y_gt_arr.shape) > 1: - y_gt_arr = y_gt_arr[:, 0] - y_gt_arr = np.squeeze(y_gt_arr) - X_arr = np.array(X) - if x_context_window > 0: - X_arr = X_arr[:, -int(x_context_window) :, :] - X_arr = X_arr.reshape([X_arr.shape[0], -1]) - return X_arr, y_gt_arr, timestamp_id_value_dic - - -def _get_tsad_aligned_data( - df_data, dataset_config, ad_config, tsmodel_prediction_dictionary -): - from tsfm_public.toolkit.time_series_preprocessor import create_timestamps - - context_length = ad_config["context_length"] - prediction_length = ad_config["prediction_length"] - scaling = ad_config["scaling"] - ix_target_features = list( - np.arange(len(dataset_config["column_specifiers"]["target_columns"])) - ) - - df_data[dataset_config["column_specifiers"]["timestamp_column"]] = pd.to_datetime( - df_data[dataset_config["column_specifiers"]["timestamp_column"]] - ) - dataset_inference = _get_tsfm_dataloaders( - df_data, - {"prediction_length": prediction_length, "context_length": context_length}, - dataset_config, - scaling=scaling, - ) - X, y_gt, timestamp_id_value_dic = _tsfm_dataloader_to_array( - dataset_inference, ix_target_features, x_context_window=context_length - ) - - source_timestamp = np.array(tsmodel_prediction_dictionary["timestamp"])[:, 0] - target_timestamp = timestamp_id_value_dic["timestamp"] - - forecast_horizon = 1 - target_timestamp_updated = [] - for ts in target_timestamp: - ts_updated = create_timestamps( - last_timestamp=ts, - time_sequence=target_timestamp, - periods=forecast_horizon, - )[0] - target_timestamp_updated.append(ts_updated) - target_timestamp = np.array( - np.array(target_timestamp_updated, dtype="datetime64[ns]") - ) - source_timestamp = np.array(np.array(source_timestamp, dtype="datetime64[ns]")) - - frequency_sampling_median = np.median(target_timestamp[1:] - target_timestamp[:-1]) - tolerance_frequency_sampling = 0.2 - - time_diff = np.abs(target_timestamp[:, None] - source_timestamp) - matching_pairs = np.where( - time_diff <= frequency_sampling_median * tolerance_frequency_sampling - ) - index_timestamp = matching_pairs[0] - index_timestamp_source = matching_pairs[1] - - X_cp = X[index_timestamp] - y_gt_cp = y_gt[index_timestamp] - y_pred = np.array(tsmodel_prediction_dictionary["target_prediction"])[ - index_timestamp_source, 0, 0 - ] - timestamps_source = np.array(source_timestamp)[index_timestamp_source] - - return { - "X": X_cp, - "y_gt": y_gt_cp, - "y_pred": y_pred, - "timestamp": timestamps_source, - } - - -# ── TSAD orchestration ──────────────────────────────────────────────────────── - -_AD_CONFIG_DEFAULT = { - "ad_model_type": "timeseries_conformal", - "context_length": 1, - "false_alarm": 0.01, - "window_size": None, - "weighting": "uniform", - "weighting_params": {}, -} - - -class _TimeSeriesAnomalyDetectionConformalWrapper: - def run( - self, - dataset_path: str, - dataset_config_dictionary: dict, - tsmodel_prediction_dictionary: dict, - ad_model_checkpoint: str = None, - ad_model_save: str = None, - task: str = "inference", - ad_model_type: str = None, - n_calibration=None, - false_alarm: float = None, - context_length: int = None, - ) -> dict: - ad_model = None - if ad_model_checkpoint is not None: - if os.path.exists(ad_model_checkpoint): - assert os.path.exists(ad_model_checkpoint + "/model.pkl") - assert os.path.exists(ad_model_checkpoint + "/config.json") - with open(ad_model_checkpoint + "/model.pkl", "rb") as _f: - ad_model = pickle.load(_f) - with open(ad_model_checkpoint + "/config.json") as _f: - ad_config = json.load(_f) - ad_model_type = ad_config["ad_model_type"] - context_length = ad_config["context_length"] - if false_alarm is None: - false_alarm = ad_config["false_alarm"] - elif ad_config["false_alarm"] != false_alarm: - if task != "fit": - false_alarm = ad_config["false_alarm"] - else: - ad_config = { - "context_length": context_length - if context_length is not None - else _AD_CONFIG_DEFAULT["context_length"], - "false_alarm": false_alarm - if false_alarm is not None - else _AD_CONFIG_DEFAULT["false_alarm"], - "ad_model_type": ad_model_type - if ad_model_type is not None - else _AD_CONFIG_DEFAULT["ad_model_type"], - } - context_length = ad_config["context_length"] - false_alarm = ad_config["false_alarm"] - ad_model_type = ad_config["ad_model_type"] - - df_data = _read_ts_data( - dataset_path, dataset_config_dictionary=dataset_config_dictionary - ) - context_length = ad_config["context_length"] - output_tsad_aligned = _get_tsad_aligned_data( - df_data, - dataset_config_dictionary, - ad_config={ - "prediction_length": 1, - "context_length": context_length, - "scaling": False, - }, - tsmodel_prediction_dictionary=tsmodel_prediction_dictionary, - ) - - timestamps_source = output_tsad_aligned["timestamp"] - X_cp = output_tsad_aligned["X"] - y_gt_cp = output_tsad_aligned["y_gt"] - y_pred = output_tsad_aligned["y_pred"] - - output_ad: dict = {} - if task == "fit": - if n_calibration is None: - n_calibration = y_pred.shape[0] - if n_calibration < 1: - n_calibration = int(np.ceil(y_pred.shape[0] * n_calibration)) - n_calibration = int(n_calibration) - n_critical = int(np.ceil(1 / false_alarm)) - assert n_calibration >= n_critical, ( - f" n_calibration should be >= {n_critical}, " - f"otherwise increase false alarm to {round(1 / n_calibration, 2)}" - ) - - X_cp_cal = X_cp[:n_calibration] - y_gt_cp_cal = y_gt_cp[:n_calibration] - y_pred_cal = y_pred[:n_calibration] - - if ad_model_type in ( - "timeseries_conformal", - "timeseries_conformal_adaptive", - ): - update = ad_model_type == "timeseries_conformal_adaptive" - ad_model = _TSADWeightedConformalWrapper( - false_alarm=false_alarm, - weighting=_AD_CONFIG_DEFAULT["weighting"], - window_size=_AD_CONFIG_DEFAULT["window_size"], - weighting_params=_AD_CONFIG_DEFAULT["weighting_params"], - online_adaptive=update, - ) - ad_model.fit( - y_cal_pred=np.array(y_pred_cal), y_cal_gt=np.array(y_gt_cp_cal) - ) - output_prediction = ad_model.predict( - y_pred=np.array(y_pred), y_gt=np.array(y_gt_cp), update=update - ) - output_ad = { - "timestamp": timestamps_source, - "KPI": tsmodel_prediction_dictionary["target_columns"], - "value": np.array(y_gt_cp), - "upper_bound": output_prediction["prediction_interval"]["y_high"], - "lower_bound": output_prediction["prediction_interval"]["y_low"], - "anomaly_score": 1 - output_prediction["outliers_scores"], - "anomaly_label": output_prediction["outliers"] == 1, - "split": [ - "calibration" if i < n_calibration else "test" - for i in range(y_pred.shape[0]) - ], - } - - if ad_model is not None and ad_model_save is not None: - with open(ad_model_save + "/model.pkl", "wb") as _f: - pickle.dump(ad_model, _f) - with open(ad_model_save + "/config.json", "w") as _f: - json.dump(ad_config, _f) - - if task == "inference": - if false_alarm is None: - false_alarm = ad_model.false_alarm - output_prediction = ad_model.predict( - y_pred=np.array(y_pred), y_gt=np.array(y_gt_cp) - ) - output_ad = { - "timestamp": timestamps_source, - "KPI": tsmodel_prediction_dictionary["target_columns"], - "value": np.array(y_gt_cp), - "upper_bound": output_prediction["prediction_interval"]["y_high"], - "lower_bound": output_prediction["prediction_interval"]["y_low"], - "anomaly_score": 1 - output_prediction["outliers_scores"], - "anomaly_label": output_prediction["outliers"] == 1, - "split": ["test"] * output_prediction["outliers_scores"].shape[0], - } - - return output_ad diff --git a/src/servers/tsfm/artifacts/output/tuned_models/args_config.yml b/src/servers/tsfm/artifacts/output/tuned_models/args_config.yml deleted file mode 100644 index 7d8432954..000000000 --- a/src/servers/tsfm/artifacts/output/tuned_models/args_config.yml +++ /dev/null @@ -1,20 +0,0 @@ -backbone_frozen: false -batch_size: 32 -context_length: 96 -decoder_mode: mix_channel -encode_categorical: false -epochs: 4 -epochs_warmup: 5 -es_patience: 15.0 -es_th: 0.0001 -forecast_horizon: 28 -head_dropout: 0.7 -lr: 0.0 -model_type: ttm -num_workers: 4 -optim: AdamW -p_validation: 0.1 -patch_length: 64 -scaling: '' -scheduler: OneCycleLR -seed: 42 diff --git a/src/servers/tsfm/artifacts/output/tuned_models/training_config.yml b/src/servers/tsfm/artifacts/output/tuned_models/training_config.yml deleted file mode 100644 index 580890c0f..000000000 --- a/src/servers/tsfm/artifacts/output/tuned_models/training_config.yml +++ /dev/null @@ -1,17 +0,0 @@ -dataloader_num_workers: 4 -do_eval: true -evaluation_strategy: epoch -greater_is_better: false -learning_rate: 0.0 -load_best_model_at_end: true -logging_dir: /Users/shuxin.lin/Projects/TSFMAgent/output/tuned_models/log/ -logging_strategy: epoch -metric_for_best_model: eval_loss -num_train_epochs: 4 -output_dir: /Users/shuxin.lin/Projects/TSFMAgent/output/tuned_models/fewshot/ -overwrite_output_dir: true -per_device_eval_batch_size: 32 -per_device_train_batch_size: 32 -save_strategy: epoch -save_total_limit: 3 -warmup_steps: 1 diff --git a/src/servers/tsfm/artifacts/output/tuned_models/tsp.pickle b/src/servers/tsfm/artifacts/output/tuned_models/tsp.pickle deleted file mode 100644 index 791542d3d..000000000 Binary files a/src/servers/tsfm/artifacts/output/tuned_models/tsp.pickle and /dev/null differ diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_512_96/config.json b/src/servers/tsfm/artifacts/tsfm_models/ttm_512_96/config.json deleted file mode 100644 index 2efa6275b..000000000 --- a/src/servers/tsfm/artifacts/tsfm_models/ttm_512_96/config.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "adaptive_patching_levels": 3, - "architectures": [ - "TinyTimeMixerForPrediction" - ], - "categorical_vocab_size_list": null, - "context_length": 96, - "d_model": 24, - "d_model_scale": 3, - "decoder_adaptive_patching_levels": 0, - "decoder_d_model": 16, - "decoder_d_model_scale": 2, - "decoder_mode": "common_channel", - "decoder_num_layers": 2, - "decoder_raw_residual": false, - "distribution_output": "student_t", - "dropout": 0.2, - "enable_forecast_channel_mixing": false, - "exogenous_channel_indices": null, - "expansion_factor": 2, - "fcm_context_length": 1, - "fcm_gated_attn": true, - "fcm_mix_layers": 3, - "fcm_prepend_past": true, - "fcm_use_mixer": true, - "frequency_token_vocab_size": 8, - "gated_attn": true, - "head_dropout": 0.2, - "init_processing": true, - "init_std": 0.02, - "loss": "mse", - "mode": "common_channel", - "model_type": "tinytimemixer", - "norm_eps": 1e-05, - "norm_mlp": "LayerNorm", - "num_input_channels": 1, - "num_layers": 2, - "num_parallel_samples": 100, - "num_patches": 12, - "patch_last": true, - "patch_length": 8, - "patch_stride": 8, - "positional_encoding_type": "sincos", - "post_init": false, - "prediction_channel_indices": null, - "prediction_filter_length": null, - "prediction_length": 28, - "resolution_prefix_tuning": false, - "scaling": "std", - "self_attn": false, - "self_attn_heads": 1, - "stride_ratio": 1, - "torch_dtype": "float32", - "transformers_version": "4.37.2", - "use_decoder": true, - "use_positional_encoding": false -} diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_512_96/generation_config.json b/src/servers/tsfm/artifacts/tsfm_models/ttm_512_96/generation_config.json deleted file mode 100644 index babac5557..000000000 --- a/src/servers/tsfm/artifacts/tsfm_models/ttm_512_96/generation_config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "_from_model_config": true, - "transformers_version": "4.37.2" -} diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_512_96/model.safetensors b/src/servers/tsfm/artifacts/tsfm_models/ttm_512_96/model.safetensors deleted file mode 100644 index 1e9ff86ac..000000000 Binary files a/src/servers/tsfm/artifacts/tsfm_models/ttm_512_96/model.safetensors and /dev/null differ diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_512_96/training_args.bin b/src/servers/tsfm/artifacts/tsfm_models/ttm_512_96/training_args.bin deleted file mode 100644 index b1d8d3c79..000000000 Binary files a/src/servers/tsfm/artifacts/tsfm_models/ttm_512_96/training_args.bin and /dev/null differ diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_52_16_mae/ttm_model/config.json b/src/servers/tsfm/artifacts/tsfm_models/ttm_52_16_mae/ttm_model/config.json deleted file mode 100644 index 15f7cd10e..000000000 --- a/src/servers/tsfm/artifacts/tsfm_models/ttm_52_16_mae/ttm_model/config.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "adaptive_patching_levels": 0, - "architectures": [ - "TinyTimeMixerForPrediction" - ], - "categorical_vocab_size_list": null, - "context_length": 52, - "d_model": 20, - "d_model_scale": 5, - "decoder_adaptive_patching_levels": 0, - "decoder_d_model": 16, - "decoder_d_model_scale": 4, - "decoder_mode": "common_channel", - "decoder_num_layers": 4, - "decoder_raw_residual": false, - "distribution_output": "student_t", - "dropout": 0.3, - "enable_forecast_channel_mixing": false, - "exogenous_channel_indices": null, - "expansion_factor": 3, - "fcm_context_length": 1, - "fcm_gated_attn": true, - "fcm_mix_layers": 3, - "fcm_prepend_past": true, - "fcm_prepend_past_offset": null, - "fcm_use_mixer": true, - "frequency_token_vocab_size": 10, - "gated_attn": true, - "head_dropout": 0.3, - "init_embed": "pytorch", - "init_linear": "pytorch", - "init_processing": true, - "init_std": 0.02, - "loss": "mae", - "mask_value": 0, - "masked_context_length": null, - "mode": "common_channel", - "model_type": "tinytimemixer", - "norm_eps": 1e-05, - "norm_mlp": "LayerNorm", - "num_input_channels": 1, - "num_layers": 12, - "num_parallel_samples": 100, - "num_patches": 14, - "patch_last": true, - "patch_length": 4, - "patch_stride": 4, - "positional_encoding_type": "sincos", - "post_init": false, - "prediction_channel_indices": null, - "prediction_filter_length": null, - "prediction_length": 16, - "resolution_prefix_tuning": true, - "scaling": "std", - "self_attn": false, - "self_attn_heads": 1, - "stride_ratio": 1, - "torch_dtype": "float32", - "transformers_version": "4.38.0", - "use_decoder": true, - "use_positional_encoding": false -} diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_52_16_mae/ttm_model/generation_config.json b/src/servers/tsfm/artifacts/tsfm_models/ttm_52_16_mae/ttm_model/generation_config.json deleted file mode 100644 index 1299bb081..000000000 --- a/src/servers/tsfm/artifacts/tsfm_models/ttm_52_16_mae/ttm_model/generation_config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "_from_model_config": true, - "transformers_version": "4.38.0" -} diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_52_16_mae/ttm_model/model.safetensors b/src/servers/tsfm/artifacts/tsfm_models/ttm_52_16_mae/ttm_model/model.safetensors deleted file mode 100644 index d2a656129..000000000 Binary files a/src/servers/tsfm/artifacts/tsfm_models/ttm_52_16_mae/ttm_model/model.safetensors and /dev/null differ diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_52_16_mae/ttm_model/training_args.bin b/src/servers/tsfm/artifacts/tsfm_models/ttm_52_16_mae/ttm_model/training_args.bin deleted file mode 100644 index 1c6a8db1e..000000000 Binary files a/src/servers/tsfm/artifacts/tsfm_models/ttm_52_16_mae/ttm_model/training_args.bin and /dev/null differ diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mae/config.json b/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mae/config.json deleted file mode 100644 index ff280da9c..000000000 --- a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mae/config.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "adaptive_patching_levels": 0, - "architectures": [ - "TinyTimeMixerForPrediction" - ], - "categorical_vocab_size_list": null, - "context_length": 90, - "d_model": 63, - "d_model_scale": 7, - "decoder_adaptive_patching_levels": 0, - "decoder_d_model": 18, - "decoder_d_model_scale": 2, - "decoder_mode": "common_channel", - "decoder_num_layers": 4, - "decoder_raw_residual": false, - "distribution_output": "student_t", - "dropout": 0.4, - "enable_forecast_channel_mixing": false, - "exogenous_channel_indices": null, - "expansion_factor": 2, - "fcm_context_length": 1, - "fcm_gated_attn": true, - "fcm_mix_layers": 3, - "fcm_prepend_past": true, - "fcm_prepend_past_offset": null, - "fcm_use_mixer": true, - "frequency_token_vocab_size": 10, - "gated_attn": true, - "head_dropout": 0.4, - "init_embed": "pytorch", - "init_linear": "pytorch", - "init_processing": true, - "init_std": 0.02, - "loss": "mae", - "mask_value": 0, - "masked_context_length": null, - "mode": "common_channel", - "model_type": "tinytimemixer", - "norm_eps": 1e-05, - "norm_mlp": "LayerNorm", - "num_input_channels": 1, - "num_layers": 20, - "num_parallel_samples": 100, - "num_patches": 11, - "patch_last": true, - "patch_length": 9, - "patch_stride": 9, - "positional_encoding_type": "sincos", - "post_init": false, - "prediction_channel_indices": null, - "prediction_filter_length": null, - "prediction_length": 30, - "resolution_prefix_tuning": true, - "scaling": "std", - "self_attn": false, - "self_attn_heads": 1, - "stride_ratio": 1, - "torch_dtype": "float32", - "transformers_version": "4.38.0", - "use_decoder": true, - "use_positional_encoding": false -} diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mae/generation_config.json b/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mae/generation_config.json deleted file mode 100644 index 1299bb081..000000000 --- a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mae/generation_config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "_from_model_config": true, - "transformers_version": "4.38.0" -} diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mae/model.safetensors b/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mae/model.safetensors deleted file mode 100644 index ed1b9dd05..000000000 Binary files a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mae/model.safetensors and /dev/null differ diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mae/training_args.bin b/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mae/training_args.bin deleted file mode 100644 index e2753cac8..000000000 Binary files a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mae/training_args.bin and /dev/null differ diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mse/config.json b/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mse/config.json deleted file mode 100644 index 72eb5b955..000000000 --- a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mse/config.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "adaptive_patching_levels": 0, - "architectures": [ - "TinyTimeMixerForPrediction" - ], - "categorical_vocab_size_list": null, - "context_length": 90, - "d_model": 90, - "d_model_scale": 10, - "decoder_adaptive_patching_levels": 0, - "decoder_d_model": 18, - "decoder_d_model_scale": 2, - "decoder_mode": "common_channel", - "decoder_num_layers": 4, - "decoder_raw_residual": false, - "distribution_output": "student_t", - "dropout": 0.3, - "enable_forecast_channel_mixing": false, - "exogenous_channel_indices": null, - "expansion_factor": 2, - "fcm_context_length": 1, - "fcm_gated_attn": true, - "fcm_mix_layers": 3, - "fcm_prepend_past": true, - "fcm_prepend_past_offset": null, - "fcm_use_mixer": true, - "frequency_token_vocab_size": 10, - "gated_attn": true, - "head_dropout": 0.3, - "init_embed": "pytorch", - "init_linear": "pytorch", - "init_processing": true, - "init_std": 0.02, - "loss": "mse", - "mask_value": 0, - "masked_context_length": null, - "mode": "common_channel", - "model_type": "tinytimemixer", - "norm_eps": 1e-05, - "norm_mlp": "LayerNorm", - "num_input_channels": 1, - "num_layers": 20, - "num_parallel_samples": 100, - "num_patches": 11, - "patch_last": true, - "patch_length": 9, - "patch_stride": 9, - "positional_encoding_type": "sincos", - "post_init": false, - "prediction_channel_indices": null, - "prediction_filter_length": null, - "prediction_length": 30, - "resolution_prefix_tuning": true, - "scaling": "std", - "self_attn": false, - "self_attn_heads": 1, - "stride_ratio": 1, - "torch_dtype": "float32", - "transformers_version": "4.38.0", - "use_decoder": true, - "use_positional_encoding": false -} diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mse/generation_config.json b/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mse/generation_config.json deleted file mode 100644 index 1299bb081..000000000 --- a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mse/generation_config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "_from_model_config": true, - "transformers_version": "4.38.0" -} diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mse/model.safetensors b/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mse/model.safetensors deleted file mode 100644 index e5eb65a83..000000000 Binary files a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mse/model.safetensors and /dev/null differ diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mse/training_args.bin b/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mse/training_args.bin deleted file mode 100644 index 319a2aaf8..000000000 Binary files a/src/servers/tsfm/artifacts/tsfm_models/ttm_90_30_mse/training_args.bin and /dev/null differ diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_96_28/config.json b/src/servers/tsfm/artifacts/tsfm_models/ttm_96_28/config.json deleted file mode 100644 index 2efa6275b..000000000 --- a/src/servers/tsfm/artifacts/tsfm_models/ttm_96_28/config.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "adaptive_patching_levels": 3, - "architectures": [ - "TinyTimeMixerForPrediction" - ], - "categorical_vocab_size_list": null, - "context_length": 96, - "d_model": 24, - "d_model_scale": 3, - "decoder_adaptive_patching_levels": 0, - "decoder_d_model": 16, - "decoder_d_model_scale": 2, - "decoder_mode": "common_channel", - "decoder_num_layers": 2, - "decoder_raw_residual": false, - "distribution_output": "student_t", - "dropout": 0.2, - "enable_forecast_channel_mixing": false, - "exogenous_channel_indices": null, - "expansion_factor": 2, - "fcm_context_length": 1, - "fcm_gated_attn": true, - "fcm_mix_layers": 3, - "fcm_prepend_past": true, - "fcm_use_mixer": true, - "frequency_token_vocab_size": 8, - "gated_attn": true, - "head_dropout": 0.2, - "init_processing": true, - "init_std": 0.02, - "loss": "mse", - "mode": "common_channel", - "model_type": "tinytimemixer", - "norm_eps": 1e-05, - "norm_mlp": "LayerNorm", - "num_input_channels": 1, - "num_layers": 2, - "num_parallel_samples": 100, - "num_patches": 12, - "patch_last": true, - "patch_length": 8, - "patch_stride": 8, - "positional_encoding_type": "sincos", - "post_init": false, - "prediction_channel_indices": null, - "prediction_filter_length": null, - "prediction_length": 28, - "resolution_prefix_tuning": false, - "scaling": "std", - "self_attn": false, - "self_attn_heads": 1, - "stride_ratio": 1, - "torch_dtype": "float32", - "transformers_version": "4.37.2", - "use_decoder": true, - "use_positional_encoding": false -} diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_96_28/generation_config.json b/src/servers/tsfm/artifacts/tsfm_models/ttm_96_28/generation_config.json deleted file mode 100644 index babac5557..000000000 --- a/src/servers/tsfm/artifacts/tsfm_models/ttm_96_28/generation_config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "_from_model_config": true, - "transformers_version": "4.37.2" -} diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_96_28/model.safetensors b/src/servers/tsfm/artifacts/tsfm_models/ttm_96_28/model.safetensors deleted file mode 100644 index 1e9ff86ac..000000000 Binary files a/src/servers/tsfm/artifacts/tsfm_models/ttm_96_28/model.safetensors and /dev/null differ diff --git a/src/servers/tsfm/artifacts/tsfm_models/ttm_96_28/training_args.bin b/src/servers/tsfm/artifacts/tsfm_models/ttm_96_28/training_args.bin deleted file mode 100644 index b1d8d3c79..000000000 Binary files a/src/servers/tsfm/artifacts/tsfm_models/ttm_96_28/training_args.bin and /dev/null differ diff --git a/src/servers/tsfm/bootstrap.py b/src/servers/tsfm/bootstrap.py new file mode 100644 index 000000000..6e8359021 --- /dev/null +++ b/src/servers/tsfm/bootstrap.py @@ -0,0 +1,62 @@ +"""Seed loader — populate the model & feature catalogs into a Store.""" + +from __future__ import annotations + +import json +import os + +from .core import store as store_mod +from .stores import model_store +from .stores import feature_store + +# Single source of truth: the catalog lives with the other AssetOpsBench CouchDB collection data +# at src/couchdb/scenarios_data/shared/tsfm/ (the package no longer ships its own seeds/ copy). +# Override with TSFM_SEEDS_DIR when the package is relocated or for tests. +_SEEDS = os.environ.get("TSFM_SEEDS_DIR") or os.path.normpath( + os.path.join( + os.path.dirname(__file__), + "..", + "..", + "couchdb", + "scenarios_data", + "shared", + "tsfm", + ) +) + + +def load_seeds(store) -> dict: + """Load the two collection files — one file == one collection, exactly like the other + AssetOpsBench servers. model_catalog.json holds every model card (foundation + classical + + anomaly/clustering components); feature_catalog.json holds the feature transforms + the full + FLOps extractor library (baked). Both are CouchDB-ready (each doc carries `_id`). Source: + src/couchdb/scenarios_data/shared/tsfm/ (or $TSFM_SEEDS_DIR).""" + counts = {} + mc = json.load(open(os.path.join(_SEEDS, "model_catalog.json"))) + for doc in mc: + doc.setdefault("_id", f"model:{doc['model_id']}") + store.put(model_store.COLLECTION, doc) + counts["model_catalog"] = len(mc) + fc = json.load(open(os.path.join(_SEEDS, "feature_catalog.json"))) + for doc in fc: + doc.setdefault( + "_id", + f"feature:{doc.get('feature_id') or doc.get('extractor_name') or doc.get('name')}", + ) + doc.setdefault("kind", "transform") + store.put(feature_store.COLLECTION, doc) + counts["feature_catalog"] = len(fc) + return counts + + +def fresh_store(load: bool = True): + """Make the configured store; seed the default catalog ONLY when it's empty. + + This is what makes the catalog a per-scenario dataset: a JSON store dir (or CouchDB) that + already carries a catalog — even a partial one — is used as-is and never overwritten by the + packaged seeds. An empty/new store (incl. the in-memory test store) gets the curated defaults. + """ + s = store_mod.make_store() + if load and not s.find(model_store.COLLECTION, limit=1): + load_seeds(s) + return s diff --git a/src/servers/tsfm/config.py b/src/servers/tsfm/config.py new file mode 100644 index 000000000..5012f62b0 --- /dev/null +++ b/src/servers/tsfm/config.py @@ -0,0 +1,11 @@ +"""config.py — environment + shared constants (single source of truth). + +Centralizes the few env knobs and collection names so modules don't hard-code them. Existing +modules keep working unchanged; new code should read settings from here. +""" + +from __future__ import annotations + +# run/plan ledgers +RUNS_COLLECTION = "tsfm_runs" +PLANS_COLLECTION = "tsfm_plans" diff --git a/src/servers/tsfm/core/__init__.py b/src/servers/tsfm/core/__init__.py new file mode 100644 index 000000000..777f94f99 --- /dev/null +++ b/src/servers/tsfm/core/__init__.py @@ -0,0 +1 @@ +"""core layer.""" diff --git a/src/servers/tsfm/core/glossary.py b/src/servers/tsfm/core/glossary.py new file mode 100644 index 000000000..6cfee9287 --- /dev/null +++ b/src/servers/tsfm/core/glossary.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from typing import Dict, List + +# term → (definition, how it relates to the others / which tool exposes it) +TERMS: Dict[str, Dict[str, str]] = { + "task": { + "definition": "a standardized TS-AI problem type (8 total: forecasting, regression, " + "classification, anomaly_detection, imputation, evaluation, " + "similarity_search, clustering). Defines required inputs, output, eval.", + "tool": "list_tasks", + }, + "component": { + "definition": "any catalog entry you can place in a recipe — a MODEL or a FEATURE. " + "Components are DATA (cards), not tools.", + "tool": "get_component", + }, + "model": { + "definition": "an estimator card (forecaster / classifier / detector / clusterer) that " + "points at an sktime or foundation-model class.", + "tool": "find_models", + }, + "candidate": { + "definition": "a model proposed AND ranked for a task (HuggingGPT-style, by description + " + "popularity). A shortlist — you still choose.", + "tool": "describe_candidates", + }, + "feature": { + "definition": "a transform/extractor card (normalization, lag/rolling, catch22, a " + "FLOps-selected set). Applied before the estimator.", + "tool": "find_features", + }, + "transform": { + "definition": "a feature used as a preprocessing/extraction step inside a recipe; some " + "are invertible (round-trip back to input space).", + "tool": "find_features", + }, + "ensemble": { + "definition": "a recipe that combines several models (mean / median / weighted / stack).", + "tool": "run_recipe", + }, + "recipe": { + "definition": "the declarative spec YOU author: transforms + a single model OR an " + "ensemble + optional conformal/finetune/anomaly blocks + eval protocol.", + "tool": "run_recipe / run_tabular_recipe / run_plan", + }, + "training_regime": { + "definition": "how much training a model needs: zero_shot (pretrained, no training — the " + "default) | fit_on_series | fine_tune.", + "tool": "discover_components", + }, + "param_schema": { + "definition": "per-model parameter hints + ranges you reason over (context_length, sp, " + "n_neighbors, …) from the data evidence.", + "tool": "get_component", + }, + "data_ref": { + "definition": "data is passed BY REFERENCE — a file pointer (dataset_path: a CSV/parquet " + "path or file:// URI), never inlined. Results return as a results_file pointer.", + "tool": "profile_series / run_recipe", + }, + "evidence": { + "definition": "the facts about the data you reason from (seasonality, stationarity, " + "channels, length). The server states facts; YOU decide.", + "tool": "profile_series", + }, + "result": { + "definition": "a written per-task output record in a result table (forecast/anomaly/…).", + "tool": "get_result / list_results", + }, + "run": { + "definition": "a recipe-execution record with provenance, param_audit and lineage.", + "tool": "get_run / list_runs", + }, + "evolve": { + "definition": "an AlphaEvolve-style loop: ask for parents+inspirations from an archive of " + "elites (best program per behaviour cell), mutate into a new candidate, tell " + "it back to be validated/scored/archived. You propose; the server grades.", + "tool": "evolve_ask / evolve_tell / evolve_best", + }, +} + +# the canonical sequence — what to call, in order +WORKFLOW: List[str] = [ + "1. list_tasks — pick the task (defines inputs/output/eval).", + "2. profile_series(dataset_path) — read the data evidence.", + "3. discover_components(task) / describe_candidates / find_models / find_features — see the menu.", + "4. get_component(id) — read a card + its param_schema; reason the parameters.", + "5. author a recipe → run_recipe / run_tabular_recipe / run_plan (zero-shot first).", + "6. evaluate (GIFT-Eval) → inspect get_run → revise the recipe and iterate.", +] + +PRINCIPLES: List[str] = [ + "Models & features are DATA (catalog cards), not tools — composed via recipes.", + "Data crosses the boundary as file pointers; results come back as file pointers.", + "The agent reasons every choice; the server gives evidence + grades, it does not decide.", + "Zero-shot is the default; fine-tune is an explicit, optional escalation.", +] + + +def glossary() -> dict: + """The full vocabulary + workflow + principles — safe to embed anywhere the agent reads.""" + return {"terms": TERMS, "workflow": WORKFLOW, "principles": PRINCIPLES} + + +def short_glossary() -> str: + """A compact one-line-per-term string for the server instructions (always in context).""" + return " · ".join(f"{k}: {v['definition'].split('.')[0]}" for k, v in TERMS.items()) diff --git a/src/servers/tsfm/core/results_models.py b/src/servers/tsfm/core/results_models.py new file mode 100644 index 000000000..7d5452025 --- /dev/null +++ b/src/servers/tsfm/core/results_models.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict + + +class ErrorResult(BaseModel): + error: str + + +# ---- discovery / catalog (no bulk data) ---- +class TasksResult(BaseModel): + tasks: List[dict] + + +class ComponentsResult(BaseModel): + task: str + components: dict + + +class CandidatesResult(BaseModel): + task_id: str + candidates: List[dict] + + +class ModelsResult(BaseModel): + models: List[dict] + + +class FeaturesResult(BaseModel): + features: List[dict] + + +class ComponentResult(BaseModel): + model_config = ConfigDict(extra="allow", protected_namespaces=()) + component_id: str + + +# ---- evidence / learn ---- +class ProfileResult(BaseModel): + model_config = ConfigDict(extra="allow") + source: str + n_observations: int + n_channels: int + dominant_period: Optional[int] = None + + +class FeatureSelectionResult(BaseModel): + selected: List[str] + lookback: int + reference: str + scorers: List[str] + detail_file: str # file pointer to the full per-scorer scores + + +class CharacterizeResult(BaseModel): + """Pattern EVIDENCE for a series: per-group state+rate phases + bivariate relations + a + shape-only NL summary. Domain-agnostic; full structured evidence at evidence_file. + """ + + model_config = ConfigDict(extra="allow") + status: str + summary: str # shape-only NL description (never names a fault) + n_observations: int + evidence_file: str # file pointer to the full pattern object + message: str + + +# ---- compose + run (file pointers) ---- +class RecipeResult(BaseModel): + """A recipe run. Forecasting carries metric+backtest_score; anomaly carries n_anomalies+ + n_observations (extra-allowed) — run_recipe dispatches by recipe.task.""" + + model_config = ConfigDict(extra="allow") + status: str + run_id: str + results_file: ( + str # file pointer to the run record (forecast/intervals OR anomaly labels) + ) + training_regime: str + message: str + metric: Optional[str] = None + backtest_score: Optional[float] = None + + +class TabularResult(BaseModel): + status: str + run_id: str + results_file: str + task: str + metric: str + cv_score: Any + n_features: int + message: str + + +class DataQualityResult(BaseModel): + model_config = ConfigDict(extra="allow") + status: str + cleaned_file: str # file pointer to the cleaned series + rows_in: int + rows_out: int + message: str + + +class PlanResult(BaseModel): + model_config = ConfigDict(extra="allow") + status: str + results_file: str + message: str + + +class EvaluateResult(BaseModel): + model_config = ConfigDict(extra="allow") + status: str + results_file: str + message: str + + +# ---- write-back / results ---- +class RegisterResult(BaseModel): + model_config = ConfigDict(extra="allow", protected_namespaces=()) + status: str + id: str + + +class CardResult(BaseModel): + """A single catalog card returned by update / deprecate / new_version / register_finetuned.""" + + model_config = ConfigDict(extra="allow", protected_namespaces=()) + + +class LineageResult(BaseModel): + """A card's evolution chain (ancestors + descendants / supersedes links).""" + + model_config = ConfigDict(extra="allow") + + +class ResultsListResult(BaseModel): + results: List[dict] + + +class ResultRecord(BaseModel): + """A single stored result-table record (task-polymorphic; extra fields allowed).""" + + model_config = ConfigDict(extra="allow") + + +class RunRecord(BaseModel): + """A single stored run record (recipe run / plan; extra fields allowed).""" + + model_config = ConfigDict(extra="allow") + run_id: str + + +class RunsResult(BaseModel): + runs: List[dict] + plans: List[dict] + + +# ---- evolve (AlphaEvolve loop) ---- +class EvolveAskResult(BaseModel): + model_config = ConfigDict(extra="allow") + task: str + kind: str + + +class EvolveTellResult(BaseModel): + model_config = ConfigDict(extra="allow") + accepted: bool + + +class EvolveBestResult(BaseModel): + model_config = ConfigDict(extra="allow") + task: str + +class ExtractResult(BaseModel): + n_windows: int + window: Optional[int] + columns: List[str] # feature columns: '.' (or '') + features: List[List[float]] # n_windows rows x columns (whole-series => 1 row) + message: str \ No newline at end of file diff --git a/src/servers/tsfm/core/schemas.py b/src/servers/tsfm/core/schemas.py new file mode 100644 index 000000000..bae7b4b31 --- /dev/null +++ b/src/servers/tsfm/core/schemas.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from enum import Enum +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +# --------------------------------------------------------------------------- # +class Provenance(str, Enum): + pretrained = "pretrained" + finetuned = "finetuned" + trained = "trained" + external_hf = "external_hf" + external_service = "external_service" + toolkit = "toolkit" + + +class Status(str, Enum): + active = "active" + deprecated = "deprecated" + experimental = "experimental" + superseded = "superseded" + + +class Modality(str, Enum): + timeseries = "timeseries" + vision = "vision" + text = "text" + multimodal = "multimodal" + audio = "audio" + + +class Metric(BaseModel): + model_config = ConfigDict(extra="allow") + metric: str + value: Optional[float] = None + + +# --------------------------------------------------------------------------- # +class ModelCard(BaseModel): + """A model-store entry. Pointer index: weights live at one of artifact_path / hf_repo / + remote_endpoint / model_checkpoint (toolkit).""" + model_config = ConfigDict(extra="allow", protected_namespaces=()) + + model_id: str + description: str = Field(min_length=3) + task_ids: List[str] = Field(min_length=1) + + model_checkpoint: Optional[str] = None + framework: Optional[str] = None + model_family: Optional[str] = None + modality: Modality = Modality.timeseries + provenance: Provenance = Provenance.pretrained + base_model_id: Optional[str] = None + usage_modes: List[str] = Field(default_factory=list) + output_type: Optional[str] = None + context_length: Optional[int] = None + prediction_length: Optional[int] = None + domain: str = "general" + frequency: Optional[str] = None + + source: Optional[str] = None + artifact_path: Optional[str] = None + hf_repo: Optional[str] = None + remote_endpoint: Optional[str] = None + pipeline_type: Optional[str] = None + + metrics: List[Metric] = Field(default_factory=list) + trained_on: Optional[Any] = None + tags: List[str] = Field(default_factory=list) + status: Status = Status.active + version: str = "1" + created_by: str = "seed" + created_at: str = Field(default_factory=_now) + + @field_validator("context_length", "prediction_length") + @classmethod + def _non_negative(cls, v): + if v is not None and v < 0: + raise ValueError("length must be >= 0") + return v + + @model_validator(mode="after") + def _resolvable(self): + # must be loadable somewhere (else it's a catalog-only stub — allowed but flagged) + refs = [self.artifact_path, self.hf_repo, self.remote_endpoint, self.model_checkpoint] + object.__setattr__(self, "resolvable", any(refs) or self.source == "toolkit") + if self.provenance == Provenance.finetuned and not self.base_model_id: + raise ValueError("finetuned model requires base_model_id (lineage)") + return self + + def to_doc(self) -> dict: + d = self.model_dump(mode="json") + d["_id"] = f"model:{self.model_id}" + return d + + +# --------------------------------------------------------------------------- # +class Interface(str, Enum): + fit_transform = "fit_transform" + fit_transform_inverse = "fit_transform_inverse" + + +class FeatureCard(BaseModel): + """A feature-store entry — an EFE-style fit/transform program stored as code.""" + model_config = ConfigDict(extra="allow") + + feature_id: str + interface: Interface + code: str = Field(min_length=10) + class_name: str = "Transformation" + name: Optional[str] = None + modality: Modality = Modality.timeseries + invertible: bool = False + + provenance: str = "handwritten" # handwritten | evolved | library + method: Optional[str] = None + parent_feature_id: Optional[str] = None + generation: int = 0 + target_task: Optional[str] = None + target_model: Optional[str] = None + dataset: Optional[str] = None + output_type: Optional[str] = None + metrics: List[Metric] = Field(default_factory=list) + columns_added: List[str] = Field(default_factory=list) + columns_dropped: List[str] = Field(default_factory=list) + validity: Dict[str, Any] = Field(default_factory=dict) + tags: List[str] = Field(default_factory=list) + status: Status = Status.active + version: str = "1" + created_by: str = "seed" + created_at: str = Field(default_factory=_now) + + @model_validator(mode="after") + def _invertible_iface(self): + if self.invertible and self.interface != Interface.fit_transform_inverse: + raise ValueError("invertible=True requires interface=fit_transform_inverse") + if "inverse_transform" in self.code and not self.invertible: + # allow, but normalize the flag + object.__setattr__(self, "invertible", True) + return self + + def to_doc(self) -> dict: + d = self.model_dump(mode="json") + d["_id"] = f"feature:{self.feature_id}" + return d + + +def validate_model(doc: dict) -> dict: + return ModelCard(**doc).to_doc() + + +def validate_feature(doc: dict) -> dict: + return FeatureCard(**doc).to_doc() diff --git a/src/servers/tsfm/core/store.py b/src/servers/tsfm/core/store.py new file mode 100644 index 000000000..82a4ad9f1 --- /dev/null +++ b/src/servers/tsfm/core/store.py @@ -0,0 +1,177 @@ +from __future__ import annotations +import os +from typing import Any, Dict, Iterable, List, Optional + + +# --------------------------------------------------------------------------- # +# selector matching (Mango subset, used by MemoryStore and as a fallback) +# --------------------------------------------------------------------------- # +def _match(doc: dict, selector: dict) -> bool: + for field, cond in (selector or {}).items(): + val = doc.get(field) + if isinstance(cond, dict): + for op, operand in cond.items(): + if op == "$gte" and not (val is not None and val >= operand): + return False + elif op == "$lte" and not (val is not None and val <= operand): + return False + elif op == "$gt" and not (val is not None and val > operand): + return False + elif op == "$eq" and not (val == operand): + return False + elif op == "$ne" and not (val != operand): + return False + elif op == "$in" and not (val in operand): + return False + elif op == "$elemMatch": + if not isinstance(val, list) or not any( + _match({"_": v}, {"_": operand}) for v in val + ): + # $elemMatch with a scalar equality operand + if not (isinstance(val, list) and operand in val): + return False + else: + if val != cond: + return False + return True + + +def _strip(doc: dict) -> dict: + d = dict(doc) + d.pop("_rev", None) + return d + + +class Store: + """Backend-agnostic interface.""" + + def get(self, collection: str, doc_id: str) -> Optional[dict]: ... + def put(self, collection: str, doc: dict) -> dict: ... + def find( + self, collection: str, selector: Optional[dict] = None, limit: int = 1000 + ) -> List[dict]: ... + def delete(self, collection: str, doc_id: str) -> bool: ... + def list_collections(self) -> List[str]: ... + def export_state( + self, collections: Optional[Iterable[str]] = None + ) -> Dict[str, List[dict]]: + cols = list(collections) if collections else self.list_collections() + return {c: self.find(c) for c in cols} + + +# --------------------------------------------------------------------------- # +class MemoryStore(Store): + """In-memory store — TEST DOUBLE ONLY (hermetic suite). Production never uses this; the + server runs on CouchStore (CouchDB). Selected only when TSFM_STORE=memory, which the test + conftest sets.""" + + def __init__(self): + self._db: Dict[str, Dict[str, dict]] = {} + + def _col(self, c): + return self._db.setdefault(c, {}) + + def get(self, collection, doc_id): + d = self._col(collection).get(doc_id) + return _strip(d) if d else None + + def put(self, collection, doc): + if "_id" not in doc: + raise ValueError("doc requires _id") + self._col(collection)[doc["_id"]] = dict(doc) + return _strip(doc) + + def find(self, collection, selector=None, limit=1000): + docs = [ + _strip(d) + for d in self._col(collection).values() + if _match(d, selector or {}) + ] + return docs[:limit] + + def delete(self, collection, doc_id): + return self._col(collection).pop(doc_id, None) is not None + + def list_collections(self): + return [c for c, docs in self._db.items() if docs] + + +# --------------------------------------------------------------------------- # +class CouchStore(Store): + """CouchDB backend — same wire format as the AssetOpsBench loader.""" + + def __init__(self, url=None, auth=None): + import requests # lazy + + self._requests = requests + self.url = ( + url or os.environ.get("COUCHDB_URL", "http://localhost:5984") + ).rstrip("/") + self.auth = auth or ( + os.environ.get("COUCHDB_USERNAME", "admin"), + os.environ.get("COUCHDB_PASSWORD", "password"), + ) + + def _u(self, *p): + return "/".join([self.url, *p]) + + def _ensure(self, collection): + r = self._requests.head(self._u(collection), auth=self.auth, timeout=10) + if r.status_code == 404: + self._requests.put(self._u(collection), auth=self.auth, timeout=10) + + def get(self, collection, doc_id): + r = self._requests.get(self._u(collection, doc_id), auth=self.auth, timeout=10) + if r.status_code == 404: + return None + r.raise_for_status() + return _strip(r.json()) + + def put(self, collection, doc): + self._ensure(collection) + ex = self._requests.get( + self._u(collection, doc["_id"]), auth=self.auth, timeout=10 + ) + body = dict(doc) + if ex.status_code == 200: + body["_rev"] = ex.json()["_rev"] + r = self._requests.put( + self._u(collection, doc["_id"]), json=body, auth=self.auth, timeout=15 + ) + r.raise_for_status() + return _strip(doc) + + def find(self, collection, selector=None, limit=1000): + r = self._requests.post( + self._u(collection, "_find"), + json={"selector": selector or {"_id": {"$gt": None}}, "limit": limit}, + auth=self.auth, + timeout=20, + ) + if r.status_code == 404: + return [] + r.raise_for_status() + return [_strip(d) for d in r.json().get("docs", [])] + + def delete(self, collection, doc_id): + ex = self._requests.get(self._u(collection, doc_id), auth=self.auth, timeout=10) + if ex.status_code != 200: + return False + self._requests.delete( + self._u(collection, doc_id), + params={"rev": ex.json()["_rev"]}, + auth=self.auth, + timeout=10, + ) + return True + + def list_collections(self): + r = self._requests.get(self._u("_all_dbs"), auth=self.auth, timeout=10) + r.raise_for_status() + return [d for d in r.json() if not d.startswith("_")] + + +def make_store() -> Store: + """Backend from TSFM_STORE: couch (default, like the other AssetOpsBench servers) | memory + (used for the hermetic test suite).""" + return MemoryStore() if os.environ.get("TSFM_STORE") == "memory" else CouchStore() diff --git a/src/servers/tsfm/core/tasks.py b/src/servers/tsfm/core/tasks.py new file mode 100644 index 000000000..ba31e417a --- /dev/null +++ b/src/servers/tsfm/core/tasks.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Optional + + +@dataclass(frozen=True) +class TSTask: + task_id: str + output_verb: str # predict | score | transform | assign | metrics + output_type: str + eval_protocol: str # backtest | blocked_cv | stratified_blocked_cv | + # mask_and_score | range_pr | retrieval_at_k | silhouette | meta + metrics: List[str] + supervised: bool + selection_signal: ( + str # mase | f1 | auc_pr | em_al | silhouette | recall_at_k | meta + ) + result_collection: str + required_inputs: List[str] # keys a request must supply + requires_inverse: bool = ( + False # transforms must round-trip (output back in input space) + ) + leakage_split: str = "blocked" # blocked | stratified_blocked | none(meta) + notes: str = "" + description: str = "" # plain-language: what this task does (for the agent) + + +TASKS: Dict[str, TSTask] = { + "tsfm_forecasting": TSTask( + "tsfm_forecasting", + "predict", + "forecast", + "backtest", + ["mase", "wql", "smape", "mae"], + True, + "mase", + "forecast_result", + ["series", "horizon"], + requires_inverse=True, + notes="AutoAI-TS pipelines + TSFMs; horizon from the request; invertible scaling/flatten", + description="Predict future values of a series over a horizon.", + ), + "tsfm_regression": TSTask( + "tsfm_regression", + "predict", + "value", + "blocked_cv", + ["r2", "mae", "rmse"], + True, + "r2", + "regression_result", + ["series", "target"], + notes="FLOps feature extraction+selection → regressor (RUL is a variant)", + description="Predict a real-valued target from a series (e.g. remaining useful life).", + ), + "tsfm_classification": TSTask( + "tsfm_classification", + "predict", + "class", + "stratified_blocked_cv", + ["accuracy", "f1", "auroc"], + True, + "f1", + "classification_result", + ["series", "labels"], + leakage_split="stratified_blocked", + notes="catch22/representation features → classifier (PHM fault classification)", + description="Assign a discrete class/label to a series (e.g. fault type).", + ), + "tsfm_anomaly_detection": TSTask( + "tsfm_anomaly_detection", + "score", + "score_label_contribution", + "range_pr", + ["auc_pr", "range_f1"], + False, + "em_al", + "anomaly_result", + ["series"], + notes="AnomalyKiTS pipelines; label-free EM/AL when no GT; static/dynamic threshold", + description="Score/flag abnormal points or ranges in a series.", + ), + "tsfm_imputation": TSTask( + "tsfm_imputation", + "predict", + "filled_series", + "mask_and_score", + ["mae", "crps"], + True, + "mae", + "imputation_result", + ["series", "mask"], + requires_inverse=True, + notes="TSPulse/MOMENT; hide observed points and score reconstruction", + description="Fill missing values in a series.", + ), + "tsfm_evaluation": TSTask( + "tsfm_evaluation", + "metrics", + "metrics", + "meta", + ["mae", "rmse"], + True, + "meta", + "evaluation_result", + ["predictions", "ground_truth"], + leakage_split="none", + notes="meta task: score predictions / benchmark pipelines", + description="Score predictions against ground truth / benchmark pipelines.", + ), + "tsfm_similarity_search": TSTask( + "tsfm_similarity_search", + "transform", + "embedding", + "retrieval_at_k", + ["recall_at_k", "map"], + False, + "recall_at_k", + "similarity_result", + ["series", "query"], + notes="TSPulse embeddings → vector index → top-k", + description="Find the series most similar to a query (embedding + top-k retrieval).", + ), + "tsfm_clustering": TSTask( + "tsfm_clustering", + "assign", + "assignments", + "silhouette", + ["silhouette", "ari"], + False, + "silhouette", + "clustering_result", + ["series_set"], + notes="catch22/embedding features → clusterer; label-free silhouette", + description="Group similar series into clusters (unsupervised).", + ), +} + + +def get_task(task_id: str) -> TSTask: + if task_id not in TASKS: + raise ValueError(f"unknown task '{task_id}'. Known: {list(TASKS)}") + return TASKS[task_id] + + +def list_tasks() -> List[dict]: + return [t.__dict__ for t in TASKS.values()] + + +def validate_request(task_id: str, request: dict) -> dict: + """Check a request supplies the task's required inputs; report leakage/inverse expectations.""" + t = get_task(task_id) + missing = [k for k in t.required_inputs if request.get(k) is None] + return { + "task_id": task_id, + "ok": not missing, + "missing_inputs": missing, + "supervised": t.supervised, + "leakage_split": t.leakage_split, + "requires_inverse_transforms": t.requires_inverse, + "selection_signal": t.selection_signal, + "result_collection": t.result_collection, + } diff --git a/src/servers/tsfm/docs/ALPHAEVOLVE_ALIGNMENT.md b/src/servers/tsfm/docs/ALPHAEVOLVE_ALIGNMENT.md new file mode 100644 index 000000000..ce7d37ace --- /dev/null +++ b/src/servers/tsfm/docs/ALPHAEVOLVE_ALIGNMENT.md @@ -0,0 +1,51 @@ +# AlphaEvolve alignment — and how we support Evolve + +AlphaEvolve (DeepMind, arXiv 2506.13131) is an evolutionary coding agent: an LLM proposes edits +to a *program*, an automatic *evaluator* scores it, and a MAP-Elites / island *database* keeps a +diverse set of elites; the loop repeats. Our server adopts the same machinery, with one +principle preserved — **the server never calls an LLM; the agent is the proposer.** + +## Component mapping + +| AlphaEvolve | TSFM server | Where | +|---|---|---| +| **EVOLVE-BLOCK** (the program) | a **recipe** (declarative compose spec) OR an EFE **feature program** (fit/transform/inverse) | `evolve_tell(kind=...)` | +| **`evaluate(h)`** evaluator | `run_recipe` / `run_tabular_recipe` (→ backtest/CV) and the EFE validity gate → a scalar **fitness** | `engine/evolve._evaluate` | +| **"avoid incorrect suggestions"** | EFE gate (entry points, no-inplace, invertibility round-trip) + recipe-shape check; invalid ⇒ rejected, not archived | `feature_runner`, `_evaluate` | +| **Program database** (MAP-Elites + islands) | archive keyed by **behaviour cell** (task × regime × #components × ensemble; or out_type × invertible) + **islands** for diversity; best-per-cell = elite | `engine/evolve` (`ARCHIVE`) | +| **Prompt sampling** (parents + inspirations + context) | `evolve_ask` samples top parents + diverse inspirations (distinct cells/islands) + `profile_series` evidence + task contract | `evolve_ask` | +| **LLM ensemble** (proposer) | the **agent** — outside the server, by design | (agent) | +| **Evolutionary lineage** | `parent_id` + `generation` per candidate | archive docs | +| **Evaluation cascade** | reuses T-Daub reverse-allocation; cheap configs first (future) | `selector` | + +## The loop (agent-generates / server-grades) +``` +evolve_ask(task[, dataset_path]) → {parents, inspirations, evidence, contract, instructions} + ↓ the AGENT mutates/recombines the parents into ONE new candidate (recipe or feature) +evolve_tell(task, kind, program[, parent_id, dataset_path, …]) + → validate (gate/shape) → evaluate → fitness → MAP-Elites cell/island + lineage + → {accepted, fitness, is_new_elite, cell, generation} +evolve_best(task) → the elites (best program per behaviour cell) +``` +Fitness is always higher-is-better (forecasting: −backtest error; tabular: CV accuracy/r²/ +silhouette; feature: validity + invertibility bonus). Different *structure* lands in a different +cell, so the archive keeps a diverse frontier rather than collapsing to one design. + +## What we evolve +- **Recipes** — which models/transforms/ensemble/params to compose (compose-level search). +- **Feature programs** — EFE `fit/transform/inverse` code the agent writes; gated + archived. + +## Status +Built + tested (`engine/evolve.py`, `tests/test_evolve.py`, 6 tests): seed → grow archive across +cells, elite replacement within a cell, ask returns parents/inspirations/evidence, empty-archive +seed instructions, invalid-candidate rejection, feature-program evolution through the gate. MCP +tools `evolve_ask` / `evolve_tell` / `evolve_best` are live (27 tools total). + +## Gaps vs full AlphaEvolve (future) +- **Evaluation cascade** as an explicit evolve gate (staged cheap→expensive configs). +- **Island migration** (periodic cross-island copying) — islands are stored + used for sampling + diversity, but migration is not yet scheduled. +- **Multi-objective elites** (Pareto front over accuracy × cost/latency) — today fitness is one + scalar per cell. + +Sources: AlphaEvolve — arXiv 2506.13131 (Novikov et al., DeepMind, 2025). diff --git a/src/servers/tsfm/docs/ARCHITECTURE.svg b/src/servers/tsfm/docs/ARCHITECTURE.svg new file mode 100644 index 000000000..1441dd853 --- /dev/null +++ b/src/servers/tsfm/docs/ARCHITECTURE.svg @@ -0,0 +1,111 @@ + +TSFM MCP Server architecture +Inputs as file pointers feed a TSFM MCP server built on sktime: evidence and discovery tools, model and feature stores, a recipe engine that composes the pipeline, GIFT-Eval scoring, eight result tables, and a downstream handoff to FMSR, WO and Spot. + + + + + + +INPUTS · FILE POINTERS → COUCHDB + +iot +asset_id · timestamp · channels + +vibration +high-rate channels + +dataset_path +CMAPSS · CWRU · GIFT-Eval + + + +TSFM MCP SERVER — 16 tools · sktime substrate · the agent composes recipes + +EVIDENCE & DISCOVERY · agent reads the menu, then decides + +profile_series +facts only · no decisions + +list_tasks +8 task contracts + +discover_components +menu + training regime + +find_models · find_features +rank candidates (HuggingGPT) + +REGISTRIES · catalog = pointer index (state-exportable, #394) + +Model Store +cards · zero-shot / fit / fine-tune · local · hf · api-async + +Feature Store +FLOps select (multi-config) · EFE evolve · fit / transform / inverse + + +RECIPE ENGINE · the pipeline is composed, not fixed + +run_recipe +transforms + single / ensemble + conformal + +run_tabular_recipe +FeatureUnion → estimator + +run_plan +DAG · file-pointer chaining +sktime compiles & runs each recipe · agent mixes, scores, iterates · zero-shot skips the retrain loop + + + +evaluate · GIFT-Eval +seasonal-naive-normalized MASE + CRPS · geometric mean · mean-rank leaderboard + + +RESULT TABLES · write_result(task) + +forecasting · regression · classification · anomaly · imputation · clustering · similarity · evaluation +pointer (results CSV) + summary + provenance (model · features · dataset) + param_audit + training_regime + + +register_model · register_feature · finetune → checkpoint @path · agent-decided · weights live where saved + +writes back to catalog +Reasoning split — server gives evidence + grades; the agent decides the recipe and every parameter. +anomaly_result / forecast_result → handoff to downstream · no alerts inside TSFM · export_state() + + +DOWNSTREAM · other MCP servers (consumers) + +FMSR +anomaly → failure code + +WO +SR · work order + +Spot +verify · confirm / false + + + + + +TSFM-owned · all in CouchDB + +downstream consumer (read-only) +accent = the run path +zero-shot default · fine-tune optional + diff --git a/src/servers/tsfm/docs/CORE_CAPABILITY.md b/src/servers/tsfm/docs/CORE_CAPABILITY.md new file mode 100644 index 000000000..7b4dcdcb3 --- /dev/null +++ b/src/servers/tsfm/docs/CORE_CAPABILITY.md @@ -0,0 +1,87 @@ +# Core capability — the agentic composition loop (mix-and-match · ensemble · iterate) + +The defining capability of the TSFM MCP server: the agent **discovers** all components, +**composes** a recipe (mix-and-match, including ensembles), the server **runs/backtests** it on +the sktime substrate and returns **diagnostics**, and the agent **iterates** to a revised +recipe — a closed loop, GIFT-Eval-style ensemble search driven by the agent. Built on sktime, +persisted as state. (`composition.py`, verified end-to-end on real sktime.) + +## The loop (4 MCP tools) + +``` + discover_components ─▶ [agent composes a RECIPE] ─▶ run_recipe ─▶ diagnostics + ▲ │ + └────────────── agent revises (drop/reweight/swap) ◀────────┘ (parent_run_id link) +``` + +1. **`discover_components(task)`** — returns everything composable: installed sktime models for + the task's scitype, the **foundation forecasters** (TTM, Chronos, MOIRAI, TimesFM, MOMENT, + TimeMoE, PatchTST, …), catalog models (incl. fine-tunes), transforms (feature store), + **combiners** (mean/median/min/max/weighted/stack), metrics, splitters. The agent reasons + over this menu. + +2. **The RECIPE** (declarative, the agent authors it): +```jsonc +{ "task": "tsfm_forecasting", "fh": [1..H], + "transforms": [ {"name":"diff","sktime_class":"...Differencer"} ], // optional, auto-inverse + // EITHER a single model … + "estimator": {"model_id":"ttm_512_96"} | {"sktime_class":"...","params":{}}, + // … OR an ensemble (mix-and-match) + "ensemble": {"combine":"mean|median|weighted|stack", + "members":[ {"model_id":"ttm_512_96"}, {"sktime_class":"...ChronosForecaster"}, + {"sktime_class":"...NaiveForecaster","params":{"strategy":"drift"}} ], + "weights":[0.5,0.3,0.2]}, + "eval": {"splitter":"expanding","initial_window":...,"step":...,"metrics":["smape","mase"]} } +``` + +3. **`run_recipe(recipe)`** — compiles to a single sktime forecaster: + - ensemble → `EnsembleForecaster` (aggfunc / weights) or `StackingForecaster` + - transforms → `TransformedTargetForecaster` (forward + **automatic reverse-order inverse**) + then **backtests** via `ExpandingWindowSplitter` + `sktime.evaluate`, fits on full history, + and returns: `backtest_score`, **`per_member_score`** (each ensemble member alone — the key + diagnostic for iteration), `forecast_head`, and a persisted `run_id`. + +4. **Iterate** — the agent reads `per_member_score`, drops the weak member / reweights toward + the strong one / swaps in another foundation model, and calls `run_recipe(..., parent_run_id= + previous)`. Each run is stored in `tsfm_runs` with its parent → the **refinement trajectory + is state** (and `improved` is reported vs the parent). + +## Verified (real sktime, in `test_composition.py` + a live demo) + +- discover lists 50+ installed forecasters + the 11 foundation models + combiners. +- single drift model: sMAPE 0.149. +- 4-member mean ensemble: 0.187, with per-member `{last:0.152, mean:0.662, drift:0.149, + trend:0.210}` — the loop **exposes** that the `mean` member is dragging the ensemble. +- agent iterates (drop `mean`, weight toward `drift`) → **0.150, improved=True**, with the + parent link persisted. This is the agent making the ensemble better, not the server. + +## Why this is the right core capability + +- **It is the agent's reasoning surface.** The agent doesn't call a black-box "forecast"; it + *composes* from a discovered menu and *learns from diagnostics* — exactly the mix-and-match + + feedback you described, and what separates a strong agent from a naive one (benchmark-able). +- **Ensembles are first-class** (GIFT-Eval): combine foundation models + classical + features by + mean/median/weighted/stack — the recipe makes this one line, sktime executes it. +- **Closed-loop refinement** with persisted lineage: round 2 uses round 1's per-member scores; + the trajectory (and whether it improved) is in `tsfm_runs`, so a scenario can score *the + search*, not just the final number. +- **Substrate-clean**: the recipe compiles to sktime; nothing bespoke runs the models. New + foundation model = a new catalog card; it's immediately composable. + +## How it composes the rest of the stack + +- members/estimator resolve through the **model store** (card → sktime class) — fine-tunes and + remote models are composable too. +- transforms resolve through the **feature store** (sktime transformers / EFE-evolved). +- selection: the agent can compose *by hand* (this loop) **or** call `find_models` (T-Daub) to + get a ranked shortlist to ensemble — the two compose. +- results + runs land in CouchDB → `export_state()` scoring. + +## MCP tools to expose +`discover_components`, `run_recipe`, `get_run`, `list_runs` (lineage). With `find_models` +(T-Daub shortlist) and `profile_series` (evidence) these give the agent: see the data → see the +menu → compose → run → diagnose → iterate. + +## Files +`composition.py` (discover/build/run/iterate), `tests/test_composition.py` (3 tests), +`sktime_resolver.py` (card→sktime), `selector.py` (T-Daub shortlist to feed ensembles). diff --git a/src/servers/tsfm/docs/COUCHDB_LOADING.md b/src/servers/tsfm/docs/COUCHDB_LOADING.md new file mode 100644 index 000000000..31bda3660 --- /dev/null +++ b/src/servers/tsfm/docs/COUCHDB_LOADING.md @@ -0,0 +1,75 @@ +# Loading the catalogs into CouchDB + +The model & feature catalogs load into CouchDB through the **standard AssetOpsBench loader** +(`src/couchdb/loader.py` + `collections.json` + scenario manifests) — exactly like `iot`, `wo`, +and `vibration`. No TSFM-specific loading code. The server then reads those two databases via its +CouchDB backend (`TSFM_STORE=couch`, the default). + +The data files are the curated catalogs, kept with the other shared CouchDB collection data: +`src/couchdb/scenarios_data/shared/tsfm/model_catalog.json` (48 docs) and `feature_catalog.json` +(115 docs) — each doc already carries an `_id` (`model:` / `feature:`). +This is the single source of truth; the package no longer ships its own `seeds/` copy, and +`bootstrap.load_seeds` reads from here (override with `$TSFM_SEEDS_DIR`). + +## 1. Register the two collections — `src/couchdb/collections.json` +Add (describes how each is keyed + indexed; `_id` is `id_prefix:primary_key`, which already +matches our docs so they're preserved verbatim): + +```jsonc +"model_catalog": { + "format": "json", + "primary_key": ["model_id"], + "id_prefix": "model", + "indexes": [["task_ids"], ["status"], ["model_family"], ["domain"]] +}, +"feature_catalog": { + "format": "json", + "primary_key": ["feature_id"], + "id_prefix": "feature", + "indexes": [["scenario_categories"], ["kind"], ["status"]] +} +``` +(The `task_ids` / `scenario_categories` indexes back the `$elemMatch` filters that +`find_models` / `find_features` use.) + +## 2. Point a manifest at the data files +Make the catalog JSONs reachable by the loader (it resolves a manifest path against the manifest +folder → its parent → the couchdb dir). Drop copies under the shared corpus, e.g. +`src/couchdb/shared/tsfm/model_catalog.json` and `feature_catalog.json`, then add to +`scenarios_data/default/manifest.json`: + +```json +{ + "model_catalog": "shared/tsfm/model_catalog.json", + "feature_catalog": "shared/tsfm/feature_catalog.json" +} +``` + +(db name = the manifest key, so this creates the `model_catalog` and `feature_catalog` +databases.) + +## 3. Load +Already wired into the CouchDB container (`couchdb_setup.sh` runs `init_data.py`). Manually: + +```bash +COUCHDB_URL=http://localhost:5984 python -m couchdb.init_data # default manifest +COUCHDB_URL=http://localhost:5984 python -m couchdb.init_data 304 # a scenario's manifest +``` + +## Per-scenario catalogs (the "models as a variable" requirement) +A scenario folder `scenarios_data/scenario_/manifest.json` can list **only the catalog that +scenario should see** — a subset of models, a different feature set, or none: + +```json +{ "iot": "...", "model_catalog": "scenario_42/models_subset.json" } +``` +`init_data(42)` then loads exactly that catalog into CouchDB; the TSFM server reads it unchanged. +This is how the catalog varies scenario-to-scenario, with zero server changes. + +## What the server does +`make_store()` → `CouchStore` (default) reads `model_catalog` / `feature_catalog`. The discovery +tools (`find_models`, `find_features`, `describe_candidates`, `get_component`) query those DBs; +write-backs (`register_model`, `register_feature`, evolve) persist there. (`fresh_store` also +self-seeds an empty DB from the packaged seeds as a dev convenience; once the loader has run it's +a no-op.) +``` diff --git a/src/servers/tsfm/docs/DESIGN.md b/src/servers/tsfm/docs/DESIGN.md new file mode 100644 index 000000000..e8c8b48c6 --- /dev/null +++ b/src/servers/tsfm/docs/DESIGN.md @@ -0,0 +1,265 @@ +> 🟡 **HISTORICAL (still mostly valid).** The unified 8-task taxonomy, T-Daub selection, and +> GIFT-Eval framing here all hold. The one update: the estimator/transform **contract is now +> sktime** (not the custom algebra in §1) — see `SKTIME_NATIVE_DESIGN.md` and `STORES.md` for the +> current substrate. Read this for the task taxonomy + selection rationale; read STORES.md for +> the implemented stores. + +# TSFM-Kit — a unified MCP server for time-series AI tasks + +*A from-scratch synthesis of four IBM systems — AnomalyKiTS (AAAI'22), AutoAI-TS (SIGMOD'21), +FLOps (BigData'20), and Evolutionary Feature Engineering (NeurIPS'26) — into one rigorous, +agent-operable building block that standardizes the full family of TS AI tasks.* + +--- + +## 0. Thesis + +Every time-series AI task — forecasting, regression, classification, anomaly detection, +imputation, evaluation, similarity search, clustering — is the **same object**: + +``` + Pipeline = T_k ∘ … ∘ T_1 → E → T_1⁻¹ ∘ … ∘ T_k⁻¹ (transforms, estimator, inverse) +``` + +a (possibly invertible) composition of **transforms** feeding an **estimator**, evaluated by a +**task protocol**, and selected under a **compute budget**. The four papers each own one layer +of this object; the MCP server makes the object **agent-operable** and the choices +**benchmark-scorable**. Standardizing on this algebra — rather than a bag of bespoke tools — is +what makes the server strong and extensible. + +| Layer of the Pipeline object | Paper | Contribution used | +|---|---|---| +| The transforms `T_i` (with `T_i⁻¹`) | **AutoAI-TS** | Difference / Flatten / Localized / Normalized (T2R), inverse-in-reverse | +| Expanding the transform space | **EFE** | LLM-evolved `fit/transform/inverse` programs, selected by downstream metric | +| Choosing transforms per dataset | **FLOps** | importance scoring vs a Reference Feature + Critical-Difference filter; auto look-back | +| The estimator `E` + std. output + label-free ranking | **AnomalyKiTS** | sklearn Operators/Pipelines; score+label(+1/−1)+contribution; EM/AL ranking | +| Selecting `E`/Pipeline under budget | **AutoAI-TS** | **T-Daub** reverse-progressive data allocation; backtest; zero-conf | + +Everything else (model store, feature store, result tables, reasoning, eval-by-state) is the +agentic + persistence machinery built around this object. + +--- + +## 1. The two contracts (everything reduces to these) + +**Transform** (feature store unit) — sklearn-compatible, optionally invertible: +``` +fit(X, y=None, meta) -> state +transform(X, state) -> X' +inverse_transform(X', state) -> X # required iff the transform is invertible +``` +Kinds: `scaler` (Normalized), `stationarizer` (Difference), `windowizer` (Flatten/T2R/lookback), +`extractor` (FLOps scalar features), `selector` (FLOps/channel), `learned` (EFE-evolved). + +**Estimator** (model store unit) — sklearn-compatible, capability-tagged: +``` +fit(X, y=None, meta) -> self # zero-shot models are a no-op fit +predict(X) -> Y | score(X) -> per-step anomaly score +transform(X) -> embedding | assign(X) -> cluster id +capabilities: [task_id...] family: stat | ml | hybrid | dl | tsfm | ad_pipeline | encoder +``` +Both share the **state/lineage/validity envelope** (provenance, version, parent, metrics, +EFE validity checks). A Pipeline is just `[Transform…] + Estimator`, and AutoAI-TS's invertible +transform stack and AnomalyKiTS's pipelines are *instances* of it. + +--- + +## 2. Standardized TS-AI task taxonomy + +A `TSTask` spec declares the contract once; tasks differ only in `inputs`, `output_type`, +`eval_protocol`, `selection`, and `supervised`. This is the standardization the server enforces. + +| task_id | inputs | output_type | eval protocol | selection signal | result table | +|---|---|---|---|---|---| +| **forecasting** | X(T×C), horizon H, [exog] | forecast (+quantiles) | rolling **backtest** | MASE / WQL / sMAPE (T-Daub) | forecast_result | +| **regression** | windowed X → y | scalar/vector ŷ | holdout / blocked CV | R²/MAE (FLOps features) | regression_result | +| **classification** | windowed X → class | label + proba | stratified blocked CV | F1 / AUROC | classification_result | +| **anomaly_detection** | X(T×C) | score + label(±1) + contribution | range-based PR/AUROC **or label-free** | AUC-PR / **EM·AL** | anomaly_result | +| **imputation** | X + mask | filled X | **mask-and-score** held-out observed | MAE/CRPS on masked | imputation_result | +| **evaluation** | ŷ, y (or pipeline+data) | metrics | backtest / holdout | n/a (meta) | evaluation_result | +| **similarity_search** | X → embedding; query | top-k neighbors | retrieval P@k / MAP | recall@k | similarity_result | +| **clustering** | {X_i} | assignments | silhouette / ARI | silhouette (label-free) | clustering_result | + +Specializations folded in, not duplicated: **prognostics/RUL** = regression with a degradation +target + monotonicity-aware metric; **forecasting_finetune** = forecasting + a write-back of +the trained estimator. Every row is one `TSTask` object → uniform tools, uniform scoring. + +Two structural facts a TS researcher will check, handled explicitly: +- **Temporal leakage**: all splits are *blocked/sequential* (no shuffling); transforms `fit` on + train only; T-Daub uses most-recent slices. Encoded in `eval_protocol`, enforced by the runner. +- **Supervised vs unsupervised**: `supervised=False` tasks (anomaly, clustering, similarity) use + label-free selection (EM/AL score, silhouette) — AnomalyKiTS's contribution — so selection + works without ground truth. + +--- + +## 3. Building block I — Feature / Transform Store + +Three operations, one library: +- **provide** — `find_features(category, task, modality)`: AutoAI-TS canonical transforms + + AnomalyKiTS operators + seeds, all under the Transform contract. +- **generate** — `register_feature(program)` (EFE): LLM-evolved transforms, **validity-gated** + (entry points, no in-place mutation, invertibility round-trip, no leakage) and lineage-tracked + (`parent_feature_id`, `generation`). +- **select** — `select_features(series, …)` (FLOps): score each candidate extractor on *this* + dataset, rank vs a **Reference Feature**, keep those past the **Critical-Difference** margin; + write importance back to the catalog (the catalog *learns* per domain). + +Invertibility is first-class (AutoAI-TS/EFE): a forecasting pipeline applies scaler→difference→ +flatten forward and the exact inverse stack to map predictions back to physical units. The +store records `interface ∈ {fit_transform, fit_transform_inverse}` and verifies the round-trip +on register. + +## 4. Building block II — Model / Estimator Store + +A **pointer catalog** of estimators/pipelines spanning AutoAI-TS families (stat/ML/hybrid/DL), +TSFMs (TTM, TSPulse, Chronos, MOMENT, Moirai, TimesFM), AnomalyKiTS AD pipelines (DeepAD/ +RelationshipAD/ReconstructAD/WindowAD), encoders (TSPulse embeddings), and registered +fine-tunes. Each card: capabilities (`task_ids`), family, modality, context_length, +input_spec, source (local/hf/toolkit/api), provenance + `base_model_id` lineage, metrics. + +Selection is **T-Daub** (AutoAI-TS), not a hand-rank: see §5. Fine-tunes are **agent-decided** +write-backs that point the catalog at a checkpoint; ephemeral tunes never get registered. + +## 5. Selection & search — T-Daub + label-free ranking + +For supervised tasks, choosing among many candidate pipelines naively (train all on full data) +is infeasible. **T-Daub (reverse progressive data allocation)**: +1. **Fixed allocation** — train each pipeline on increasing *recent* slices `T1[L−mΔ:L]`, score + on holdout `T2`; **fit a linear learning curve and project the score at full length L**; rank. +2. **Acceleration** — give geometrically increasing data only to the **top** pipeline(s); + re-project, re-rank. +3. **Score** — return ranked pipelines; train the winner to completion. + +This makes "search 10³ catalog models" tractable and is the principled answer to model +selection under budget (sibling of successive-halving/Hyperband, but for sequential TS with a +learning-curve projection on recent data). For **unsupervised** tasks, rank by AnomalyKiTS +**EM/AL** scores or **silhouette** — no labels needed. Conformal calibration (AnomalyKiTS) +sets anomaly thresholds (static otsu / dynamic). + +## 6. Evaluation protocols (per task, leakage-safe) + +`backtest` (rolling-origin) for forecasting; `blocked_cv` for regression/classification; +`mask_and_score` for imputation (hide observed points, score reconstruction); `range_pr` / +label-free for anomaly; `retrieval@k` for similarity; `silhouette/ARI` for clustering; +`meta` for evaluation. Each `TSTask` carries its protocol + metric set; the runner refuses a +non-blocked split for sequential data. + +## 7. Reasoning ownership — agent decides; server is evidence + grader + +The server does **not** pre-decide lookback/context/horizon/pipeline/threshold (that would make +the benchmark grade the server, not the agent). Instead: +- **evidence tools** (`profile_series`: detrended dominant period, stationarity, channel + correlation, length; `available_contexts`, `available_features`) — facts, no advice; +- the **agent** chooses every parameter (tools require them; no silent defaults); +- the server records a factual **param_audit** (e.g. context≥lookback) into the result, and the + **benchmark** grades the choices vs hidden references (`scoring.py`): lookback ∈ [1×,3×] + period, context ≥ lookback, horizon == requested, AD pipeline fits channel structure, + thresholding == dynamic iff non-stationary. (Auto look-back from AutoAI-TS exists as an + *optional advisor / ablation*, not the default path.) + +## 8. Results & state — eval-by-snapshot + +Every registry + result table is a CouchDB collection → captured by `export_state()` (#394 / +PR #400). A scenario is scored on the **end-state**: right task object instantiated, right +transforms/model selected (and *why* — the audited parameters), correct result row vs the +utterance's `characteristic_form`, fine-tune registered with correct lineage only when +warranted. This grades the *decision chain*, not just the final number. + +## 9. MCP tool surface (grouped) + +evidence: `profile_series`, `available_contexts`, `available_features` +feature store: `find_features`, `get_feature`, `select_features`, `register_feature`, `update/deprecate` +model store: `find_models` (T-Daub), `get_model`, `register_model`, lineage/search, `update/deprecate` +run (per task, explicit params): `run_forecast`, `run_regression`, `run_classification`, + `run_anomaly`, `run_imputation`, `run_similarity`, `run_clustering`, `evaluate` +results: `get_result`, `list_results` +(All on one `FastMCP("tsfm")`; no read-only split; downstream alert/SR/WO/verification live on FMSR/WO/Spot.) + +## 10. Package structure + +``` +tsfm/ + store.py core registry (Memory|Couch) + export_state + schemas.py ModelCard / FeatureCard (pydantic, validated) + task_spec.py the TSTask registry — 8 standardized tasks (the standardization) + transforms/ Transform contract + AnomalyKiTS ops + AutoAI-TS transforms + EFE runner + feature_store.py provide / generate(EFE) / select(FLOps) + extractor library + feature_selection.py FLOps scorer + auto look-back + model_store.py estimator catalog + lineage + register + selector.py T-Daub reverse progressive allocation (+ label-free EM/AL/silhouette) + estimators/ StubCompute + RealCompute (tsfm_public, AnomalyKiTS) by family + pipeline.py Transform*→Estimator→Inverse* execution per TSTask + profile.py evidence tools (no decisions) + scoring.py reasoning rubric + param_audit (benchmark-only) + results.py per-task result tables + runner.py availability (local/hf/toolkit/api) + leakage-safe splits + tools.py MCP tool registration + seeds/ tests/ +``` + +--- + +## 11. Self-critique → refinements (the review cycles) + +**C1 — "Is forecasting really the same object as clustering?"** Risk: forcing 8 tasks into one +abstraction becomes a leaky generalization. *Refinement:* the shared object is the **Pipeline +(transform→estimator)**; tasks differ only in the estimator's *output verb* +(`predict/score/transform/assign`) and the **protocol**. The `TSTask` spec carries those two +differences explicitly, so the abstraction is honest, not forced. (Validated in `task_spec.py`.) + +**C2 — "Invertibility doesn't apply to classification/anomaly."** *Refinement:* invertibility is +a per-transform property (`fit_transform_inverse`), required only when the task maps back to the +input space (forecasting, imputation). Classification/anomaly use forward-only transforms. +The contract makes inverse *optional*; the validity gate only enforces the round-trip when +declared. No over-reach. + +**C3 — "T-Daub is forecasting-specific."** *Refinement:* T-Daub's mechanism is *learning-curve +projection under data allocation* — task-agnostic given a per-task score and a blocked split. We +reuse it for regression/classification (CV score) and gate it off for label-free tasks (use +EM/AL/silhouette instead). Selection is thus uniform but signal-appropriate. + +**C4 — "Server-side reasoning kills the benchmark."** Caught earlier; *refinement:* moved all +parameter decisions to the agent, server provides evidence + grades (§7). The AutoAI-TS +auto-look-back becomes an *advisor/ablation*, not the default. + +**C5 — "Leakage."** *Refinement:* blocked/sequential splits everywhere; transforms fit on train +only; T-Daub on recent slices; runner rejects shuffled CV for TS. Encoded in `eval_protocol`. + +**C6 — "Scale of the catalog vs availability."** *Refinement:* catalog is a pointer index; +availability (local/hf/toolkit/api) resolved at call time; unavailable models are still +selectable/plannable but fail compute cleanly — so the catalog can be huge while few are installed. + +**C7 — "Naive T-Daub mis-ranks."** Building it surfaced a real bug: linearly *extrapolating* an +exponential learning curve overshoots, and steeper curves (worse pipelines) get the most +optimistic projections, so the true best never enters the accelerated set (verified: it picked +a worse pipeline). *Refinement:* rank the fixed phase by the score at the **largest observed +allocation** (most reliable), accelerate the top-K to full length, and finalize on the +**measured-at-L** score; keep the linear projection only as a reported auxiliary. Now it selects +the true best at ~0.65× the train-all budget on 24 candidates (the saving grows with the catalog +size). This is the realistic regime for T-Daub — the win is large precisely when there are many +pipelines, which is the catalog case. + +## 12. Literature map (grounding, beyond the four) + +TSFMs: TTM, TSPulse, Chronos, MOMENT, Moirai, TimesFM (zero-shot estimators). +Feature libraries for FLOps: tsfresh, tsfel, catch22, STUMPY. +AD pipelines for AnomalyKiTS: DeepAD, TadGAN, DAGMM, graphical-lasso, conformal AD; benchmarks +TSB-AD, range-based PR (Wu & Keogh's critique → we use range-aware metrics). +Selection lineage for T-Daub: successive halving / Hyperband (we add learning-curve projection +on recent data for sequential TS). +Transform generation for EFE: AlphaEvolve / OpenEvolve (LLM program search). + +## 13. Novelty (what makes this *the* TSFM MCP server) + +1. **One Pipeline algebra** unifies 8 TS-AI tasks behind a single contract — not a tool zoo. +2. **Feature store = generate(EFE) + select(FLOps) + canonical(AutoAI-TS) operators** under one + invertible transform contract. +3. **Model store with T-Daub budget-aware selection** over a pointer catalog of TSFMs + + classical + AD pipelines + fine-tunes. +4. **Agent owns the reasoning** (lookback/context/horizon/pipeline/threshold); server is an + honest evidence-broker and an objective grader → a genuine test of TS reasoning. +5. **Eval-by-state**: the whole decision chain is CouchDB state, scored deterministically. + +This is the standardization a TS-AI agent benchmark has been missing: rigorous enough to be +correct, general enough to cover the field, and operable + scorable as an MCP server. diff --git a/src/servers/tsfm/docs/DOCS_INDEX.md b/src/servers/tsfm/docs/DOCS_INDEX.md new file mode 100644 index 000000000..6836e76d0 --- /dev/null +++ b/src/servers/tsfm/docs/DOCS_INDEX.md @@ -0,0 +1,89 @@ +# TSFM MCP Server — documentation index & current design (canonical) + +The design evolved through several critique cycles. This index is the **single source of +truth** for what's current vs superseded, plus the canonical 12-line architecture. Read it +first. + +## Current architecture (12 lines) + +1. **Substrate = sktime.** Every model/transform is an sktime estimator (or sklearn/PyOD via + adapter, or custom `Base*`); sktime downloads weights. (`sktime_resolver.py`) +2. **Core = a CouchDB/Memory store** (`store.py`) holding cards + result/run tables; + `export_state()` snapshots everything for scoring (#394). +3. **Model store** = sktime-card catalog (superset of the registry): find/select/register/ + lineage. (`model_store.py`, `schemas.py`) — see **STORES.md**. +4. **Feature store** = transforms (sktime/EFE) + FLOps extractors: provide/generate/select. + (`feature_store.py`, `feature_selection.py`, `feature_runner.py`) — see **STORES.md**. +5. **8 standardized TS-AI tasks** bound to sktime scitypes. (`task_spec.py`) +6. **Data by file pointer.** Agent downloads a window → CSV → `data_ref`; never inline. + (`io_refs.py`) +7. **Recipe** = an sktime-compiled spec (transforms + single/ensemble + conformal + eval). + (`composition.py`) +8. **Recipe DAG** = HuggingGPT task-list: steps with `dep` + `@resource` file-pointer chaining. + (`plan.py`) +9. **Selection (plural):** description/HuggingGPT (`describe_candidates`), tags (sktime), + budget T-Daub (`selector.py`). +10. **Scoring = GIFT-Eval**: seasonal-naive-normalized MASE+CRPS, geo-mean, mean-rank + leaderboard. (`gifteval.py`) +11. **Reasoning is the agent's**: server gives evidence (`profile.py`), agent decides, server + grades (`scoring.py`). (`REASONING_OWNERSHIP.md`) +12. **Agent loop:** discover candidates → compose recipe/DAG (file pointers) → run on sktime → + GIFT-Eval score → iterate (lineage persisted). + +## Document status + +| Doc | Status | What it is | +|---|---|---| +| **STORES.md** | ✅ authoritative | the definitive Model + Feature store spec (read this) | +| **DOCS_INDEX.md** | ✅ authoritative | this file — current design + status map | +| **SKTIME_NATIVE_DESIGN.md** | ✅ authoritative | the sktime-substrate pivot (the decision) | +| **CORE_CAPABILITY.md** | ✅ authoritative | the agentic compose→run→diagnose→iterate loop | +| **GIFTEVAL_APPROACH.md** | ✅ authoritative | evaluation backbone (MASE/CRPS, seasonal-naive norm, rank) | +| **HUGGINGGPT_FILEPOINTER_DESIGN.md** | ✅ authoritative | recipe DAG + file pointers + candidate selection | +| **CAPABILITY_MATRIX.md** | ✅ authoritative | conformal / sklearn-PyOD / TSPulse coverage check (verified) | +| **REASONING_OWNERSHIP.md** | ✅ authoritative | agent reasons; server = evidence + grader | +| **README.md** | ✅ current | quick start + module map (sktime-native) | +| **DESIGN.md** | 🟡 historical | the unified 8-task abstraction + T-Daub; correct, but the + estimator/transform contract is now sktime (see SKTIME_NATIVE_DESIGN). Task taxonomy + T-Daub + + GIFT-Eval framing still hold. | +| **BUILDING_BLOCK.md** | ⛔ superseded | the custom typed Operator algebra. **sktime base + classes replace it.** Kept for the *ideas* (leakage-safe fit/transform, invertibility, + quality-gate, zero-model) — all now realized via sktime. | +| **TOOLS_ARE_COMPLEX.md** | 🟡 historical | the parameter-reasoning argument; superseded in + *ownership* by REASONING_OWNERSHIP (agent decides, server grades), but the decision-space + tables are still useful. | + +## Superseded / dropped (do not build) +- **`operators.py`** (custom algebra) → sktime base classes. **`planner.py`** server-side + parameter planner → demoted to optional advisor (agent reasons; `scoring.py` grades). +- **External AI4Industry AD service** → in-toolkit (AnomalyKiTS detectors + TSPulse + PyOD), + all in sktime. +- **Inline data** → file pointers (`io_refs.py`). + +## Module → doc map +- stores: `model_store.py`,`feature_store.py`,`schemas.py` → STORES.md +- substrate: `sktime_resolver.py`,`store.py` → SKTIME_NATIVE_DESIGN.md +- recipe/loop: `composition.py`,`plan.py`,`io_refs.py` → CORE_CAPABILITY / HUGGINGGPT_FILEPOINTER +- features: `feature_selection.py`(FLOps),`feature_runner.py`(EFE) → STORES.md +- selection: `selector.py`(T-Daub) → DESIGN.md §5 / STORES §1.3 +- eval: `gifteval.py` → GIFTEVAL_APPROACH.md +- reasoning: `profile.py`,`scoring.py` → REASONING_OWNERSHIP.md +- tasks: `task_spec.py` → DESIGN.md §2 / STORES +- (legacy: `operators.py`,`runner.py`,`compute.py`,`pipeline.py`,`io.py` — pre-sktime; keep + `runner.py` availability logic, fold the rest into the sktime path) + +## Provenance — the four papers, mapped +- **AutoAI-TS**: zero-model, invertible transform stacks, **T-Daub** selection → `selector.py`, + sktime `TransformedTargetForecaster` / `NaiveForecaster`. +- **AnomalyKiTS**: AD pipelines + Static/Dynamic thresholding + EM/AL → sktime `detector`s. +- **FLOps**: typed extractor library + reference-feature/CD selection → `feature_selection.py`. +- **EFE**: evolved fit/transform programs, validity-gated, archived → `feature_runner.py` + + feature store lineage. +- **HuggingGPT**: task-DAG with resource (file-pointer) dependencies + model selection by + description → `plan.py` + `model_store.describe_candidates`. +- **GIFT-Eval**: seasonal-naive-normalized MASE/CRPS + rank aggregation → `gifteval.py`. + +## Test status +38+ tests green across: stores, schemas, FLOps select, EFE validity gate, task_spec, T-Daub, +composition loop (ensemble + iterate), GIFT-Eval leaderboard, conformal/PyOD/TSPulse resolution, +file-pointer DAG, candidate ranking. diff --git a/src/servers/tsfm/docs/FLOPS_PLAN.md b/src/servers/tsfm/docs/FLOPS_PLAN.md new file mode 100644 index 000000000..46530f138 --- /dev/null +++ b/src/servers/tsfm/docs/FLOPS_PLAN.md @@ -0,0 +1,107 @@ +# Feature extraction (FLOps) — the plan, on sktime + +FLOps = **extract → score → rank vs a Reference Feature → Critical-Difference filter → reuse**, +done dynamically per dataset. On the sktime substrate almost every piece already exists; our +job is the *selection/learning* layer + persistence + the agent loop. (Verified: sktime +extractors + `FeatureUnion` run; our `feature_selection.select_features` picks among the library.) + +## 1. The FLOps pipeline (5 stages → sktime mapping) + +| FLOps stage | What it does | sktime / our piece | +|---|---|---| +| **Library** | 130+ extractors (scalar/vector/functional), grouped (Data Profiling, Data Quality, temporal, frequency) | `Catch22`, `TSFreshFeatureExtractor`, `SummaryTransformer`, `FourierFeatures`, `WindowSummarizer`, `Signature*`, `(Mini)Rocket` + our FLOps refs (`feature_selection.EXTRACTORS`) + EFE-evolved | +| **Extract** | apply the chosen extractors, concatenate | `FeatureUnion([... ])` (compose many) → tabular | +| **Score** | per-feature importance, multiple configs (F-test / MI / model importance) | sktime `FeatureSelection` (filter/wrapper) + `TSFreshRelevantFeatureExtractor` (built-in hypothesis-test relevance) + our `select_features` | +| **Rank + filter** | rank vs a **Reference Feature**, keep those past the **Critical-Difference** margin | our `feature_selection` (reference + CD) — the FLOps signature step | +| **Reuse** | persist the selected set; reapply on new data | a fitted **feature_set card** = `FeatureUnion(selected) → ColumnSelect`, lineage-tracked | + +## 2. Dynamic, dataset-specific selection (the core FLOps idea) + +`select_features(series, reference_feature, cd_margin)` (built, tested): +1. discover **look-back** from spectral period (FLOps: `lw ≈ 1.25 × seasonal length`), +2. tabulate, score each candidate extractor on *this* dataset, +3. rank vs the Reference Feature, keep those beating it by the CD margin, +4. (`select_features_from_catalog(..., write_back=True)`) write the importance back onto the + extractor cards so the catalog **learns** which features matter per domain. + +So the feature set is chosen *for the data in front of the agent*, not fixed. + +## 3. Where extracted features go, per task (data-shape aware) + +- **classification / regression / clustering / similarity** (series → tabular): extraction is + the *main* transform → `Tabularizer`/`FeatureUnion` → the estimator (or via `make_reduction`). + RUL/PHM regression is exactly this (FLOps' original setting). +- **forecasting**: FLOps features become **exogenous regressors** `X` (or reduction features) + alongside the look-back windowizer — sktime `ForecastingPipeline` / `make_reduction`. +- **anomaly**: residual / spectral / trend features feed the detector (AnomalyKiTS pipelines). + +`output_cardinality` on each FeatureCard (series / scalar / vector / functional) tells the +recipe compiler how the shape changes, so composition is type-checked. + +## 4. In the recipe (agent mixes-and-matches features too) + +```jsonc +{ "task":"tsfm_regression", + "transforms":[ {"feature_id":"channel_select_v1","params":{"channel_indices":[0,1,2]}}, + {"sktime_class":"sktime.transformations.panel.catch22.Catch22"}, + {"feature_id":"flops_selected_set@chiller_6"} ], // a persisted FLOps set + "estimator":{"sktime_class":"...RandomForestRegressor via reduction"}, + "eval":{"metrics":["r2","mae"]} } +``` +The agent: `find_features(category)` → `select_features(data_ref)` (FLOps) → compose the chosen +extractors into the recipe → `run_recipe` (sktime) → CV / GIFT-Eval score → iterate. + +## 5. EFE generation when the library isn't enough + +If no library extractor scores above the reference, the agent (EFE) **generates** a new +`fit/transform` extractor (custom sktime `BaseTransformer`), which is validity-gated +(entry points, no in-place, no leakage), archived with lineage (`parent_feature_id`, +`generation`), and re-entered into selection. Library-first, generate-on-demand. + +## 6. Parameter reasoning for extractors (same as models) + +Extractors have parameters too (`window_size`, `n_jobs`, `Catch22` feature subset, `FourierFeatures` +sp_list/fourier_terms). Each FeatureCard carries a `param_schema` (introspected) + hints +(`window_size ≈ 1× dominant_period`, `sp_list = [dominant_period]`); the agent reasons them from +`profile_series`; `validate_params` checks them. (Same machinery as `param_space.py`.) + +## 7. Output: a reusable, scored, lineage-tracked feature set + +A FLOps run yields a **feature_set card** (`kind=feature_set`): the selected extractors + +`ColumnSelect`, with `metrics:[{flops_importance}, {cv_gain_vs_baseline}]`, `dataset`, +`scenario_categories`, lineage. It is registered (validity-gated), discoverable by +`find_features`, and re-applicable to new windows — so selection is *amortized*, not redone +every call. + +## 8. Build steps + +1. **Seed the extractor catalog** from sktime feature transformers (Catch22, TSFresh, Summary, + Fourier, WindowSummarizer, Rocket) + the FLOps library refs — `register_extractor_library` + (done for the FLOps refs; add the sktime classes as cards). *[small]* +2. ✅ **FLOps selector v2** (DONE): multi-config scoring (|corr| + F-test + mutual-info + + model-importance) aggregated by **mean rank**, on top of reference + CD. `select_features` + takes `scorers=[…]` (default all four; `["corr"]` = fast path) and returns `per_scorer` + detail. 4 tests. *(next: wire sktime `TSFreshRelevant`/`FeatureSelection` as extra scorers.)* +3. **Persist feature sets** (`kind=feature_set`) with importance + CV gain + lineage; expose via + `find_features`. *[small]* +4. ✅ **Recipe compile for extraction — tabular** (DONE): `run_tabular_recipe` compiles a + FeatureUnion of extractors (our dependency-free library / sktime panel transformers / + `flops_select` column selection) → estimator for regression / classification / clustering; + CV-scored (accuracy/r2) or silhouette; same recipe grammar + persistence + regime as + forecasting. 5 tests. *(next: exogenous-feature path for forecasting.)* +5. **Param reasoning for extractors** (param_space hints for window_size/sp_list/etc.). *[small]* +6. **EFE generate-on-demand** when library < reference. *[later]* + +## 9. Why this is strong +- The extractor **library is sktime's** (battle-tested: Catch22/TSFresh/Rocket) — we don't + reimplement features; we **select** them (FLOps) and **generate** new ones (EFE). +- Selection is **dynamic + dataset-specific + principled** (reference + CD), **persisted** + (amortized), and **graded** (importance + CV/GIFT-Eval gain) — the full FLOps contribution, + agent-operable. +- It composes into the same recipe/DAG and is scored the same way (GIFT-Eval / CV), so feature + engineering is part of the agent's mix-and-match search, not a separate pipeline. + +## Files +`feature_selection.py` (FLOps selector + look-back), `feature_store.py` (catalog + +register_extractor_library + select_features_from_catalog), `feature_runner.py` (EFE gate), +`param_space.py` (extractor params). Design: `STORES.md` §2, `DESIGN.md` (FLOps mapping). diff --git a/src/servers/tsfm/docs/FORECASTING_WORKFLOW.md b/src/servers/tsfm/docs/FORECASTING_WORKFLOW.md new file mode 100644 index 000000000..191e9a7bb --- /dev/null +++ b/src/servers/tsfm/docs/FORECASTING_WORKFLOW.md @@ -0,0 +1,78 @@ +# Forecasting end-to-end (the run_tsfm_forecasting replacement) + +The old `run_tsfm_forecasting` tool is gone (the whole `legacy/` server was removed). Forecasting +is a **`run_recipe` call with a forecaster card** — one expression of the general recipe engine, +composable (ensemble, conformal, swap models) and consistent with every other task. + +## The end-to-end workflow +``` +CouchDB (iot DB) + │ IoT server: history(site, asset, start, final) ← time-series DATA lives in CouchDB + ▼ +local storage ──► /shared/iot/chiller6.csv ← IoT materialises it to a local file + │ agent passes the path + ▼ +TSFM.run_recipe(dataset_path=…, timestamp_column=…, target_columns=[…], + recipe={estimator:{model_id:"ttm_96_28"}, fh:[1..H]}) + │ resolve TTM card (sktime TinyTimeMixerForecaster) → zero-shot predict (no training) + ▼ +results_file (file pointer) ──► handoff downstream (FMSR → WO → Spot) +``` + +**Responsibility split** +- **time-series data**: IoT owns it — CouchDB → local file. TSFM never reads the data from CouchDB. +- **catalog (models/features)**: TSFM reads from CouchDB (`model_catalog`, `feature_catalog`). +- **forecast on a file pointer → results_file**: TSFM `run_recipe`. +- **chaining IoT→TSFM→downstream**: the agent (or `run_plan` within one server). + +## The forecast call +```jsonc +run_recipe( + dataset_path = "/shared/iot/chiller6.csv", // file pointer (IoT-materialised) + timestamp_column = "timestamp", + target_columns = ["temp"], + recipe = { "estimator": {"model_id": "ttm_96_28"}, "fh": [1, …, 28] } +) +→ RecipeResult{ run_id, results_file, metric, backtest_score, training_regime:"zero_shot" } +``` +`ttm_96_28` resolves to `sktime.forecasting.ttm.TinyTimeMixerForecaster` with +`model_path=ibm-granite/granite-timeseries-ttm-r2` — **sktime wraps the `tsfm_public` TTM**, run +through the uniform fit/predict interface and the file-pointer contract. + +Swap the card to forecast with anything in the catalog (Chronos, MOIRAI, AutoARIMA, an ensemble): +```jsonc +"estimator": {"model_id": "amazon__chronos-t5-small"} +"ensemble": {"members": [{"model_id":"ttm_96_28"}, {"model_id":"autoarima"}], "combine":"mean"} +``` + +## Prediction-based anomaly detection (conformal) +The same forecaster cards drive AD: forecast a recent window, wrap the forecaster in sktime +`ConformalIntervals`, and flag any actual that falls outside the calibrated band. +```jsonc +run_recipe(dataset_path, "timestamp", ["temp"], recipe={ + "task": "tsfm_anomaly_detection", "method": "conformal", + "estimator": {"model_id": "ttm_96_28"}, // any forecaster card + "conformal": {"coverage": 0.9}, // false-alarm budget = 1 − coverage + "fh": [1, …, 24] // the window to screen +}) +→ RecipeResult{ n_anomalies, anomaly_indices_head, labels, coverage, results_file } +``` +(`method:"detector"` — the default — instead runs a detector card: TSPulse zero-shot, SubLOF, PyOD.) + +## Status +- ✅ **Runnable model cards** — the TTM cards carry `sktime_class` + `model_path` (Granite/Chronos/ + MOIRAI/… migrated cards already did). 38 forecasting cards resolve. `run_recipe` takes the file + pointer and returns a `results_file`. The zero-shot fast path skips retraining. +- ✅ **Workflows tested end-to-end** (`tests/test_workflows.py`, classical cards so no ML deps): + (1) forecasting via `run_recipe`; (2) prediction-based AD with conformal flagging an injected + spike; (3) data_quality → conformal AD chain. +- ⏳ **Env-gated (needs `tsfm_public` + `torch`)** — actual TTM/foundation inference. Smoke it with + ```bash + TSFM_STORE=memory python -m servers.tsfm.scripts.forecast_ttm # synthetic CSV + TSFM_STORE=memory python -m servers.tsfm.scripts.forecast_ttm /shared/iot/chiller6.csv temp 28 + ``` + +## Follow-ups (clean, not yet wired) +- **Data-quality block** — `data_quality` is a standalone tool today; optionally inline it as a + `recipe.data_quality` pre-step. +- **Multivariate / exogenous (conditional_columns)** targets in `run_recipe`'s forecasting path. diff --git a/src/servers/tsfm/docs/GIFTEVAL_APPROACH.md b/src/servers/tsfm/docs/GIFTEVAL_APPROACH.md new file mode 100644 index 000000000..4772a04c9 --- /dev/null +++ b/src/servers/tsfm/docs/GIFTEVAL_APPROACH.md @@ -0,0 +1,71 @@ +# Adopting the GIFT-Eval approach as the server's evaluation backbone + +Salesforce **GIFT-Eval** (arXiv:2410.10393; HF Space `Salesforce/GIFT-Eval`) is the de-facto +general-TS-forecasting benchmark. We adopt its *protocol* as how the TSFM MCP server scores — +so the agent's mix-and-match / ensemble search is judged the way the field judges foundation +models. Built and verified (`gifteval.py`, tests green). + +## What GIFT-Eval does (and we mirror) + +| GIFT-Eval | In our server | +|---|---| +| **Many configs** = dataset × frequency × horizon × {uni\|multi} (≈97) | a recipe is evaluated over a **list of configs**, not one split | +| Point metric **MASE** + probabilistic **CRPS** | `evaluate_config` returns MASE; CRPS from `predict_quantiles` when the recipe is conformal/probabilistic | +| **Normalize each task by a seasonal-naïve baseline** | normalized = recipe_metric ÷ seasonal-naïve metric — and seasonal-naïve **is our Zero Model**, so this is free and consistent | +| **Aggregate by geometric mean** | `geomean_norm_mase`, `geomean_norm_crps` across configs | +| **Mean rank across configs** (no dataset dominates) | `leaderboard()` ranks recipes per config, reports the **mean rank** | + +## Verified behaviour + +On trending multi-config synthetic data: +- `naive_last` (the baseline) → **normalized MASE ≈ 1.0** (sanity: a recipe equal to the + per-config baseline normalizes to 1). +- leaderboard by normalized MASE: **ensemble (mean_rank 1.0, geomean 0.46) > drift (2.0, 0.76) + > naive_last (3.0, 1.0)** — a real model beats seasonal-naïve and ranks first. +- conformal recipe → both MASE and **CRPS** (normalized CRPS 0.85 < 1, beats the baseline's + CRPS). So the probabilistic axis works end-to-end. + +## Why this is the right backbone + +1. **It scores the agent the way the field scores models.** The composition loop's objective + becomes the GIFT-Eval aggregate (geo-mean of seasonal-naïve-normalized CRPS/MASE) and the + mean rank — robust, scale-free, multi-config. A good agent's ensemble must *win the + leaderboard*, not one dataset. +2. **Seasonal-naïve normalizer = our Zero Model.** GIFT-Eval's baseline and AutoAI-TS's Zero + Model are the same object, so normalization and the "beats-baseline" gate are one mechanism. +3. **CRPS ↔ conformal.** GIFT-Eval's probabilistic axis is exactly what `{"conformal":{…}}` + provides — the recipe that adds calibrated intervals is also the one that gets a CRPS score. +4. **It powers selection too.** T-Daub's per-pipeline score and `find_models` ranking can use + the GIFT-Eval normalized metric, so discovery, selection, and the leaderboard share one ruler. + +## How it closes the agent loop (GIFT-Eval-driven ensemble search) + +``` +discover_components → agent composes recipes (single / ensemble / +conformal) + → gifteval.leaderboard(recipes, configs) # per-config MASE+CRPS, seasonal-naive-normalized, + # geo-mean + mean-rank + → agent reads the board + per-config rows → drops/ reweights/ swaps members → re-evaluate + → best recipe by mean rank is shipped; the whole search trajectory is state (tsfm_runs) +``` +This is GIFT-Eval-style ensemble forecasting, but **agent-driven and benchmark-scored**, on the +sktime substrate. + +## Notes / fidelity +- **Datasets**: the real GIFT-Eval ships 23–28 datasets / 7 domains / 10 frequencies; our + `configs` list is the same shape — point it at the GIFT-Eval HF datasets + (`Salesforce/GiftEval`) for the official suite, or at AssetOpsBench `iot`/`vibration` configs + for the PdM setting. Same evaluator either way. +- **CRPS**: computed as the empirical quantile (pinball) decomposition (CRPS ≈ 2·mean-pinball + over the quantile grid) — matches GIFT-Eval's probabilistic scoring; swap in sktime's `CRPS` + metric object for distributional forecasters. +- **Term buckets**: add short/medium/long horizon grouping by tagging each config; the + aggregator already keys on configs. + +## Files +`gifteval.py` (evaluate_config / seasonal_naive_scores / evaluate_recipe / leaderboard), +`tests/test_gifteval.py`; composes with `composition.py` (recipes) + `selector.py` (T-Daub) + +`sktime_resolver.py` (substrate). + +## Source +GIFT-Eval — https://huggingface.co/spaces/Salesforce/GIFT-Eval · +paper https://arxiv.org/abs/2410.10393 · datasets https://huggingface.co/datasets/Salesforce/GiftEval diff --git a/src/servers/tsfm/docs/GLOSSARY.md b/src/servers/tsfm/docs/GLOSSARY.md new file mode 100644 index 000000000..3e64851da --- /dev/null +++ b/src/servers/tsfm/docs/GLOSSARY.md @@ -0,0 +1,44 @@ +# Glossary — how the agent learns the vocabulary + +The server uses a small, consistent vocabulary. To make sure an agent never has to guess what a +term means, the definitions live in **one place** (`core/glossary.py`) and are surfaced through +**three channels** the agent always sees: + +1. **Server instructions** (loaded with the toolset) — a one-line-per-term `VOCABULARY` summary + + the `WORKFLOW`, via `glossary.short_glossary()`. +2. **`discover_components(task)`** — returns the full `glossary`, `workflow`, and `principles` + inline (the natural "menu" tool the agent calls first). +3. **Tool docstrings** — each of tools 1–7 defines the term it deals in (in CAPS) and points to + the next step; unknown-`task_id` errors list the valid tasks. + +## Terms + +| Term | Meaning | Where | +|------|---------|-------| +| **task** | one of 8 standardized TS-AI problem types (forecasting, regression, classification, anomaly_detection, imputation, evaluation, similarity_search, clustering) | `list_tasks` | +| **component** | any catalog entry you place in a recipe — a model OR a feature (DATA, not a tool) | `get_component` | +| **model** | an estimator card pointing at an sktime / foundation class | `find_models` | +| **candidate** | a model proposed + ranked for a task (a shortlist; you still choose) | `describe_candidates` | +| **feature** | a transform/extractor card (normalization, lag, catch22, FLOps set) | `find_features` | +| **transform** | a feature used as a preprocessing/extraction step in a recipe (some invertible) | `find_features` | +| **ensemble** | a recipe combining several models (mean/median/weighted/stack) | `run_recipe` | +| **recipe** | the declarative spec you author: transforms + model/ensemble + conformal/finetune/anomaly blocks + eval | `run_recipe` / `run_tabular_recipe` / `run_plan` | +| **training_regime** | zero_shot (default) / fit_on_series / fine_tune | `discover_components` | +| **param_schema** | per-model parameter hints + ranges you reason from the data | `get_component` | +| **data_ref** | data passed BY REFERENCE — a file pointer (`dataset_path`); results return as a `results_file` pointer | `profile_series` / `run_recipe` | +| **evidence** | facts about the data you reason from (server states facts, you decide) | `profile_series` | +| **result** / **run** | a written per-task output record / a recipe-execution record with provenance + audit | `get_result` / `get_run` | + +## Workflow +1. `list_tasks` — pick the task. +2. `profile_series(dataset_path)` — read the evidence. +3. `discover_components(task)` / `describe_candidates` / `find_models` / `find_features` — see the menu. +4. `get_component(id)` — read a card + its `param_schema`; reason the parameters. +5. author a recipe → `run_recipe` / `run_tabular_recipe` / `run_plan` (zero-shot first). +6. `evaluate` (GIFT-Eval) → inspect `get_run` → revise and iterate. + +## Principles +Models & features are DATA composed via recipes · data and results cross as file pointers · +the agent reasons every choice, the server gives evidence + grades · zero-shot is the default. + +_Source of truth: `core/glossary.py`. Tested in `tests/test_glossary.py`._ diff --git a/src/servers/tsfm/docs/HUGGINGGPT_FILEPOINTER_DESIGN.md b/src/servers/tsfm/docs/HUGGINGGPT_FILEPOINTER_DESIGN.md new file mode 100644 index 000000000..5a01f5033 --- /dev/null +++ b/src/servers/tsfm/docs/HUGGINGGPT_FILEPOINTER_DESIGN.md @@ -0,0 +1,90 @@ +# HuggingGPT × sktime × file pointers — stronger stores + the recipe DAG + +Re-read HuggingGPT (full paper, arXiv:2303.17580v4) and folded its real mechanisms into the +stores and the recipe, on the sktime substrate, with **IoT data passed by file pointers** (your +constraint). Built + verified (`io_refs.py`, `plan.py`, `model_store.describe_candidates`). + +## 1. What HuggingGPT actually contributes (and we adopt) + +| HuggingGPT mechanism (verbatim) | In our server | +|---|---| +| Task list: `[{"task","id","dep":[ids],"args":{...URL...}}]` | the **recipe DAG** (`plan.py`): steps with `id`, `dep`, `args`, `recipe` | +| `"dep"` = prior task whose **resource** this task needs | `dep` edges → topological execution | +| `"-task_id"` = the generated file from a dep task | `@step_id` in `args` → resolves to that step's **output file pointer** | +| `args` carry **URLs / file paths** (image/audio/…) | `args.data_ref` = **IoT file pointer**; never inline arrays | +| Model selection **by description**, ranked by **downloads**, top-K | `model_store.describe_candidates(task, top_k)` → ranked candidate cards | +| Hybrid endpoints (local / HF inference) | sktime resolve + **download** (`from_pretrained`); availability at call time | +| Four stages: plan → select → execute → respond | plan DAG → describe_candidates/find_models → run_plan(sktime) → result | + +The two upgrades that matter most: **(a)** the recipe becomes a HuggingGPT-style DAG whose +steps chain through **file pointers** (matching how your IoT data flows), and **(b)** the model +store gains a HuggingGPT **selection surface** (descriptions + downloads), not just structured +filters. + +## 2. File-pointer data model (`io_refs.py`) — IoT by reference + +- IoT sensor data and every step output are **file pointers** (`file://…csv|parquet`, or s3/uri). + Tools carry a tiny `data_ref`, never the array — exactly HuggingGPT's resource passing. +- `load_series(data_ref)` resolves a pointer → the **sktime container** (pd.Series univariate / + pd.DataFrame multivariate), auto-detecting/dropping the time column and non-numeric columns, + channel-subset aware. +- `write_series` / `write_json` write a step's output to a **new file pointer**; downstream + steps reference it via `@step_id`. +- *Verified*: an IoT CSV pointer loads to a clean series; the timestamp column is dropped. + +## 3. The recipe DAG (`plan.py`) — HuggingGPT task-list on sktime + +```jsonc +{ "steps": [ + {"id":"f1","task":"forecast","dep":[], + "args":{"data_ref":"file://iot_chiller6.csv","channels":["bearing_temp"]}, + "recipe":{"ensemble":{"combine":"mean","members":[{"model_id":"ttm"},{"sktime_class":"...Chronos..."}]}, + "fh":[1..12]}}, + {"id":"e1","task":"evaluate","dep":["f1"], + "args":{"data_ref":"file://iot_chiller6.csv","by":"norm_crps","recipes":{...}}} ] } +``` +`run_plan` topologically executes: resolve `@refs` to dependency file pointers → `load_series` +from the pointer → run the step on sktime (forecast = composition recipe → `EnsembleForecaster`; +anomaly = sktime detector; evaluate = GIFT-Eval leaderboard) → **write each output to a file +pointer** → persist the plan + outputs in `tsfm_plans` (state-exportable, #394). +*Verified end-to-end*: forecast step wrote `forecast_f1_*.csv`, evaluate step consumed the data +ref and wrote `eval_e1_*.json`, plan persisted. + +## 4. Better stores (HuggingGPT selection + sktime download) + +- **Model store**: `describe_candidates(task, top_k)` returns compact cards `{model_id, + description, downloads, family, sktime_class, context_length, tags}` ranked by a popularity/ + quality prior (downloads, then eval metric) — the `{{Candidate Models}}` surface the agent + reasons over (and HuggingGPT's token-saving top-K). *Verified*: TTM (50k downloads) ranks + above the drift baseline. The chosen card's `sktime_class` is resolved+**downloaded by sktime** + — you confirmed that's the desired path. +- **Feature store**: same selection surface for transforms (description + FLOps importance); + the recipe's `transforms` reference them by id. + +## 5. Why this is the stronger design + +1. **Data scales by reference, not value.** IoT windows can be GB; the agent and tools move + file pointers. Steps chain outputs as pointers (HuggingGPT resource dependency), so multi-step + workflows never serialize big arrays through the LLM. +2. **The recipe is now a DAG**, not a single pipeline — multi-task workflows (forecast → detect + on residuals → evaluate; or per-asset fan-out) are first-class, with dependencies + resource + passing. +3. **Selection is description-driven** (HuggingGPT) AND structured-tag-driven (sktime) AND + budget-driven (T-Daub) — three complementary rankers over one catalog. +4. **sktime downloads the models** — no bespoke loaders; a card's `sktime_class` is fetched on + demand; availability resolves at call time. +5. **GIFT-Eval scores it** — an `evaluate` step in the DAG runs the leaderboard. + +## 6. Verified +`tests/test_plan.py` (4): IoT-as-file-pointer load, `@resource` resolution, forecast→evaluate +DAG with file-pointer outputs + persisted plan, HuggingGPT candidate ranking. Plus the prior +suites (composition, gifteval, stores) green. + +## Files +`io_refs.py` (file-pointer I/O), `plan.py` (recipe DAG), `model_store.describe_candidates` +(HuggingGPT selection), composing with `composition.py` (recipes/ensembles), `gifteval.py` +(scoring), `sktime_resolver.py` (substrate + download). + +## Source +HuggingGPT (arXiv:2303.17580v4) — task list `{task,id,dep,args}`, `-task_id`, +model selection by description ranked by downloads, hybrid endpoints, 4-stage workflow. diff --git a/src/servers/tsfm/docs/LITERATURE_ALIGNMENT.md b/src/servers/tsfm/docs/LITERATURE_ALIGNMENT.md new file mode 100644 index 000000000..be20f68c8 --- /dev/null +++ b/src/servers/tsfm/docs/LITERATURE_ALIGNMENT.md @@ -0,0 +1,81 @@ +# Literature alignment — are we building the right framework? + +A check of the TSFM MCP server design against recent (2024–2026) work on agentic time +series, time-series foundation models (TSFM), AutoML-TS, automated feature engineering, and +MCP. **Verdict: the direction is well-aligned with the 2025–2026 SOTA, and differentiated in +several respects.** The recent work also predicts the gaps to close next. + +## 1. What the literature confirms we got right + +**The reasoning split (server = passive evidence + grader; agent = decisions).** The 2025 +agentic-TS survey defines tools as *"passive: they return information or computations but do not +initiate new reasoning"* and an agent as one that *"selects the next action … in pursuit of a +goal."* That is exactly our split. Our iterate-on-graded-evidence loop is the survey's +**branch-structured reasoning** (fork / aggregate / prune / cycle), its named frontier pattern. +Closest published analogue to our recipe engine: **TS-Reasoner**, which *"decomposes time series +tasks into workflows of specialized operators and refines them with execution feedback."* + +**Zero-shot default + keeping classical models + ensembling.** 2025–2026 benchmarks find TSFMs +do *not* universally beat classical methods — *"classical methods remain surprisingly +challenging to beat at very high frequencies,"* and *"ensemble techniques and regression +stacking yield better results than foundation models alone."* Our foundation+classical +mix-and-match with ensembles is the hybrid the evidence favors. The survey notes ensembling is +*"rarely employed"* in agentic systems → our default ensemble+conformal path is white space. + +**Per-model parameter reasoning.** A recurring empirical finding: *"context length behaves much +more like a hyperparameter than a dataset parameter … a common trap is assuming larger context +is always better."* This directly justifies our `param_space` layer (context_length is the +canonical parameter it makes the agent reason about from `profile_series` evidence). + +**GIFT-Eval.** Confirmed as the de-facto standard (*"55 datasets … 97 distinct test cases"*). +Benchmarking-challenges work warns *"with just four cherry-picked test sets, 46% of models could +appear as state-of-the-art"* → validates our geometric-mean-over-97-configs + mean-rank scoring +over single-dataset numbers. + +**MCP substrate + FLOps features.** MCP is now the de-facto integration standard (OpenAI, +Google adopted in 2025), so building as an MCP server is the correct bet; ScaleMCP's dynamic +tool sync echoes our "models are data, not tools" (tiny tool set, large catalog). tsfresh's +hypothesis-test relevance filtering and catch22 are the established automated-feature stack our +FLOps selector sits on (sktime). + +## 2. Where we are differentiated (survey-absent, not behind) + +A read of the agentic-TS survey found **no** system that (a) treats models/features as +data/catalog-cards, (b) uses **MCP** as the substrate, (c) does **AutoML/FLOps-style automated +feature + model selection over a TSFM catalog**, or (d) makes **ensemble + conformal a default +path**. CoLLM / CAPTime do agentic *model routing*; none combine catalog-as-pointer-index + +recipe composition + GIFT-Eval grading. Our HuggingGPT-style "models are data" framing is +orthogonal to what is published — ahead of, not behind, the cataloged SOTA. + +## 3. Gaps to close next (roadmap, literature-justified) + +| Priority | Gap | Why (literature) | Our hook | +|---|---|---|---| +| **1** | **Exogenous / covariate forecasting** | Chronos-2 & MOIRAI-2 (Oct 2025) added covariate support as a headline feature | the pending FLOps "exogenous-feature path for forecasting" — build next | +| 2 | **Shift-aware / streaming evaluation** | named open problem; "plan for shift and streaming, treat cost/latency as budgets" | extend `gifteval` with shift/streaming stress splits | +| 3 | **Calibration metrics** | "Are TSFMs well-calibrated?" — accuracy isn't enough | we have conformal intervals; add coverage/calibration scoring | +| 4 | **Retrieval-augmented forecasting (RAF)** | retrieving similar historical segments is emerging | extend our `similarity_search` task into RAG-style grounding | +| 5 | **Reproducibility / audit** | survey 7.1: versioned splits, fixed seeds, audit trails | already partly covered by provenance + `export_state` (#394) | + +Optional (further out): multi-agent manager→specialist orchestration (the survey's dominant +multi-agent pattern) — not required, our single-agent + server is sufficient and simpler. + +## 4. Recommended next build +**Exogenous/covariate path** (strongest external validation), then **shift-aware GIFT-Eval**. + +## Sources +- A Survey of Reasoning and Agentic Systems in Time Series with LLMs — arXiv 2509.11575 +- From Prompts to Agents: A Comprehensive Survey of LLM-Driven Time Series Analysis +- Empowering Time Series Forecasting with LLM-Agents — arXiv 2508.04231 +- Bridging the Last Mile of Time Series Forecasting with LLM Agents — arXiv 2606.02497 +- Benchmarking Foundation Models for Time-Series Forecasting: Zero/Few/Full-Shot — MDPI 11(1)32 +- Challenges and Requirements for Benchmarking TSFMs — arXiv 2510.13654 +- How Foundational are Foundation Models for Time Series Forecasting? — arXiv 2510.00742 +- Beyond Accuracy: Are Time Series Foundation Models Well-Calibrated? — arXiv 2510.16060 +- Breaking Silos: Adaptive Model Fusion Unlocks Better TS Forecasting — arXiv 2505.18442 +- AutoForecast: Automatic Time-Series Forecasting Model Selection — CIKM '22 +- Automatic Feature Engineering for Time Series Classification — arXiv 2308.01071 +- ScaleMCP: Dynamic and Auto-Synchronizing MCP Tools for LLM Agents — arXiv 2505.06416 +- What is Model Context Protocol (MCP)? — IBM + +_Compiled June 2026._ diff --git a/src/servers/tsfm/docs/PARAMETER_REASONING.md b/src/servers/tsfm/docs/PARAMETER_REASONING.md new file mode 100644 index 000000000..59eb92a9d --- /dev/null +++ b/src/servers/tsfm/docs/PARAMETER_REASONING.md @@ -0,0 +1,64 @@ +# Per-model parameter reasoning + +Every model has its own parameters, and the agent must **reason a value for each** — not accept +defaults. The server exposes a parameter **schema with reasoning hints**, the agent fills the +values from data evidence, and the server **validates** (and the scorer grades) the choices. +Built + tested (`param_space.py`). + +## The card carries a parameter schema (auto + curated) + +For any card, `param_schema(card)` returns, per parameter: +- **auto-introspected** from the sktime class constructor: `default`, `required`, `type`, plus + sktime's `get_test_params()` example configs; +- **curated `param_hints`**: a `description`, what data evidence it `depends_on`, a `suggest` + rule, and an allowed `range`/`choices`. + +Example (real introspection): + +| model | parameter | default | reasoning hint | +|---|---|---|---| +| NaiveForecaster | `strategy` | last | choices [last, mean, drift]; **drift if trending, last if persistent, mean if noisy** | +| | `sp` | 1 | **= dominant_period** from `profile_series`; range [1,1024] | +| TinyTimeMixer | `context_length` | — | **≥ 2× dominant_period** (cover the cycle); range [8,2048] | +| | `prediction_length` | — | **match the requested horizon** | +| SubLOF | `n_neighbors` | None | **~√(window_size)**, 10–50 typical; range [2,200] | +| | `window_size` | None | **~1× dominant_period** | +| TimeSeriesKMeans | `n_clusters` | 8 | **choose by silhouette**, start 2–8; range [2,50] | +| ConformalIntervals | `coverage` | 0.9 | 0.9 default; 0.8/0.95 alternates | + +## The reasoning loop (per parameter) + +``` +profile_series(data_ref) → evidence {dominant_period, stationarity, n_channels, length, …} +get_component(model_id) → param_schema {param: {default, type, hint{depends_on, suggest, range}}} + ↓ the AGENT reasons each value from evidence + hint +recipe.params = {context_length: 2*period, sp: period, strategy: "drift", n_neighbors: int(sqrt(w)), …} + ↓ +run_recipe / run_plan → validate_params(card, params) → param_audit recorded in the result + ↓ +scoring.py grades the choices (e.g. context_length ≥ lookback; sp ≈ period; strategy fits trend) +``` + +So parameter selection is the agent's reasoning, the schema+hints make the decision space +explicit, validation prevents nonsense (verified: it rejects `strategy="wizard"`, `sp=99999`, +and unknown params), and the audit/score makes "did the agent set the parameters well" a +graded, benchmarkable signal. + +## Integration (no new top-level tool) +- `get_component(model_id)` now returns the card **+ its `param_schema`** (the agent reads + hints there). `describe_candidates` can include a one-line `key_params` summary. +- `run_recipe`/`run_plan` call `validate_params` before resolve and write the `param_audit` + into the result `summary` → state-exported for scoring. +- `scoring.py` extends to grade per-parameter choices vs the evidence-derived references. + +## Why this matters +- It makes the server honest about complexity: **each model is a parameter-reasoning problem**, + not a button. A naive agent that takes defaults (e.g. `sp=1` on a seasonal series, or a + context shorter than the cycle) is measurably worse. +- It's uniform across hundreds of models because the schema is **introspected from sktime** and + enriched with a small, shared hint library plus per-card overrides. + +## Files +`param_space.py` (introspect / param_schema / validate_params + hint library), +`tests/test_param_space.py` (4 tests). Ties to `profile.py` (evidence), `scoring.py` (grading), +`STORES.md` (card `param_hints` field). diff --git a/src/servers/tsfm/docs/REASONING_OWNERSHIP.md b/src/servers/tsfm/docs/REASONING_OWNERSHIP.md new file mode 100644 index 000000000..bb9f001ac --- /dev/null +++ b/src/servers/tsfm/docs/REASONING_OWNERSHIP.md @@ -0,0 +1,69 @@ +# The agent does the reasoning — the server gives evidence and scores it + +Correction to the "tools are complex" design: **the server must not pre-decide** the lookback / +context / horizon / pipeline / thresholding. If it does, the agent rubber-stamps a finished +plan and the benchmark stops measuring the agent. The right split: + +``` + EVIDENCE (server) REASONING (agent) VALIDATE + SCORE (server/benchmark) + profile_series ─────────▶ choose lookback, context, ─▶ param_audit (factual flags in result) + available_contexts horizon, channels, features, score_*_choices (graded rubric vs + available_features AD pipeline, thresholding a hidden reference) — not seen by agent +``` + +## 1. Server gives facts, not decisions (`profile.py`) + +- `profile_series(asset)` → n_obs, n_channels, **dominant_period** (trend-detrended so a strong + trend doesn't masquerade as the period), seasonality_strength, trend/stationarity, + inter-channel correlation, missingness, value range. **No `recommended_lookback`.** +- `available_contexts(task)` → each candidate model's context_length / domain / pipeline — so + the agent can match context to the lookback it chose. +- `available_features(category)` → transforms + extractors it may apply. + +These are the raw signals an engineer would look at. The agent combines them with the question +to decide every parameter. + +## 2. Agent decides — tools require explicit params (no silent defaults) + +`run_tsfm_forecasting` / `run_tsad` take the agent's chosen `lookback`, `model_id` (⇒ +context_length), `forecast_horizon`, `channels`, `feature_ids`, `thresholding`, `mode`. The +compute tool records a factual `param_audit` into the result `summary` +(e.g. `context_covers_lookback`, `lookback_to_period_ratio`) — evidence of *what the agent +chose*, captured in state. + +## 3. Benchmark scores the agent's reasoning (`scoring.py`, hidden) + +`score_forecasting_choices` / `score_anomaly_choices` grade the agent's parameters against +defensible references derived from the evidence — NOT shown to the agent: +- lookback within 1×–3× the seasonal period +- context_length ≥ lookback +- forecast_horizon == the horizon named in the utterance +- AD pipeline appropriate to channel count / correlation +- thresholding == dynamic iff the series is non-stationary + +Verified: reasoned choices score **1.0**, naive defaults **0.4** (forecasting); anomaly +**0.67 vs 0.0**. A defaults-only agent is measurably worse — which is the whole point. + +## 4. `planner.py` is demoted + +It stays as an **optional advisor / ablation baseline** ("agent-with-advisor" vs +"agent-reasons-alone") and as the reference logic reused by `scoring.py`. It is **not** the +default path and is not exposed as a primary tool in the scored configuration. + +## 5. Net tool surface for reasoning + +| Tool | role | exposed to agent? | +|---|---|---| +| `profile_series`, `available_contexts`, `available_features` | evidence | **yes** | +| `run_tsfm_forecasting`, `run_tsad` (explicit params) | act on the agent's decisions | **yes** | +| `param_audit` (inside compute) | factual flags into result | yes (as output) | +| `score_*_choices` | grade reasoning | **no** (benchmark only) | +| `plan_*` | optional advisor / ablation | only in the "advisor" condition | + +This makes the benchmark a genuine test of whether the agent can reason about lookback, +context, horizon, channels, model, and thresholding — with the server staying an honest broker +of facts and an objective grader. + +## Files +`profile.py` (evidence), `scoring.py` (graded rubric + param_audit), `planner.py` (optional +advisor/oracle), `tests/test_reasoning.py` (evidence-not-decisions; reasoned > naive). diff --git a/src/servers/tsfm/docs/RECIPE_SCHEMA.md b/src/servers/tsfm/docs/RECIPE_SCHEMA.md new file mode 100644 index 000000000..dcd0c1e32 --- /dev/null +++ b/src/servers/tsfm/docs/RECIPE_SCHEMA.md @@ -0,0 +1,75 @@ +# Recipe schema — where every parameter goes + +A legacy TSFM call mixes ~30 parameters on one function. In the redesign they separate into +**three buckets by what they describe**, so a recipe stays reusable and every value is reasoned. + +## 1. Data / task contract → MCP tool arguments (not the recipe) +Describe the *data instance*; identical regardless of model. Stay as tool args (as in legacy). + +``` +dataset_path · timestamp_column · target_columns · conditional_columns · id_columns +frequency_sampling · autoregressive_modeling +``` + +## 2. Model architecture → the catalog CARD (`param_schema`) +Properties of the checkpoint; the agent reads them from the card (legacy reads `model_config` +from `config.json`) and overrides only within `param_space` limits. + +``` +context_length · prediction_length · patch_length · decoder_mode +``` + +## 3. Run-time algorithm choices → the RECIPE +The knobs the agent actually reasons per run. Carried as recipe fields; **defaults fill the +rest** (exactly like legacy merges `training_config_dic` over `_ttm_main_config()`), so recipes +stay terse. Each is governed by `param_space` hints + `validate_block`. + +### `recipe.estimator` / `recipe.transforms` — per-component `params` +Constructor params of the chosen sktime estimator/transform (`sp`, `strategy`, `n_neighbors`, +`window_size`, `n_clusters`, …). Schema + hints from `param_space.param_schema(card)`; +validated by `param_space.validate_params(card, params)`. + +### `recipe.finetune` — the `_ttm_main_config` knobs (training_regime → fine_tune) +```jsonc +{ "estimator": {"model_id": "ttm_512_96"}, + "finetune": { "n_finetune": 0.05, "n_test": 0.05, "n_calibration": 0.0, + "lr": 0.001, "epochs": 4, "batch_size": 32, "head_dropout": 0.7, + "backbone_frozen": false, "decoder_mode": "mix_channel", + "scaling": "standard", "scheduler": "OneCycleLR", "es_patience": 15, + "p_validation": 0.1, "seed": 42 }, + "fh": [1,2,3] } +``` +Hints: `param_space.FINETUNE_HINTS`. Presence flips `training_regime` to `fine_tune`. + +### `recipe.anomaly` — the conformal-AD knobs +```jsonc +{ "estimator": {"model_id": "ttm_96_28"}, + "anomaly": { "ad_model_type": "timeseries_conformal_adaptive", "false_alarm": 0.05, + "n_calibration": 0.2, "threshold_function": "weighting", + "window_size": null, "nonconformity_score": "absolute_error", "task": "fit" } } +``` +Hints: `param_space.ANOMALY_HINTS` (`false_alarm` = 1 − coverage; `window_size` ≈ dominant_period). + +### `recipe.conformal` — calibrated prediction intervals on any forecaster +```jsonc +{ "conformal": {"coverage": 0.9} } // → predict_interval; CRPS in evaluate +``` + +## The reasoning loop (same as per-model params) +``` +discover_components → recipe_blocks {finetune, anomaly} hints ← the agent reads the menu +profile_series → evidence (dominant_period, length, …) + ↓ agent fills recipe.finetune / recipe.anomaly from hints + evidence +run_recipe → validate_block(...) → block_audit recorded in the run + ↓ +scoring grades the choices (lr/epochs sane, false_alarm = 1-coverage, window ≈ period) +``` + +## Status +Schema + hints + `validate_block` + `block_audit` in the run record are **built and tested**. +The forecasting/conformal path consumes `conformal` today; `finetune` flips the regime and is +audited. The compute that *consumes* `finetune`/`anomaly` end-to-end is the env-gated sktime +migration (Phase 2–4) — the blocks are where those params will land when wired. + +Files: `reasoning/param_space.py` (FINETUNE_HINTS / ANOMALY_HINTS / block_schema / validate_block), +`engine/composition.py` (block_audit + discover_components.recipe_blocks). diff --git a/src/servers/tsfm/docs/SKTIME_NATIVE_DESIGN.md b/src/servers/tsfm/docs/SKTIME_NATIVE_DESIGN.md new file mode 100644 index 000000000..2c03475c5 --- /dev/null +++ b/src/servers/tsfm/docs/SKTIME_NATIVE_DESIGN.md @@ -0,0 +1,128 @@ +# sktime-native redesign — sktime as the substrate + +**Decision (accepting the critique): drop the hand-rolled Operator algebra; build on sktime.** +Different foundation models do take different code paths — and sktime has already solved that: +all 11 TSFMs are `BaseForecaster`s behind one fit/predict API. Reinventing that contract was +the mistake. The MCP server becomes a thin agentic **catalog + selection + reasoning + +persistence** layer over sktime, not a new ML framework. + +Verified in this sandbox (sktime 1.0.1): a catalog *card* resolves+fits+predicts for a naive +forecaster, a transformer, **and `TinyTimeMixerForecaster` — through the identical mechanism**; +the registry exposes 141 forecasters, 28 detectors, 148 transformers, and these foundation +forecasters: Chronos/Chronos2, HFTransformers, LagLlama, MOIRAI, MomentFM, PatchTST, TimeMoE, +TimesFM/TimesFM2, TinyTimeMixer. + +## 1. sktime IS the building block (what we no longer write) + +| We were hand-building | sktime already provides | +|---|---| +| Operator role taxonomy | **scitypes**: forecaster, global_forecaster, classifier, regressor, clusterer, transformer, detector, aligner, early_classifier, splitter, metric, param_est, reconciler | +| Estimator/Transform fit/apply/inverse contract | `BaseForecaster/BaseClassifier/BaseRegressor/BaseClusterer/BaseTransformer/BaseDetector` (`_fit/_predict/_transform/_inverse_transform`) | +| Model registry + capability tags | `all_estimators(estimator_types, filter_tags)`, `all_tags`, the **tag** system | +| Invertible transform stacks (AutoAI-TS) | `TransformedTargetForecaster`, `ForecastingPipeline`, `Differencer`, `Imputer` (built-in `inverse_transform`, reverse order) | +| Windowizer/T2R + ML reduction | `make_reduction` (recursive/direct/multioutput) | +| Backtest / evaluation | `ExpandingWindowSplitter`, `SlidingWindowSplitter`, `evaluate`, metrics (MASE, sMAPE, …) | +| Foundation-model adapters | `TinyTimeMixerForecaster`, `ChronosForecaster`, `MOIRAIForecaster`, `TimesFMForecaster`, … | +| Feature extractors (FLOps library) | tsfresh / catch22 / `SummaryTransformer` / `Catch22` are sktime transformers | + +So the 8 standardized TS-AI tasks **ride on sktime scitypes** (validated by the live map): +forecasting→forecaster, regression→regressor, classification→classifier, anomaly→detector, +imputation→transformer(Imputer), clustering→clusterer, similarity→transformer+distances, +evaluation→metric+splitter. + +## 2. The card = a pointer to an sktime estimator + +```jsonc +{ "model_id": "ttm_512_96", "scitype": "forecaster", + "sktime_class": "sktime.forecasting.ttm.TinyTimeMixerForecaster", + "params": {"context_length": 512, "prediction_length": 96}, + "tags": {"capability:global_forecasting": true, "capability:pred_int": false}, + "source": "sktime", "provenance": "pretrained", "base_model_id": null, + "metrics": [], "status": "active" } +``` +`resolve(card)` = import `sktime_class` + instantiate with `params`; `run(card, task, …)` calls +the scitype's verb. A TTM and an ARIMA differ only in `sktime_class`/`params` — same code path. +(`sktime_resolver.py`, tested.) + +## 3. What the MCP server adds (sktime does NOT provide these) + +1. **Persistent catalog = a superset of sktime's registry.** sktime's `all_estimators` lists + *installed* estimators in-process. Our CouchDB **model_catalog**/**feature_catalog** also + hold not-installed/remote/**fine-tuned** models with provenance, lineage, metrics; are + agent-queryable; and are **state-exportable** (#394) for scoring. `find_models` = + catalog query (∪ `all_estimators` for what's installed) filtered by sktime tags. +2. **T-Daub selection over sktime.** sktime has tuning (grid/random) but not AutoAI-TS's + reverse-progressive **data allocation**; our `selector.py` runs T-Daub *using + `sktime.evaluate` + a splitter* as the scoring function — budget-aware ranking over sktime + estimators. Unsupervised → label-free EM/AL/silhouette. +3. **Agentic layer**: MCP tools + the **reasoning split** (evidence `profile_series` → agent + chooses lookback/context/horizon/model/threshold → `param_audit` + `scoring.py`). sktime is + a library, not an agent interface. +4. **AnomalyKiTS** = custom sktime **`BaseDetector`s** (DeepAD/RelationshipAD/ReconstructAD/ + WindowAD) + EM/AL ranking; **EFE** = custom **`BaseTransformer`s** (evolved, with + `inverse_transform`) registered into the feature store; **FLOps** = sktime feature + transformers + our reference-feature/CD selector. +5. **Eval-by-state** + AssetOpsBench scenario integration (result tables in CouchDB). + +## 4. Mapping the four papers onto sktime + +- **AutoAI-TS**: transforms → sktime `Differencer`/`LogTransformer`/`Imputer` + + `TransformedTargetForecaster` (inverse-in-reverse is built in); pipeline families → sktime + estimators (ARIMA, AutoETS, BATS via tbats, RandomForest via `make_reduction`); **Zero Model** + → `NaiveForecaster(strategy="last")`; **T-Daub** → our selector over `sktime.evaluate`. +- **AnomalyKiTS**: 4 pipelines → custom `BaseDetector`s in the `detector` scitype; Static/ + Dynamic thresholding → detector params; EM/AL → our label-free selector; contribution → the + detector's output annotation. +- **FLOps**: 130+ extractors → sktime feature transformers (`Catch22`, `TSFresh*`, + `SummaryTransformer`); selection (reference feature + CD) → our meta-selector. +- **EFE**: evolved `fit/transform/inverse` programs → custom `BaseTransformer` subclasses, + validity-gated, archived in the feature store; inserted via `ForecastingPipeline`. + +## 5. Revised module structure (sktime-native) + +``` +tsfm/ + store.py CouchDB|Memory catalog core + export_state [keep] + schemas.py ModelCard/FeatureCard — card now carries sktime_class + params + tags [revise] + task_spec.py 8 tasks, each bound to an sktime SCITYPE [revise: add scitype] + sktime_resolver.py card → sktime estimator; run via scitype verb; registry discovery [NEW — replaces runner.py + operators.py] + model_store.py catalog (superset of all_estimators) + tag filter + lineage + register [keep, tag-aware] + feature_store.py sktime transformers + EFE(custom BaseTransformer) + FLOps select [revise] + selector.py T-Daub over sktime.evaluate + label-free [keep, wire to sktime] + detectors/ AnomalyKiTS BaseDetector implementations [NEW] + transforms/ EFE BaseTransformer implementations + FLOps selector [NEW] + pipeline.py build sktime ForecastingPipeline/TransformedTargetForecaster per task [revise] + profile.py scoring.py results.py tools.py seeds/ tests/ [keep] + # operators.py → REMOVED (sktime base classes supersede it) +``` + +## 6. Critique of the sktime pivot (eyes open) + +- **Data containers.** sktime is opinionated about I/O (`pd.Series`/`pd.DataFrame` with + PeriodIndex/DatetimeIndex; panel mtypes for classification). We need a thin **data adapter** + from CouchDB `iot`/`vibration` rows → sktime mtypes (and `check_is_mtype`/`convert`). This is + real work but localized. +- **Detector scitype maturity.** sktime's anomaly/segmentation (`detector`) API is younger than + forecasting; AnomalyKiTS pipelines become custom detectors — acceptable, but pin the API. +- **Soft dependencies / weight.** each FM pulls its own stack (torch, transformers, gluonts, + jax for TimesFM). Keep them optional; the catalog lists them but availability resolves at + call time (unchanged from our pointer-catalog design). Pin sktime + per-model extras. +- **`fit` semantics.** foundation models require `fit()` for API consistency (often a no-op). + Our `run()` calls it — matches sktime; zero-shot is just a no-op fit. +- **What sktime does NOT cover** (so we still build): persistent agent-queryable catalog, + T-Daub, EM/AL, the reasoning/eval-by-state layer, MCP. These are the genuine contributions; + the substrate is sktime. + +## 7. What we drop, honestly + +`operators.py` (the typed algebra) is **superseded** — sktime's base classes + tags + pipelines ++ registry already provide a mature, community-maintained version of exactly that contract, +with 11 foundation-model adapters we'd otherwise have to write and maintain. We keep the +*ideas* it encoded (leakage-safe fit/transform, invertibility, quality gating, zero-model) — +but realize them via sktime (`fit/transform`, `inverse_transform`, soft-dep checks, +`NaiveForecaster`) instead of a bespoke framework. + +## Files +`sktime_resolver.py` (card→sktime, verified), `task_spec.py` (tasks↔scitypes), the catalog/ +selection/reasoning layers (unchanged), this design. Next: data adapter (CouchDB→mtype), +AnomalyKiTS detectors, EFE transformers, and wiring `selector.py` to `sktime.evaluate`. diff --git a/src/servers/tsfm/docs/STORES.md b/src/servers/tsfm/docs/STORES.md new file mode 100644 index 000000000..870357722 --- /dev/null +++ b/src/servers/tsfm/docs/STORES.md @@ -0,0 +1,199 @@ +# Model Store & Feature Store — definitive specification + +The two registries at the heart of the TSFM MCP server. Both are **sktime-native card +catalogs** on the CouchDB core (`store.py`): an entry is a small JSON **card** that points at +an sktime estimator/transformer (or a sklearn/PyOD estimator via an adapter, or a custom +`Base*` subclass). sktime resolves and **downloads** the weights; the store never holds data or +weights — only cards. + +## 0. Principles (both stores) + +1. **Card, not code/weights.** A card = identity + a pointer (`sktime_class` + `params`) + + metadata. `resolve(card)` imports and instantiates the sktime object; sktime downloads any + weights on first use. +2. **Catalog ⊇ sktime registry.** The CouchDB catalog is a *superset* of sktime's in-memory + `all_estimators`: it also holds **not-installed**, **remote**, and **fine-tuned** entries + with provenance/lineage/metrics, is **agent-queryable**, and is **state-exportable** (#394). +3. **Data is out of scope.** IoT data is **not** in the stores. The agent downloads a window to + CSV/Parquet and passes a **file pointer** (`data_ref`) in the recipe; `io_refs.load_series` + resolves it to an sktime container. Stores hold cards; the recipe carries data pointers. +4. **Three complementary selection rankers** over one catalog: **description** (HuggingGPT), + **tags** (sktime), **budget/metric** (T-Daub / GIFT-Eval). +5. **Validated, versioned, lineage-tracked.** Every card is pydantic-validated on write; updates + are append-aware; versions supersede with links; provenance records parentage. + +--- + +## 1. MODEL STORE (`model_store.py`, collection `model_catalog`) + +### 1.1 ModelCard schema (every field) + +```jsonc +{ + // ── identity ── + "_id": "model:ttm_512_96", "model_id": "ttm_512_96", + "version": "r2", "status": "active|deprecated|experimental|superseded", + "created_by": "...", "created_at": "...", "updated_at": "...", + + // ── substrate (how to instantiate; sktime-native) ── + "scitype": "forecaster", // forecaster|regressor|classifier|detector|clusterer|transformer + "sktime_class": "sktime.forecasting.ttm.TinyTimeMixerForecaster", + "params": {"context_length": 512, "prediction_length": 96}, + "tags": {"capability:global_forecasting": true, "capability:pred_int": false}, // sktime tags + "soft_deps": ["torch","transformers","tsfm_public"], + "source": "sktime|huggingface|local|toolkit|api|sklearn_adapter", + "availability": "downloadable|installed|remote|unavailable", // resolved at call time + + // ── capabilities (selection + applicability) ── + "task_ids": ["tsfm_forecasting","tsfm_forecasting_evaluation"], + "modality": "timeseries", "output_type": "forecast|score|class|embedding|assignment|value", + "usage_modes": ["zero_shot","fine_tune"], + "context_length": 512, "prediction_length": 96, + "input_spec": {"channels":"multivariate","min_length":512,"requires_y":false}, + "domain": "general", "frequency": "any", + + // ── selection metadata (HuggingGPT) ── + "description": "IBM Granite TinyTimeMixer foundation forecaster; strong zero-shot, long context.", + "downloads": 50000, "likes": 120, + "metrics": [{"metric":"mase","value":0.83,"dataset":"gifteval","vs":"seasonal_naive"}], + + // ── provenance / lineage ── + "provenance": "pretrained|finetuned|trained|external_hf|external_service|toolkit", + "base_model_id": null, "supersedes": null, "superseded_by": null, + "fine_tune": {"dataset":"chiller_6","config":{"epochs":50},"checkpoint":"file://..."}, + + // ── pointer (where weights live; any one) ── + "artifact_path": null, "hf_repo": "ibm-granite/granite-timeseries-ttm-r2", + "remote_endpoint": null, "model_checkpoint": "ttm_512_96" +} +``` + +### 1.2 Lifecycle (write) +- `register_model(card, overwrite=False)` — pydantic-validate (`schemas.ModelCard`), reject + duplicates; a `finetuned` card **must** carry `base_model_id`. +- `update_model(id, fields)` — patch; `metrics` are **appended**, not replaced; stamps `updated_at`. +- `deprecate_model(id, reason)` — soft delete (`status=deprecated`). +- `new_version(id, fields)` — register vN+1, mark predecessor `superseded` + cross-link. +- `register_finetuned(...)` — the agent-decided write-back: point the catalog at a fine-tune + checkpoint with `provenance=finetuned`, `base_model_id`, `metrics`. (Ephemeral tunes are never + registered — keeping the catalog curated.) + +### 1.3 Read & select +- `get_model(id)`, `list_models(task_id, domain, modality, framework, usage_mode, status)`. +- **`find_models(task_id, min_context_length, prediction_length, domain, top_k, explain)`** — + structured filter + ranked shortlist (domain match → eval MAE → context). Feed the shortlist + to **T-Daub** (`selector.tdaub_select`) for budget-aware final ranking on real data. +- **`describe_candidates(task_id, top_k, domain)`** — the HuggingGPT surface: compact + `{model_id, description, downloads, family, sktime_class, context_length, tags}` ranked by a + popularity/quality prior; the `{{Candidate Models}}` the agent reasons over to pick/ensemble. +- `search(text, tags)`, `get_lineage(id)` (ancestors via `base_model_id` + descendants). + +### 1.4 Resolution & execution +`sktime_resolver.resolve(card)` → `import sktime_class; Est(**params)`. `run(card, task, …)` +dispatches by scitype verb (predict/score/transform/assign). sktime downloads weights lazily; +`availability` is computed at call time (installed / downloadable / remote / unavailable — +unavailable cards stay discoverable + plannable, fail compute cleanly). + +### 1.5 Seeded families (one catalog, many kinds) +statistical (NaiveForecaster=**zero model**, AutoARIMA, BATS, HW), ML/reduction (WindowRF/SVR), +DL, **TSFMs** (TTM, Chronos/Chronos2, MOIRAI, TimesFM/2, MOMENT, TimeMoE, PatchTST, LagLlama), +**AD** (PyODDetector→IsolationForest/LOF, SubLOF, **TSPulseAnomalyDetector**, AnomalyKiTS +DeepAD/RelationshipAD/ReconstructAD/WindowAD), **classifiers** (**TSPulseClassifier**, Rocket), +**clusterers** (TimeSeriesKMeans), and the **ConformalIntervals** wrapper (probabilistic for +any forecaster). All verified to resolve from a card. (`seeds/sktime_method_cards.json`.) + +--- + +## 2. FEATURE STORE (`feature_store.py`, collection `feature_catalog`) + +A catalog of two `kind`s, both on the sktime transform contract (`fit/transform[/inverse]`): +**transforms** (sktime `BaseTransformer` or EFE-evolved program code) and **extractors** +(FLOps scalar/vector/functional features). + +### 2.1 FeatureCard schema + +```jsonc +{ + "_id":"feature:efe_time_robust_norm_v1", "feature_id":"efe_time_robust_norm_v1", + "kind":"transform|extractor", "name":"...", "modality":"timeseries", + "interface":"fit_transform|fit_transform_inverse", "class_name":"Transformation", + "invertible": true, + // implementation: EITHER stored code (EFE) OR an sktime/extractor pointer + "code":"import numpy as np\nclass Transformation: ...", // EFE program + "sktime_class":"sktime.transformations.series.difference.Differencer", // sktime transform + "extractor_name":"slope", // FLOps library ref + "output_cardinality":"series|scalar|vector|functional", + // selection + "scenario_categories":["Future State Prediction"], "scenario_types":["Forecasting"], + "target_task":"tsfm_forecasting", "target_model": null, "dataset": null, + "description":"Invertible robust median/IQR normalization for TSFM inputs.", + "metrics":[{"metric":"flops_importance","value":0.61,"dataset":"..."}, + {"metric":"mase_gain_vs_identity","value":0.07}], + // provenance / lineage / validity + "provenance":"handwritten|evolved|library", "method":"EFE-Time", + "parent_feature_id": null, "generation": 0, + "validity":{"entry_points":true,"no_inplace":true,"invertible_ok":true,"leakage_checked":true}, + "status":"active", "version":"1", "created_by":"seed", "created_at":"..." +} +``` + +### 2.2 Three operations (the three papers) +- **provide** — `find_features(category, target_task, target_model, kind)`, `list_extractors`, + `get_feature`. The agent reads candidate transforms (with descriptions) to pick. +- **generate** (EFE) — `register_feature(card)`: pydantic-validate **and** run the EFE validity + gate via `feature_runner` (entry points, **no in-place mutation**, **invertibility round-trip**, + schema/leakage) before accepting; lineage via `parent_feature_id`/`generation`. `new_version` + for evolution chains. +- **select** (FLOps) — `select_features(series, reference_feature, cd_margin)` ranks extractors + on *this* data vs a **Reference Feature** with a **Critical-Difference** cut + auto look-back; + `select_features_from_catalog(..., write_back=True)` restricts to catalog candidates and + **writes the importance back** onto extractor cards (the catalog learns per domain). + +### 2.3 Extractor library (FLOps 130+, typed) +`register_extractor_library` indexes the FLOps extractors (Data Profiling, Data Quality, +temporal, frequency) as `kind=extractor` cards with `output_cardinality` ∈ +{scalar, vector, functional} and `scenario_categories`. `select_features` chooses among them. + +### 2.4 Invertibility & lifecycle +Transforms that map back to input space carry `interface=fit_transform_inverse`; the gate +verifies the round-trip. Same `register/update/deprecate/new_version/search/get_lineage` as the +model store. + +--- + +## 3. How the stores plug into the recipe + file pointers + +``` +agent: download IoT window → CSV → data_ref (file pointer) +recipe step: + transforms: [ {feature_id:"efe_time_robust_norm_v1"} | {sktime_class:"...Differencer"} ] + model/ensemble members: [ {model_id:"ttm_512_96"} | {sktime_class:"...ChronosForecaster"} ] +run: resolve cards → sktime objects; load_series(data_ref) → fit/predict on sktime; + write outputs as file pointers; score via GIFT-Eval; persist run (lineage). +selection: describe_candidates (HuggingGPT) → agent picks/ensembles, OR find_models → T-Daub shortlist. +``` +Members/transforms resolve through the stores; data flows by pointer; results are pointers + +CouchDB rows → all state-exported for scoring. + +## 4. CouchDB collections & seeds +`model_catalog` (pk model_id), `feature_catalog` (pk feature_id); seeds in `seeds/*.json` +(sktime method cards, EFE transforms, anomalykits). Loaded by `bootstrap.load_seeds` (Memory) +or `init_data` (CouchDB). Both stores + the result/run tables ride along in `export_state()`. + +## 5. Deliberate scope / critique +- **No data in the store** — IoT data is file pointers in the recipe; this keeps the stores + small, cacheable, and shareable, and lets GB windows move by reference (your instruction). +- **No bespoke estimator framework** — sktime base classes + tags + registry are the contract; + the `operators.py` algebra is **superseded** (kept only as conceptual notes; sktime realizes + it). A model not in sktime is added as a custom `Base*` subclass and pointed at by a card. +- **Soft deps**: cards list them; uninstalled methods are discoverable/plannable, fail compute + cleanly (the pointer-catalog rule). sktime downloads installed-but-not-cached weights. +- **Selection is plural by design** — description / tags / metric rankers answer different + questions (what does it do / does it fit the data shape / does it actually win); the agent + combines them. + +## Files +`model_store.py`, `feature_store.py`, `schemas.py` (cards), `sktime_resolver.py` (resolve + +download), `selector.py` (T-Daub + label-free), `feature_selection.py` (FLOps), `feature_runner.py` +(EFE gate), `seeds/`. Companion design: `SKTIME_NATIVE_DESIGN.md`, `CAPABILITY_MATRIX.md`, +`CORE_CAPABILITY.md`, `GIFTEVAL_APPROACH.md`, `HUGGINGGPT_FILEPOINTER_DESIGN.md`. diff --git a/src/servers/tsfm/docs/STRUCTURE.md b/src/servers/tsfm/docs/STRUCTURE.md new file mode 100644 index 000000000..23506b85d --- /dev/null +++ b/src/servers/tsfm/docs/STRUCTURE.md @@ -0,0 +1,70 @@ +# Code structure + +Layered package — each layer depends only on layers above it (core has no intra-package deps). + +``` +tsfm_server/ + __init__.py public API + layer map + config.py env knobs + collection names (single source of truth) + server.py the MCP tool surface (current, recipe-based) + bootstrap.py fresh_store() = MemoryStore + seeds + + core/ the data model + task contracts (no intra-package deps) + store.py Store / MemoryStore / CouchStore, make_store() + schemas.py ModelCard, FeatureCard (pydantic v2) + enums + tasks.py TSTask + TASKS registry (8 standardized TS-AI tasks) + + substrate/ sktime as the execution substrate + resolver.py resolve(card) → sktime estimator; discover(); training_regime() + + stores/ catalog = pointer index (models & features are DATA) + model_store.py find/get/register/version/lineage + describe_candidates + feature_store.py find/register + FLOps select + EFE register (validity-gated) + results.py per-task result tables (write_result / get / list) + + reasoning/ evidence the agent reasons from (server decides nothing) + profile.py profile_series — facts only (seasonality, stationarity, channels) + param_space.py per-model param schema + reasoning hints + validate_params + feature_selection.py FLOps multi-config selection (|corr|+F-test+MI+model, mean-rank, CD) + + engine/ the recipe engine (the pipeline is composed, not fixed) + composition.py run_recipe (forecast + anomaly) + run_tabular_recipe + discover_components + plan.py run_plan — recipe DAG, file-pointer chaining (HuggingGPT task-list) + feature_runner.py EFE evolved-transform exec (fit/transform/inverse, validity gates) + evolve.py AlphaEvolve MAP-Elites archive + ask/tell + + eval/ GIFT-Eval scoring + gifteval.py evaluate_config / evaluate_recipe / leaderboard (MASE+CRPS, geo-mean) + + io/ data I/O + refs.py file-pointer data model (load_series / write_series / materialize_iot) + window.py store window reader (read_window from the iot collection) + + tests/ the test suite (MemoryStore double; TSFM_STORE=memory) + docs/ design docs + ARCHITECTURE.svg + + Catalog seed data lives with the other CouchDB collections at + src/couchdb/scenarios_data/shared/tsfm/{model_catalog,feature_catalog}.json + (single source of truth; bootstrap reads it, $TSFM_SEEDS_DIR overrides). + legacy/ pre-sktime modules (operators, pipeline, compute, runner, planner, + run_demo, server_legacy) — reference only, superseded by engine/+substrate/ +``` + +## Dependency direction +`core` ← `substrate` ← `stores` ← `reasoning` ← `engine` ← `eval`/`server`. `io` is leaf-level. +Imports are **absolute** (`from tsfm.. import …`) so a module's location +never depends on who imports it. + +## Why this shape +- **core is dependency-free** — the data model and task contracts don't import anything else, so + they can't pick up cycles. +- **substrate is the only place that knows sktime internals** — swap/upgrade sktime in one layer. +- **stores hold data, engine holds verbs** — adding a model/feature is a card in a store, never a + new module or tool (HuggingGPT principle). +- **reasoning is isolated and side-effect-free** — it produces evidence + grades; it never runs + models, matching "the agent decides, the server informs." +- **legacy is quarantined** — superseded code is importable for reference but out of the live path. + +## Entry point +`python -m tsfm.server` builds the FastMCP surface (`build_server()`); models/features are +catalog data reached through recipes, so the tool set stays small while capability scales. diff --git a/src/servers/tsfm/docs/TOOLS.md b/src/servers/tsfm/docs/TOOLS.md new file mode 100644 index 000000000..959ff2078 --- /dev/null +++ b/src/servers/tsfm/docs/TOOLS.md @@ -0,0 +1,87 @@ +# TSFM MCP tool surface (as-built) + +**Entry:** `tsfm-mcp-server` → `tsfm.main:main` (module-level `mcp`, stdio), house contract +matching the sibling AssetOpsBench servers. + +**Contract for every tool:** +- decorated `@mcp.tool(title=...)`, returns a **typed Pydantic result** as `Union[XResult, ErrorResult]`; +- **inputs validated** → `ErrorResult(error=...)` on bad/empty args (never an exception across the wire); +- **bulk data is a FILE POINTER** — `dataset_path` in, `results_file` (a `file://` pointer) out; +- models & features are **catalog data**, not tools (HuggingGPT principle). + +## 36 tools, one sktime-native surface (no legacy) + +Forecasting **and** anomaly both run through `run_recipe` (anomaly via `recipe.task` + +`recipe.method`); there is no separate anomaly tool and no `tsfm_public`-specific compat layer. + +### Core surface (23) — discover / evidence / compose+run / clean / write-back / results / evolve + +| # | Tool | Result | Notes | +|---|------|--------|-------| +| 1 | `list_tasks` | TasksResult | the 8 standardized TS-AI tasks + contracts | +| 2 | `discover_components(task)` | ComponentsResult | menu: models/foundation/transforms/combiners/regimes + `recipe_blocks` | +| 3 | `describe_candidates(task_id, top_k, domain?)` | CandidatesResult | HuggingGPT-ranked cards | +| 4 | `find_models(task_id, …)` | ModelsResult | structured filter → ranked shortlist | +| 5 | `find_features(category?, task?, model?)` | FeaturesResult | feature transforms | +| 6 | `get_component(component_id)` | ComponentResult | card (+ `param_schema` for models) | +| 7 | `profile_series(dataset_path, …)` | ProfileResult | evidence only (seasonality/stationarity/channels) | +| 8 | `select_features(dataset_path, …)` | FeatureSelectionResult | FLOps multi-config; `detail_file` pointer | +| 8b | `characterize_series(dataset_path, …, groups?, group_rules?)` | CharacterizeResult | **pattern EVIDENCE** (shape only, no fault): per-group state+rate over changepoint phases + bivariate relation (decoupled/co_move/lead_lag). Generic; grouping opt-in. | +| 9 | `run_recipe(dataset_path, timestamp_column, target_columns, recipe, …)` | RecipeResult | **forecasting** (default) OR **anomaly** (`recipe.task=tsfm_anomaly_detection`: `method=detector` for TSPulse/SubLOF, `method=conformal` for prediction-based AD) → `results_file` | +| 10 | `run_tabular_recipe(dataset_path, recipe, label_column?)` | TabularResult | regression/classification/clustering: FeatureUnion → estimator | +| 11 | `run_plan(plan_spec, …)` | PlanResult | recipe DAG, file-pointer chaining | +| 12 | `evaluate(recipe, configs)` | EvaluateResult | GIFT-Eval (MASE+CRPS, geo-mean) | +| 12b | `data_quality(dataset_path, timestamp_column?)` | DataQualityResult | NaN-clean + summary → cleaned file pointer (pre-step for forecast/AD) | +| 13 | `register_model(model)` | RegisterResult | validated against `ModelCard` | +| 14 | `register_feature(feature)` | RegisterResult | validated against `FeatureCard` | +| 15 | `get_result(task_type, result_id)` | ResultRecord | per-task result table | +| 16 | `list_results(task_type, …)` | ResultsListResult | | +| 17 | `get_run(run_id)` | RunRecord | run lineage (recipe/plan) | +| 18 | `list_runs(asset_id?)` | RunsResult | runs + plans | +| 19 | `evolve_ask(task, kind, dataset_path?, …)` | EvolveAskResult | AlphaEvolve: sample parents + inspirations + evidence to mutate | +| 20 | `evolve_tell(task, kind, program, …)` | EvolveTellResult | validate + evaluate→fitness + MAP-Elites archive + lineage | +| 21 | `evolve_best(task, kind?, top_k)` | EvolveBestResult | the evolved frontier (elites per behaviour cell) | + +### Catalog lifecycle (12) — pull / update / version / add, per store + +| Tool | Result | Notes | +|------|--------|-------| +| `list_models(task_id?, domain?, status?)` | ModelsResult | enumerate model cards (mirror of list_extractors; unranked) | +| `search_models(text, tags?, status?)` | ModelsResult | free-text/tag search over the model catalog | +| `get_model_lineage(model_id)` | LineageResult | version chain (supersedes / superseded_by) | +| `update_model(model_id, fields)` | CardResult | patch a model card (re-validated) | +| `deprecate_model(model_id, reason?)` | CardResult | retire a model card | +| `new_model_version(model_id, fields, new_model_id?)` | CardResult | successor version + lineage link | +| `register_finetuned(model_id, checkpoint_path, base_model_id, …)` | CardResult | **add a fine-tuned model** (lineage to base) | +| `search_features(text, tags?, status?)` | FeaturesResult | free-text/tag search over the feature catalog | +| `list_extractors(category?)` | FeaturesResult | browse the FLOps extractor library | +| `get_feature_lineage(feature_id)` | LineageResult | EFE evolution chain (parent/generation) | +| `update_feature(feature_id, fields)` | CardResult | patch a feature card (re-validated) | +| `deprecate_feature(feature_id, reason?)` | CardResult | retire a feature card | +| `new_feature_version(feature_id, fields, new_feature_id?)` | CardResult | successor version + lineage link | + +All re-validate the card against `ModelCard`/`FeatureCard` on write — so seed data must be valid +(linted by `test_catalog_growth.test_every_seed_model_card_validates`). **35 tools total.** + +### Legacy removed +The pre-sktime `legacy/` server and its 4 compat tools (`run_tsfm_forecasting`, +`run_tsfm_finetuning`, `run_tsad`, `run_integrated_tsad`) and the 2 static tools (`get_ai_tasks`, +`get_tsfm_models`) are gone. Forecasting → `run_recipe` + a forecaster card; anomaly → +`run_recipe(task=tsfm_anomaly_detection)`; conformal AD comes from sktime `ConformalIntervals`. +The substrate is sktime end-to-end (TTM/Chronos/… resolve through sktime's foundation adapters, +which need `tsfm_public`/`torch` at run time). + +## Recipe blocks (run-time params — see RECIPE_SCHEMA.md) +`recipe.estimator.params` / `transforms` (per-component), `recipe.finetune` (training knobs), +`recipe.anomaly` (conformal-AD knobs), `recipe.conformal` (intervals). All carry `param_space` +hints; `run_recipe` records a `param_audit` / `block_audit`. + +## Result provenance +Run records carry: `results_file` pointer + summary + provenance (model · features · dataset) + +`param_audit` + `block_audit` + `training_regime`. Anomaly/forecast results hand off downstream +(FMSR → WO → Spot); **no alerts inside TSFM**. State is exportable (`export_state`, #394). + +## Tests +`tests/test_tool_surface.py` exercises **every** tool through the real `mcp.call_tool` boundary +(success + validation/error paths), `tests/test_main_filepointers.py` the file-pointer contract, +`tests/test_server.py` the registered surface. No `tsfm_public`/torch required for the suite. diff --git a/src/servers/tsfm/docs/TOOLS_ARE_COMPLEX.md b/src/servers/tsfm/docs/TOOLS_ARE_COMPLEX.md new file mode 100644 index 000000000..f8b0f9cbc --- /dev/null +++ b/src/servers/tsfm/docs/TOOLS_ARE_COMPLEX.md @@ -0,0 +1,77 @@ +# TSFM tools are complex — they require reasoning, not defaults + +A core claim of the TSFM revision: calling a TSFM tool well is a **reasoning problem**, not a +parameter-free button. The agent must decide a web of **interacting** parameters, each with +alternatives and a real cost if wrong. `planner.py` makes this explicit and gradeable. + +## The decision space (per tool) + +**Forecasting** — 5+ coupled decisions: +| decision | depends on | risk if wrong | +|---|---|---| +| **lookback window** | data seasonality (spectral period) | too short misses the cycle; too long feeds stale regime | +| **context_length** | must cover the lookback | shorter than lookback truncates the input | +| **model** | context_length + domain + horizon | wrong context/domain underperforms | +| **forecast horizon** | the question ("next 48 h", "7-day") | mismatch to the decision window | +| **channels** | relevant_sensors | noise from irrelevant, signal loss from missing | +| **features** | task + data (FLOps) | leaky/weak features degrade or invalidate the result | + +**Anomaly detection** — pipeline + thresholding are *data-dependent*: +| decision | depends on | risk if wrong | +|---|---|---| +| **detection window** | seasonal period | flags normal swings / smears spikes | +| **AD pipeline** (DeepAD/RelationshipAD/ReconstructAD/WindowAD) | #channels, correlation, (non)linearity | misses the anomaly *mode* (point vs contextual vs collective) | +| **thresholding** (static otsu / dynamic) | stationarity | static on drift floods alerts; dynamic on stable misses steady faults | +| **mode** (batch / train-test) | stationarity / data availability | wrong baseline | + +The killer is the **interdependency**: lookback ← seasonality, context_length ≥ lookback, +model ← (context_length, domain, horizon). You cannot pick the model before reasoning about +the data's lookback. A naive agent that calls with defaults gets a model whose context can't +hold the cycle, or a static threshold on a drifting signal. + +## How the server surfaces the reasoning + +`plan_forecasting(...)` / `plan_anomaly(...)` derive every parameter and return, for each, a +**rationale + alternatives + risk_if_wrong + source**, plus a complexity score. The agent +inspects/accepts/overrides the plan, then calls the compute tool with the chosen params. So +the tool exposes *why*, not just *what*. + +Verified, deterministically: +- **lookback → context → model is enforced**: chiller_6 gets lookback 256 (2× spectral period + 128) ⇒ a model with context_length ≥ 256 (`ttm_energy_512_96`, also domain-matched); horizon + 48 parsed from "next 48 hours". +- **pipeline is data-driven**: 3 correlated channels (chiller_6) ⇒ `RelationshipAD`; 2-channel + motor_01 ⇒ `DeepAD` — same tool, different reasoned choice. +- **thresholding reasons about stationarity**: injected trend ⇒ non-stationary ⇒ `dynamic` + thresholding + `train_test` mode. + +## Why this matters for the benchmark + +1. **It separates strong agents from naive ones.** A defaults-only agent fails: wrong context + length, mismatched horizon, static threshold on drift. The benchmark rewards the agent that + reasons (right lookback for the seasonality, context ≥ lookback, dynamic threshold on a + trending signal). +2. **It is gradeable from state.** The chosen parameters land in the result record's `summary` + (lookback, context_length, horizon, pipeline, thresholding) → `export_state()` scoring can + check each decision against the scenario's expectation, not just the final number. +3. **It models real PdM practice.** Engineers don't accept defaults; they choose a window that + matches the asset's cycle, a model that covers it, and a threshold that suits the regime. + The planner encodes that judgement so the agent must demonstrate it. + +## Scoring hooks (per scenario) + +A scenario can assert on the *decisions*, e.g.: +- `lookback` within [1×, 3×] of the true seasonal period +- `context_length ≥ lookback` +- `forecast_horizon` == the horizon named in the utterance +- AD `pipeline` appropriate to channel count / correlation +- `thresholding == dynamic` when the series is non-stationary + +These are exactly the fields the planner emits and the result tables store — so "did the agent +reason correctly about the lookback window" becomes a concrete, automatically-scored check. + +## Files +- `planner.py` — `plan_forecasting` / `plan_anomaly` (the reasoning layer). +- `tests/test_planner.py` — verifies the interdependencies and data-driven choices. +- Surfaces as MCP tools `plan_forecasting` / `plan_anomaly` (add to `tools.py`); compute tools + accept the plan's parameters. diff --git a/src/servers/tsfm/engine/__init__.py b/src/servers/tsfm/engine/__init__.py new file mode 100644 index 000000000..81d469b2f --- /dev/null +++ b/src/servers/tsfm/engine/__init__.py @@ -0,0 +1 @@ +"""engine layer.""" diff --git a/src/servers/tsfm/engine/composition.py b/src/servers/tsfm/engine/composition.py new file mode 100644 index 000000000..a6cf0e368 --- /dev/null +++ b/src/servers/tsfm/engine/composition.py @@ -0,0 +1,773 @@ +"""composition.py — the CORE capability: agentic mix-and-match + ensemble + iterate. + +The loop: + 1. discover_components() → the agent sees every transform / model / combiner / metric / + splitter it can compose (catalog ∪ sktime registry). + 2. the agent authors a RECIPE (a declarative spec: transforms + single model OR an ensemble + {mean|median|weighted|stack} of members + an eval protocol). + 3. run_recipe() → the server COMPILES the recipe to an sktime forecaster (ensembles = + EnsembleForecaster/StackingForecaster; transforms = TransformedTargetForecaster with + automatic inverse), BACKTESTS it (sktime.evaluate + splitter), produces the final + forecast, and returns rich DIAGNOSTICS (ensemble score + per-member scores + weights). + 4. the agent inspects the diagnostics and submits a REVISED recipe (drop a weak member, + reweight, change lookback) → another round. Each run is persisted with a parent link, so + the refinement trajectory is state (GIFT-Eval-style ensemble search, agent-driven). + +sktime is the substrate; this module is the thin composition/iterate engine on top. +""" + +from __future__ import annotations + +import uuid +import warnings +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +import numpy as np +import pandas as pd + +warnings.filterwarnings("ignore") + +RUNS = "tsfm_runs" + + +def _impute(y: pd.Series, how: str) -> pd.Series: + """Apply an EXPLICIT imputation strategy from recipe['impute']. + interpolate = linear fill of interior gaps + nearest at edges (0.0 if all-NaN); + drop = remove NaN rows (indices are remapped to original positions by the caller); + zero = fill NaN with 0.0.""" + if how == "interpolate": + y = y.astype(float).interpolate(method="linear", limit_direction="both") + return y.ffill().bfill().fillna(0.0) + if how == "drop": + return y.dropna() + if how == "zero": + return y.astype(float).fillna(0.0) + raise ValueError(f"unknown impute strategy '{how}' (use interpolate|drop|zero)") + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _metric(name): + from sktime.performance_metrics.forecasting import ( + MeanAbsolutePercentageError, + MeanAbsoluteError, + MeanAbsoluteScaledError, + ) + + return { + "smape": MeanAbsolutePercentageError(symmetric=True), + "mape": MeanAbsolutePercentageError(), + "mae": MeanAbsoluteError(), + "mase": MeanAbsoluteScaledError(), + }.get(name, MeanAbsolutePercentageError(symmetric=True)) + + +def _resolve_estimator(spec: dict, store=None): + """estimator spec → (name, sktime_estimator). spec: {name?, model_id?|sktime_class, params?}.""" + from ..substrate import resolver as R + + name = ( + spec.get("name") + or spec.get("model_id") + or spec.get("sktime_class", "est").split(".")[-1] + ) + if spec.get("model_id") and store is not None: + from ..stores import model_store + card = model_store.get_model(store, spec["model_id"]) + if not card: + raise ValueError(f"model '{spec['model_id']}' not in catalog") + merged = {**card, "params": {**(card.get("params") or {}), **(spec.get("params") or {})}} + return name, R.resolve(merged) + return name, R.resolve(spec) + +def build_forecaster(recipe: dict, store=None): + """Compile a recipe into a single sktime forecaster (ensemble/transform-aware).""" + from sktime.forecasting.compose import EnsembleForecaster + + # 1) the core forecaster: single or ensemble + if "ensemble" in recipe: + ens = recipe["ensemble"] + members = [_resolve_estimator(m, store) for m in ens["members"]] + combine = ens.get("combine", "mean") + if combine == "weighted": + base = EnsembleForecaster(members, weights=ens.get("weights")) + elif combine == "stack": + from sktime.forecasting.compose import StackingForecaster + + base = StackingForecaster(members) + else: # mean | median | min | max + base = EnsembleForecaster(members, aggfunc=combine) + else: + _, base = _resolve_estimator(recipe["estimator"], store) + + # 2) optional transforms with automatic inverse (AutoAI-TS reverse-order, via sktime) + transforms = recipe.get("transforms") or [] + if transforms: + from sktime.forecasting.compose import TransformedTargetForecaster + from ..substrate import resolver as R + + steps = [] + for i, t in enumerate(transforms): + steps.append((t.get("name", f"t{i}"), R.resolve(t))) + steps.append(("forecaster", base)) + base = TransformedTargetForecaster(steps) + + # 3) optional conformal / probabilistic wrapper — adds calibrated prediction intervals to + # ANY recipe (single, ensemble, or transformed) via sktime ConformalIntervals. + if recipe.get("conformal"): + from sktime.forecasting.conformal import ConformalIntervals + + base = ConformalIntervals(base) + return base + + +def _backtest(forecaster, y, recipe): + from sktime.split import ExpandingWindowSplitter + from sktime.forecasting.model_evaluation import evaluate + + ev = recipe.get("eval", {}) + fh = recipe.get("fh", [1, 2, 3, 4, 5]) + iw = ev.get("initial_window", max(len(y) // 2, 2 * len(fh))) + step = ev.get("step", max(len(fh), 1)) + metric = _metric((ev.get("metrics") or ["smape"])[0]) + cv = ExpandingWindowSplitter(initial_window=iw, step_length=step, fh=fh) + res = evaluate(forecaster=forecaster, y=y, cv=cv, scoring=metric) + col = [c for c in res.columns if c.startswith("test_")][0] + return float(res[col].mean()), metric.__class__.__name__, len(res) + + +def _backtest_zero_shot(forecaster, y, recipe): + """Fast path for pretrained models: NO retraining loop. One rolling-origin holdout — + set the context = all-but-last-h, run inference once, score the held-out tail. Because a + zero-shot model isn't trained, re-fitting it across many expanding folds only repeats + inference; a single holdout is the honest, cheap estimate.""" + fh = recipe.get("fh", [1, 2, 3, 4, 5]) + h = len(fh) + y_train, y_test = y.iloc[:-h], y.iloc[-h:] + metric = _metric((recipe.get("eval", {}).get("metrics") or ["smape"])[0]) + forecaster.fit(y_train, fh=fh) # fit = load weights + set context + y_pred = np.asarray(forecaster.predict())[:h] + yt = np.asarray(y_test)[: len(y_pred)] + try: + score = float(metric(yt, y_pred, y_train=np.asarray(y_train))) # MASE etc. + except TypeError: + score = float(metric(yt, y_pred)) + return score, metric.__class__.__name__, 1 + + +def _recipe_regime(recipe, store) -> str: + """Cheapest regime that covers the whole recipe: zero_shot only if every estimator is + pretrained-zero-shot and no fine-tune is requested; else fit_on_series (or fine_tune). + """ + from ..substrate import resolver as R + + if recipe.get("finetune"): + return "fine_tune" + specs = ( + recipe["ensemble"]["members"] if "ensemble" in recipe else [recipe["estimator"]] + ) + regimes = set() + for s in specs: + card = s + if s.get("model_id") and store is not None: + from ..stores import model_store + + card = model_store.get_model(store, s["model_id"]) or s + merged = { + **card, + "params": {**(card.get("params") or {}), **(s.get("params") or {})}, + } + regimes.add(R.training_regime(merged)) + if "fine_tune" in regimes: + return "fine_tune" + return "zero_shot" if regimes == {"zero_shot"} else "fit_on_series" + + +def _validate_blocks(recipe: dict) -> dict: + """Validate any finetune/anomaly recipe blocks against param_space hints (non-fatal): the + audit is recorded so the agent's run-time choices are graded and bad values surface. + """ + from ..reasoning import param_space as PS + + audit = {} + for block in ("finetune", "anomaly"): + if isinstance(recipe.get(block), dict): + audit[block] = PS.validate_block(block, recipe[block]) + return audit + + +def run_recipe( + store, + y, + recipe: dict, + *, + asset_id: str = "asset", + parent_run_id: Optional[str] = None, + scenario_id: Optional[str] = None, +) -> dict: + """Compile → backtest → final forecast → per-member diagnostics → persist run (with lineage). + Dispatches by recipe['task']: anomaly_detection routes to the detector path (run_anomaly); + everything else is forecasting.""" + if recipe.get("task") == "tsfm_anomaly_detection": + return run_anomaly( + store, + y, + recipe, + asset_id=asset_id, + parent_run_id=parent_run_id, + scenario_id=scenario_id, + ) + + # normalize to plain float64 (nullable dtypes keep pd.NA, which crashes int() inside sktime) + y = pd.Series(np.asarray(y, dtype=float)) + regime = _recipe_regime(recipe, store) + # missing values: apply an EXPLICIT recipe['impute'] (interpolate|drop|zero); otherwise, if a + # classical forecaster is asked to fit a gapped series, fail clearly. Foundation/zero-shot + # models tolerate gaps, so they proceed untouched. + if recipe.get("impute"): + y = _impute(y, recipe["impute"]) + elif y.isna().any() and regime != "zero_shot": + raise ValueError( + "target series has missing values; set recipe['impute'] to " + "'interpolate', 'drop', or 'zero' (classical forecasters cannot fit gaps)." + ) + fc = build_forecaster(recipe, store) + block_audit = _validate_blocks(recipe) + _bt = _backtest_zero_shot if regime == "zero_shot" else _backtest + score, metric_name, folds = _bt(fc, y, recipe) + + # per-member diagnostics (so the agent can drop/reweight) — only for ensembles + per_member = {} + if "ensemble" in recipe: + for m in recipe["ensemble"]["members"]: + name, est = _resolve_estimator(m, store) + try: + per_member[name] = round(_bt(est, y, recipe)[0], 4) + except Exception as e: + per_member[name] = f"err:{type(e).__name__}" + + fh = recipe.get("fh", [1, 2, 3, 4, 5]) + fc.fit(y, fh=fh) + forecast = fc.predict().round(4).tolist() + intervals = None + if recipe.get("conformal"): + cov = ( + recipe["conformal"].get("coverage", 0.9) + if isinstance(recipe["conformal"], dict) + else 0.9 + ) + try: + intervals = fc.predict_interval(coverage=cov).round(4).head(3).to_dict() + except Exception as e: + intervals = {"error": str(e)[:80]} + + run_id = f"run:{uuid.uuid4().hex[:10]}" + rec = { + "_id": run_id, + "run_id": run_id, + "parent_run_id": parent_run_id, + "asset_id": asset_id, + "scenario_id": scenario_id, + "recipe": recipe, + "training_regime": regime, + "trained": regime != "zero_shot", + "block_audit": block_audit, + "metric": metric_name, + "backtest_score": round(score, 4), + "folds": folds, + "per_member_score": per_member, + "forecast_head": forecast[:5], + "prediction_interval": intervals, + "created_at": _now(), + } + if store is not None: + store.put(RUNS, rec) + return { + "run_id": run_id, + "task": "tsfm_forecasting", + "backtest_score": round(score, 4), + "metric": metric_name, + "training_regime": regime, + "trained": regime != "zero_shot", + "block_audit": block_audit, + "per_member_score": per_member, + "forecast_head": forecast[:5], + "prediction_interval": intervals, + "parent_run_id": parent_run_id, + "improved": ( + parent_run_id is not None + and store is not None + and score + < (store.get(RUNS, parent_run_id) or {}).get("backtest_score", 9e9) + ), + } + + +TABULAR_TASKS = {"tsfm_regression", "tsfm_classification", "tsfm_clustering"} + + +def _as_panel(X): + """Accept (n, T) univariate or (n, C, T) multivariate → (n, C, T).""" + X = np.asarray(X, float) + return X[:, None, :] if X.ndim == 2 else X + + +def _tile(x, window: Optional[int]) -> np.ndarray: + """1D series -> (n_windows, window_len). window None/<=0/>=len => a single whole-series window; + otherwise NON-OVERLAPPING tiles of length `window` (trailing remainder dropped).""" + x = np.asarray(x, dtype=float) + if window is None or window <= 0 or window >= len(x): + return x[None, :] + n = len(x) // window + return x[: n * window].reshape(n, window) if n else x[None, :] + + +def extract_features(channels: Dict[str, Any], extractor_names, window: Optional[int] = None): + """Apply named FLOps extractors to each channel's windows. + channels: {column_name -> 1D array}. Returns (columns, matrix) where matrix is + n_windows x (n_channels * n_extractors); column names are '.' when + multivariate, else just ''. Whole-series => one row.""" + from ..reasoning import feature_selection as FS + + multi = len(channels) > 1 + per = {ch: _tile(x, window) for ch, x in channels.items()} + nw = min((W.shape[0] for W in per.values()), default=0) + cols, data = [], [] + for ch, W in per.items(): + W = W[:nw] + for name in extractor_names: + fn = FS.EXTRACTORS[name] + data.append([float(fn(W[i])) for i in range(nw)]) + cols.append(f"{ch}.{name}" if multi else name) + F = np.nan_to_num(np.column_stack(data)) if data else np.zeros((nw, 0)) + return cols, F + +def _lib_features(X, subset=None): + """Dependency-free 'FeatureUnion': apply the FLOps extractor library per instance/channel.""" + from ..reasoning import feature_selection as FS + + names = [n for n in (subset or FS.EXTRACTORS) if n in FS.EXTRACTORS] + Xp = _as_panel(X) + cols, colnames = [], [] + for c in range(Xp.shape[1]): + for n in names: + cols.append([FS.EXTRACTORS[n](Xp[i, c]) for i in range(Xp.shape[0])]) + colnames.append(f"c{c}.{n}" if Xp.shape[1] > 1 else n) + F = np.nan_to_num(np.column_stack(cols)) if cols else np.zeros((len(Xp), 0)) + return F, colnames + + +def _tabular_features(X, recipe, y=None): + """Compile the recipe's transforms into a tabular feature matrix (the FeatureUnion). + A transform spec may be: {extractors:[names]} (our library), {sktime_class:...} (an sktime + panel transformer, soft-deps permitting), or {flops_select:true} (extract the full library + then keep columns the multi-config FLOps scorers rank above the rest).""" + transforms = recipe.get("transforms") + if not transforms: + return _lib_features(X) # default = full library + parts, names = [], [] + for t in transforms: + if t.get("flops_select"): + from ..reasoning import feature_selection as FS + + F, nm = _lib_features(X) + if y is not None: + keep = _flops_columns(F, np.asarray(y), nm, t.get("flops_select")) + idx = [nm.index(k) for k in keep] + F, nm = F[:, idx], keep + parts.append(F) + names += nm + elif t.get("extractors"): + F, nm = _lib_features(X, t["extractors"]) + parts.append(F) + names += nm + elif t.get("sktime_class"): + from ..substrate import resolver as R + + tr = R.resolve(t) + Ft = np.nan_to_num( + np.asarray(tr.fit_transform(_as_panel(X))).reshape(len(X), -1) + ) + parts.append(Ft) + names += [ + f"{t['sktime_class'].split('.')[-1]}_{k}" for k in range(Ft.shape[1]) + ] + F = np.column_stack(parts) if parts else _lib_features(X)[0] + return np.nan_to_num(F), names + + +def _flops_columns(F, y, names, cfg): + """Reuse the v2 multi-config scorers to select tabular feature COLUMNS by relevance to y.""" + from ..reasoning import feature_selection as FS + + scorers = (cfg.get("scorers") if isinstance(cfg, dict) else None) or [ + "corr", + "f_test", + "mutual_info", + "model", + ] + use = [s for s in scorers if s in FS._SCORERS] or ["corr"] + ranks = np.zeros(len(names)) + for s in use: + sv = np.asarray(FS._SCORERS[s](F, y, names)) + order = (-sv).argsort() + rk = np.empty(len(names)) + rk[order] = np.arange(1, len(names) + 1) + ranks += rk + mean_rank = ranks / len(use) + cut = (cfg.get("top_k") if isinstance(cfg, dict) else None) or max( + 1, len(names) // 2 + ) + keep_idx = np.argsort(mean_rank)[:cut] + return [names[i] for i in sorted(keep_idx)] + + +def run_tabular_recipe( + store, + X, + recipe: dict, + *, + y=None, + asset_id: str = "asset", + parent_run_id: Optional[str] = None, + scenario_id: Optional[str] = None, +) -> dict: + """Series→tabular run path: FeatureUnion(extractors) → estimator, for regression / + classification / clustering. CV-scored (supervised) or silhouette (clustering). Same recipe + grammar + persistence + lineage as forecasting; the agent mixes-and-matches features here too. + """ + from ..substrate import resolver as R + from sklearn.model_selection import cross_val_score + + task = recipe.get("task", "tsfm_classification") + F, feat_names = _tabular_features(X, recipe, y) + regime = ( + _recipe_regime(recipe, store) + if "estimator" in recipe or "ensemble" in recipe + else "fit_on_series" + ) + _, est = _resolve_estimator(recipe["estimator"], store) + + metric, score = None, None + if task == "tsfm_clustering": + from sklearn.metrics import silhouette_score + + labels = ( + est.fit_predict(F) if hasattr(est, "fit_predict") else est.fit(F).predict(F) + ) + try: + score, metric = round(float(silhouette_score(F, labels)), 4), "silhouette" + except Exception: + score, metric = None, "silhouette" + preds = np.asarray(labels)[:10].tolist() + else: + y = np.asarray(y) + scoring = "accuracy" if task == "tsfm_classification" else "r2" + nfold = int( + min( + 5, + max( + 2, + ( + np.min(np.bincount(y)) + if task == "tsfm_classification" + else len(y) // 3 + ), + ), + ) + ) + try: + score = round( + float(cross_val_score(est, F, y, cv=nfold, scoring=scoring).mean()), 4 + ) + except Exception as e: + score = f"err:{type(e).__name__}" + metric = scoring + est.fit(F, y) + preds = np.asarray(est.predict(F))[:10].tolist() + + run_id = f"run:{uuid.uuid4().hex[:10]}" + rec = { + "_id": run_id, + "run_id": run_id, + "parent_run_id": parent_run_id, + "task": task, + "asset_id": asset_id, + "scenario_id": scenario_id, + "recipe": recipe, + "training_regime": regime, + "n_features": F.shape[1], + "feature_names": feat_names[:40], + "metric": metric, + "cv_score": score, + "predictions_head": preds, + "created_at": _now(), + } + if store is not None: + store.put(RUNS, rec) + return { + "run_id": run_id, + "task": task, + "metric": metric, + "cv_score": score, + "n_features": F.shape[1], + "feature_names": feat_names[:40], + "predictions_head": preds, + "training_regime": regime, + "parent_run_id": parent_run_id, + } + + +def _anomaly_labels(pred, n: int): + """Normalize an sktime detector's predict() to a dense 0/1 label vector + anomaly indices. + sktime detection returns either anomaly POSITIONS (a DataFrame/array of ilocs) or a dense + 0/1(-1) series — handle both.""" + arr = np.asarray(getattr(pred, "values", pred)).ravel() + labels = np.zeros(n, dtype=int) + uniq = set(np.unique(arr).tolist()) if arr.size else set() + if arr.size == n and uniq.issubset({0, 1, -1}): # dense labels + labels = (arr != 0).astype(int) + else: # anomaly positions + idx = arr[(arr >= 0) & (arr < n)].astype(int) + labels[idx] = 1 + return labels, np.where(labels == 1)[0].tolist() + + +def _conformal_ad( + store, y, recipe: dict, *, asset_id, parent_run_id, scenario_id +) -> dict: + """Prediction-based AD with CONFORMAL intervals: fit a forecaster + sktime ConformalIntervals + on the history, predict a calibrated band over the recent window, flag points whose actual + value falls OUTSIDE the band as anomalies. The forecaster is any catalog card (zero-shot TTM, + classical, …); coverage = recipe.conformal.coverage (false-alarm = 1 − coverage).""" + from ..substrate import resolver as R + from sktime.forecasting.conformal import ConformalIntervals + + y = pd.Series(np.asarray(y, float)) if not isinstance(y, pd.Series) else y + impute = recipe.get("impute") or (recipe.get("anomaly") or {}).get("impute") + if impute: + y = _impute(y, impute) + spec = recipe.get("estimator") + if not spec: + raise ValueError("conformal AD needs an 'estimator' (a forecaster card)") + card = spec + if spec.get("model_id") and store is not None: + card = model_store_get(store, spec["model_id"]) or spec + merged = { + **card, + "params": {**(card.get("params") or {}), **(spec.get("params") or {})}, + } + regime = R.training_regime(merged) + coverage = float((recipe.get("conformal") or {}).get("coverage", 0.9)) + fh = recipe.get("fh") or list( + range(1, max(2, len(y) // 5) + 1) + ) # recent window to screen + H = len(fh) + y_train, y_test = y.iloc[:-H], y.iloc[-H:] + + ci = ConformalIntervals(R.resolve(merged)) + ci.fit(y_train, fh=fh) + pi = ci.predict_interval(coverage=coverage) + low = np.asarray(pi.iloc[:, 0])[:H] + high = np.asarray(pi.iloc[:, 1])[:H] + actual = np.asarray(y_test)[:H] + out = (actual < low) | (actual > high) + labels = np.zeros(len(y), dtype=int) + labels[len(y) - H :][out] = 1 + indices = np.where(labels == 1)[0].tolist() + n_anom = int(labels.sum()) + + run_id = f"run:{uuid.uuid4().hex[:10]}" + rec = { + "_id": run_id, + "run_id": run_id, + "parent_run_id": parent_run_id, + "asset_id": asset_id, + "scenario_id": scenario_id, + "task": "tsfm_anomaly_detection", + "method": "conformal", + "recipe": recipe, + "training_regime": regime, + "coverage": coverage, + "n_anomalies": n_anom, + "n_observations": len(y), + "anomaly_indices": indices[:200], + "created_at": _now(), + } + if store is not None: + store.put(RUNS, rec) + return { + "run_id": run_id, + "task": "tsfm_anomaly_detection", + "method": "conformal", + "n_anomalies": n_anom, + "n_observations": len(y), + "anomaly_indices_head": indices[:20], + "labels": labels.tolist(), + "training_regime": regime, + "coverage": coverage, + "parent_run_id": parent_run_id, + } + + +def run_anomaly( + store, + y, + recipe: dict, + *, + asset_id: str = "asset", + parent_run_id: Optional[str] = None, + scenario_id: Optional[str] = None, +) -> dict: + """Anomaly run path. method='detector' (default): a detector card → fit → predict → dense + labels (TSPulse zero-shot, SubLOF, PyOD). method='conformal': prediction-based AD — a + forecaster + sktime ConformalIntervals → flag out-of-band points.""" + if recipe.get("method") == "conformal": + return _conformal_ad( + store, + y, + recipe, + asset_id=asset_id, + parent_run_id=parent_run_id, + scenario_id=scenario_id, + ) + from ..substrate import resolver as R + + y = pd.Series(np.asarray(y, float)) if not isinstance(y, pd.Series) else y + spec = recipe.get("estimator") or recipe.get("detector") + if not spec: + raise ValueError("anomaly recipe needs an 'estimator' (a detector card)") + card = spec + if spec.get("model_id") and store is not None: + card = model_store_get(store, spec["model_id"]) or spec + merged = { + **card, + "params": {**(card.get("params") or {}), **(spec.get("params") or {})}, + } + regime = R.training_regime(merged) + block_audit = _validate_blocks(recipe) + + det = R.resolve(merged) + n = len(y) + impute = recipe.get("impute") or (recipe.get("anomaly") or {}).get("impute") + if impute == "drop": + y_fit = y.dropna() + kept = np.where(y.notna().to_numpy())[ + 0 + ] # original positions retained after drop + elif impute: + y_fit = _impute(y, impute) # interpolate | zero (length preserved) + kept = np.arange(n) + else: + y_fit = y # no imputation → let the model decide + kept = np.arange(n) + + try: + det.fit(y_fit) + _, raw_idx = _anomaly_labels(det.predict(y_fit), len(y_fit)) + except ValueError as e: # surface the model's own NaN error + hint + if "nan" in str(e).lower() and not impute: + raise ValueError( + f"{type(det).__name__} received missing values and cannot handle NaN: {e} " + "Set recipe['impute'] to 'interpolate', 'drop', or 'zero'." + ) from e + raise + + labels = np.zeros(n, dtype=int) # map detector output back to ORIGINAL length + orig = kept[np.asarray(raw_idx, dtype=int)] if raw_idx else np.array([], dtype=int) + labels[orig] = 1 + indices = orig.tolist() + n_anom = int(labels.sum()) + + run_id = f"run:{uuid.uuid4().hex[:10]}" + rec = { + "_id": run_id, + "run_id": run_id, + "parent_run_id": parent_run_id, + "asset_id": asset_id, + "scenario_id": scenario_id, + "task": "tsfm_anomaly_detection", + "recipe": recipe, + "training_regime": regime, + "trained": regime != "zero_shot", + "block_audit": block_audit, + "n_anomalies": n_anom, + "n_observations": len(y), + "anomaly_indices": indices[:200], + "created_at": _now(), + } + if store is not None: + store.put(RUNS, rec) + return { + "run_id": run_id, + "task": "tsfm_anomaly_detection", + "n_anomalies": n_anom, + "n_observations": len(y), + "anomaly_indices_head": indices[:20], + "labels": labels.tolist(), + "training_regime": regime, + "block_audit": block_audit, + "parent_run_id": parent_run_id, + } + + +def model_store_get(store, model_id): + from ..stores import model_store + + return model_store.get_model(store, model_id) + + +def discover_components(store=None, task: str = "tsfm_forecasting") -> dict: + """Everything the agent can mix-and-match for a task — including the TRAINING REGIME of each + option, so the agent can reason cost vs. benefit and prefer zero-shot (no training) first. + """ + from ..substrate import resolver as R + + fnd = R.foundation_forecasters() + out = { + "task": task, + "scitype": R.TASK_TO_SCITYPE.get(task), + "models_installed": R.discover(R.TASK_TO_SCITYPE.get(task, "forecaster"))[:50], + "foundation_models": fnd, # all zero-shot capable + "zero_shot_models": fnd, # default: predict with NO training + "combiners": ["mean", "median", "min", "max", "weighted", "stack"], + "metrics": ["smape", "mape", "mae", "mase"], + "splitters": ["expanding", "sliding"], + "training_regimes": { + "zero_shot": "pretrained foundation model; fit() only loads weights + sets " + "context, no training on your data — cheapest, the default path", + "fit_on_series": "classical model (AutoARIMA/ETS/Theta/reduction) estimated on " + "the series' own history — cheap, no separate training set", + "fine_tune": "OPTIONAL escalation: adapt a foundation model on the data (pass " + "fine-tune params or recipe.finetune) — most expensive, rarely needed", + }, + "regime_hint": "try zero_shot first; escalate to fine_tune only if evaluate() warrants", + } + from ..reasoning import param_space as PS + + out["recipe_blocks"] = { # run-time params the agent reasons + "finetune": PS.FINETUNE_HINTS, + "anomaly": PS.ANOMALY_HINTS, + } + from ..core import glossary as _glossary # teach the vocabulary inline + + g = _glossary.glossary() + out["glossary"] = g["terms"] + out["workflow"] = g["workflow"] + out["principles"] = g["principles"] + if store is not None: + from ..stores import model_store + from ..stores import feature_store + + models = model_store.list_models(store, task_id=task) + out["catalog_models"] = [ + {"model_id": m["model_id"], "training_regime": R.training_regime(m)} + for m in models + ] + out["transforms"] = [ + f["feature_id"] for f in feature_store.find_features(store) + ] + return out diff --git a/src/servers/tsfm/engine/evolve.py b/src/servers/tsfm/engine/evolve.py new file mode 100644 index 000000000..eb5cd5421 --- /dev/null +++ b/src/servers/tsfm/engine/evolve.py @@ -0,0 +1,225 @@ +"""evolve.py — the AlphaEvolve loop, adapted to the TSFM server. + +AlphaEvolve (DeepMind, 2506.13131): an LLM proposes edits to a *program*, an automatic +*evaluator* scores it, and a MAP-Elites/island *database* keeps a diverse set of elites; the +loop repeats. We adopt the same machinery with one principle preserved: **the server does NOT +call an LLM** — the agent is the proposer. The server scaffolds the loop: + + evolve_ask(task) → sample parent(s) + diverse inspirations + data evidence + the task contract + → the AGENT mutates them into a new candidate (recipe OR feature program). + evolve_tell(cand) → VALIDATE (EFE gate / recipe shape) → EVALUATE (run_recipe / run_tabular / + feature gate → a scalar FITNESS) → place in the MAP-Elites cell / island, + keep the elite, record lineage (parent, generation). + evolve_best(task) → the current elites (best per behavior cell). + +The "program" (EVOLVE-BLOCK) is either a **recipe** (declarative compose spec) or an EFE +**feature program** (fit/transform/inverse code). Fitness is always higher-is-better. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import Dict, List, Optional + +import numpy as np +import pandas as pd + +ARCHIVE = "tsfm_evolve" +N_ISLANDS = 4 +_FORECAST = "tsfm_forecasting" +_TABULAR = {"tsfm_regression", "tsfm_classification", "tsfm_clustering"} + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +# ─────────────────────────────── fitness + descriptor ─────────────────────── +def _bucket(n: int) -> str: + return "1" if n <= 1 else ("2-3" if n <= 3 else "4+") + + +def _descriptor(store, task: str, kind: str, program: dict) -> str: + """Behaviour cell key (MAP-Elites): coarse structural signature of the candidate.""" + if kind == "feature": + out = program.get("output_type", "?") + inv = int(bool(program.get("invertible"))) + return f"feat|{task}|out={out}|inv={inv}" + from . import composition as C + regime = C._recipe_regime(program, store) + nt = len(program.get("transforms") or []) + ens = 1 if "ensemble" in program else 0 + nm = len(program["ensemble"]["members"]) if ens else 1 + return f"rec|{task}|reg={regime}|nc={_bucket(nt + nm)}|ens={ens}" + + +def _fitness_from_run(task: str, run: dict) -> float: + """Map a run result to a scalar fitness (higher = better).""" + if task == _FORECAST: + return -float(run["backtest_score"]) # smape/mase: lower is better + cv = run.get("cv_score") + return float(cv) if isinstance(cv, (int, float)) else -1e9 # acc/r2/silhouette: higher better + + +# ─────────────────────────────── evaluation ───────────────────────────────── +def _evaluate(store, task: str, kind: str, program: dict, *, data_ref=None, + timestamp_column=None, target_columns=None, label_column=None) -> dict: + """Validate + evaluate a candidate → {valid, fitness, raw, summary}. Reuses the engine.""" + from ..io import refs + if kind == "feature": + return _evaluate_feature(program, data_ref=data_ref) + from . import composition as C + if not isinstance(program, dict) or ("estimator" not in program and "ensemble" not in program): + return {"valid": False, "reason": "recipe must include an 'estimator' or an 'ensemble'"} + if task == _FORECAST: + if not data_ref or not target_columns: + return {"valid": False, "reason": "forecasting eval needs data_ref + target_columns"} + try: + obj = refs.load_series(data_ref, time_col=timestamp_column, channels=target_columns) + series = obj.iloc[:, 0] if isinstance(obj, pd.DataFrame) else obj + run = C.run_recipe(store, series, program) + except Exception as exc: + return {"valid": False, "reason": f"eval failed: {type(exc).__name__}: {str(exc)[:120]}"} + return {"valid": True, "fitness": _fitness_from_run(task, run), "raw": run["backtest_score"], + "summary": {k: run.get(k) for k in ("metric", "backtest_score", "training_regime")}} + if task in _TABULAR: + if not data_ref: + return {"valid": False, "reason": "tabular eval needs data_ref"} + try: + df = pd.read_csv(refs._path(data_ref)) + y = None + if label_column and label_column in df.columns: + y = df[label_column].to_numpy(); df = df.drop(columns=[label_column]) + X = df.apply(pd.to_numeric, errors="coerce").fillna(0.0).to_numpy() + run = C.run_tabular_recipe(store, X, {**program, "task": task}, y=y) + except Exception as exc: + return {"valid": False, "reason": f"eval failed: {type(exc).__name__}: {str(exc)[:120]}"} + cv = run.get("cv_score") + if not isinstance(cv, (int, float)): + return {"valid": False, "reason": f"eval failed: {cv}"} + return {"valid": True, "fitness": float(cv), "raw": cv, + "summary": {k: run.get(k) for k in ("task", "metric", "cv_score", "n_features")}} + return {"valid": False, "reason": f"unsupported task '{task}'"} + + +def _evaluate_feature(program: dict, *, data_ref=None) -> dict: + """Validate an EFE feature program through the gate; fitness rewards a clean, invertible op.""" + from . import feature_runner as FR + from ..io import refs + if data_ref: + obj = refs.load_series(data_ref) + X = (obj.to_frame() if isinstance(obj, pd.Series) else obj).to_numpy(dtype=float) + else: + X = np.sin(np.arange(128) / 5.0).reshape(-1, 1) # synthetic probe + try: + res = FR.validate_and_run(program, X, X, metadata=program.get("params") or {}) + except Exception as exc: + return {"valid": False, "reason": f"validity gate failed: {exc}"} + checks = res.get("checks", {}) + fitness = 1.0 + (0.5 if checks.get("invertible_ok") else 0.0) + return {"valid": True, "fitness": fitness, "raw": fitness, + "summary": {"checks": checks, "invertible_ok": checks.get("invertible_ok")}} + + +# ─────────────────────────────── archive ops ──────────────────────────────── +def _elites(store, task: Optional[str] = None, kind: Optional[str] = None) -> List[dict]: + """Best candidate per behaviour cell (the MAP-Elites grid), optionally filtered.""" + sel = {} + if task: + sel["task"] = task + if kind: + sel["kind"] = kind + cells: Dict[str, dict] = {} + for d in store.find(ARCHIVE, sel): + c = d["cell"] + if c not in cells or d["fitness"] > cells[c]["fitness"]: + cells[c] = d + return sorted(cells.values(), key=lambda d: d["fitness"], reverse=True) + + +def evolve_tell(store, task: str, kind: str, program: dict, *, parent_id: Optional[str] = None, + data_ref=None, timestamp_column=None, target_columns=None, + label_column=None) -> dict: + """Validate + evaluate the agent's candidate and place it in the archive (with lineage).""" + if kind not in ("recipe", "feature"): + return {"accepted": False, "reason": "kind must be 'recipe' or 'feature'"} + ev = _evaluate(store, task, kind, program, data_ref=data_ref, + timestamp_column=timestamp_column, target_columns=target_columns, + label_column=label_column) + if not ev.get("valid"): + return {"accepted": False, "reason": ev.get("reason", "invalid candidate")} + + cell = _descriptor(store, task, kind, program) + parent = store.get(ARCHIVE, parent_id) if parent_id else None + generation = int(parent["generation"]) + 1 if parent else 0 + island = parent["island"] if parent else len(store.find(ARCHIVE, {})) % N_ISLANDS + incumbent = next((d for d in _elites(store, task, kind) if d["cell"] == cell), None) + is_new_elite = incumbent is None or ev["fitness"] > incumbent["fitness"] + + eid = f"evo:{uuid.uuid4().hex[:10]}" + doc = {"_id": eid, "evolve_id": eid, "task": task, "kind": kind, "program": program, + "cell": cell, "island": island, "generation": generation, "parent_id": parent_id, + "fitness": round(float(ev["fitness"]), 6), "raw_score": ev.get("raw"), + "summary": ev.get("summary"), "created_at": _now()} + store.put(ARCHIVE, doc) + return {"accepted": True, "evolve_id": eid, "cell": cell, "island": island, + "generation": generation, "fitness": doc["fitness"], "is_new_elite": is_new_elite, + "incumbent_fitness": incumbent["fitness"] if incumbent else None, + "summary": ev.get("summary")} + + +def evolve_ask(store, task: str, *, kind: str = "recipe", data_ref=None, + timestamp_column=None, channels=None, n_parents: int = 2, + n_inspirations: int = 3) -> dict: + """Sample parents + diverse inspirations + data evidence so the agent can propose a mutation.""" + elites = _elites(store, task, kind) + parents = elites[:n_parents] + used = {d["cell"] for d in parents} + inspirations, seen_islands = [], set() + for d in elites[n_parents:]: # diversity: distinct cells/islands + if d["cell"] in used: + continue + if d["island"] in seen_islands and len(inspirations) >= 1: + continue + inspirations.append(d); used.add(d["cell"]); seen_islands.add(d["island"]) + if len(inspirations) >= n_inspirations: + break + + evidence = None + if data_ref: + from ..reasoning import profile + try: + evidence = profile.profile_ref(data_ref, timestamp_column=timestamp_column, channels=channels) + except Exception as exc: + evidence = {"error": str(exc)[:120]} + + from ..core import tasks as task_spec + contract = task_spec.get_task(task).__dict__ if task in task_spec.TASKS else None + seed = not elites + return { + "task": task, "kind": kind, "archive_size": len(store.find(ARCHIVE, {"task": task})), + "parents": [_view(d) for d in parents], + "inspirations": [_view(d) for d in inspirations], + "evidence": evidence, "contract": contract, + "instructions": ( + "Propose ONE new candidate by mutating/recombining the parents (use inspirations for " + "diversity); return it to evolve_tell. " + ( + "Archive is empty — propose an initial zero-shot recipe from discover_components." + if seed else + "Aim to beat the parent fitness or fill a new behaviour cell (novel structure).")), + } + + +def _view(d: dict) -> dict: + """Compact archive entry for the agent (program + where it sits + how good).""" + return {"evolve_id": d["evolve_id"], "kind": d["kind"], "cell": d["cell"], + "island": d["island"], "generation": d["generation"], "fitness": d["fitness"], + "program": d["program"], "summary": d.get("summary")} + + +def evolve_best(store, task: str, *, kind: Optional[str] = None, top_k: int = 5) -> dict: + """The current elites (best per behaviour cell) for a task — the evolved frontier.""" + elites = _elites(store, task, kind)[:top_k] + return {"task": task, "kind": kind, "n_cells": len(_elites(store, task, kind)), + "elites": [_view(d) for d in elites]} diff --git a/src/servers/tsfm/engine/feature_runner.py b/src/servers/tsfm/engine/feature_runner.py new file mode 100644 index 000000000..c19190428 --- /dev/null +++ b/src/servers/tsfm/engine/feature_runner.py @@ -0,0 +1,96 @@ +"""Dynamic feature-transform runner — execute a stored EFE-style program. + +Inspired by Evolutionary Feature Engineering (EFE, NeurIPS'26): a feature transform is a +Python program with a standardized interface, inserted before a fixed downstream model. +EFE's contract (from the paper's Fig. 1): + + class Transformation: + def fit(self, X, metadata): ... # learn state from training data + def transform(self, X, state): return X_new + def inverse_transform(self, X_new, state): return X # for invertible TS norms + +This is the *feature-store* analog of the model runner: the program lives as CODE in the +catalog (written dynamically — by an LLM evolver or a human), and is loaded, validated, and +executed on demand. So a feature transform need not be a pre-installed library function any +more than a model needs to be pre-downloaded. + +EFE-style validity checks are enforced before use: required entry points, output is a NEW +object (no in-place mutation), schema consistency, and — when declared invertible — a +round-trip check. +""" + +from __future__ import annotations + +import copy +from typing import Any, Dict, Optional + + +class FeatureValidationError(Exception): + pass + + +def load_program(code: str, class_name: str = "Transformation"): + """Exec a stored program string and return its Transformation class. + + NOTE: executes code. In the benchmark/agent setting run this in a sandbox + (restricted globals, subprocess, or container). Here we exec in a fresh namespace + with a minimal import surface, matching how EFE executes candidate programs. + """ + ns: Dict[str, Any] = {} + try: + exec(code, ns) # noqa: S102 - intentional; sandbox in production + except Exception as e: + raise FeatureValidationError(f"program failed to import: {e}") + cls = ns.get(class_name) + if cls is None: + raise FeatureValidationError(f"no class '{class_name}' defined") + for m in ("fit", "transform"): + if not callable(getattr(cls, m, None)): + raise FeatureValidationError(f"missing required method '{m}'") + return cls + + +def _no_inplace(before, after) -> bool: + """transform must return a NEW object, not mutate its input (EFE check).""" + return after is not before + + +def validate_and_run(doc: dict, X_fit, X_in, metadata: Optional[dict] = None) -> dict: + """Load the program in `doc['code']`, fit on X_fit, transform X_in. + + Returns {output, state, invertible_ok, checks}. Mirrors EFE's evaluator: it does the + structural checks and (for invertible programs) verifies a round-trip, returning a + feedback dict the caller (or an LLM evolver) can act on. + """ + checks = {"entry_points": False, "no_inplace": False, "invertible_ok": None} + cls = load_program(doc["code"], doc.get("class_name", "Transformation")) + checks["entry_points"] = True + + inst = cls() + X_fit_guard = copy.deepcopy(X_fit) + state = inst.fit(X_fit, metadata or {}) + # fit may return state or store on self; support both + if state is None: + state = getattr(inst, "state_", inst) + + out = inst.transform(X_in, state) + checks["no_inplace"] = _no_inplace(X_in, out) + + # invertibility round-trip when declared and supported + if doc.get("invertible") and hasattr(inst, "inverse_transform"): + import numpy as np + recon = inst.inverse_transform(out, state) + try: + checks["invertible_ok"] = bool(np.allclose(np.asarray(recon, dtype=float), + np.asarray(X_in, dtype=float), + rtol=1e-4, atol=1e-6)) + except Exception: + checks["invertible_ok"] = None + + if not checks["no_inplace"]: + raise FeatureValidationError("transform mutated its input in place") + if doc.get("invertible") and checks["invertible_ok"] is False: + raise FeatureValidationError("declared invertible but round-trip failed") + + return {"output": out, "state": state, "checks": checks, + "feature_id": doc.get("feature_id"), "target_model": doc.get("target_model")} diff --git a/src/servers/tsfm/engine/plan.py b/src/servers/tsfm/engine/plan.py new file mode 100644 index 000000000..dbcc4b770 --- /dev/null +++ b/src/servers/tsfm/engine/plan.py @@ -0,0 +1,106 @@ +"""plan.py — the recipe DAG (HuggingGPT task-list, on sktime, with file-pointer resources). + +A PLAN is HuggingGPT's task list: + [{"id": "s1", "task": "forecast", "dep": [], "args": {"data_ref": "file://iot.csv", ...}, + "recipe": {...composition recipe...}}, + {"id": "s2", "task": "evaluate", "dep": ["s1"], "args": {"pred": "@s1", "truth_ref": "..."}}] + +- `args` carry **file pointers** (IoT data, `data_ref`) or **resource references** `@step_id` + that resolve to the output file pointer of a dependency step (HuggingGPT's `-id`). +- The server topologically executes the DAG: load inputs from file pointers, run the step on + the sktime substrate, write each output to a NEW file pointer, and pass refs downstream. +- Every step output and the run record are file pointers / CouchDB rows → state-exportable. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import Dict, List + +import numpy as np +import pandas as pd + +from ..io import refs as io_refs +from ..engine import composition as C +from ..eval import gifteval as G + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _toposort(steps: List[dict]) -> List[dict]: + by_id = {s["id"]: s for s in steps} + seen, order = set(), [] + def visit(s): + if s["id"] in seen: + return + for d in s.get("dep", []): + visit(by_id[d]) + seen.add(s["id"]); order.append(s) + for s in steps: + visit(s) + return order + + +def _resolve_arg(v, outputs: Dict[str, dict]): + """@step_id → that step's output file pointer; otherwise pass through (a file pointer/literal).""" + if isinstance(v, str) and v.startswith("@"): + return outputs[v[1:]]["ref"] + return v + + +def run_plan(store, plan: dict, *, asset_id: str = "asset", scenario_id: str = None) -> dict: + """Execute a recipe DAG. Returns per-step outputs (file pointers + summaries) + a run id.""" + steps = _toposort(plan["steps"]) + outputs: Dict[str, dict] = {} + plan_id = f"plan:{uuid.uuid4().hex[:10]}" + + for s in steps: + task = s["task"] + args = {k: _resolve_arg(v, outputs) for k, v in (s.get("args") or {}).items()} + + if task == "forecast": + y = io_refs.load_series(args["data_ref"], value_col=args.get("value_col"), + time_col=args.get("time_col"), channels=args.get("channels")) + fc = C.build_forecaster(s["recipe"], store) + fh = s["recipe"].get("fh", list(range(1, 13))) + fc.fit(pd.Series(np.asarray(y, float).ravel()), fh=fh) + pred = fc.predict() + ref = io_refs.write_series(pred, name=f"forecast_{s['id']}") + outputs[s["id"]] = {"ref": ref, "task": task, + "summary": {"horizon": len(fh), "head": pred.round(4).head(3).values.tolist()}} + + elif task == "anomaly": + X = io_refs.load_series(args["data_ref"], channels=args.get("channels")) + from ..substrate import resolver as R + det = R.resolve(s["recipe"]["estimator"]) + Xf = pd.DataFrame(np.asarray(X, float).reshape(len(X), -1)) + det.fit(Xf) + try: + labels = det.predict(Xf) + n_anom = int(np.asarray(labels).astype(bool).sum()) + except Exception as e: + labels, n_anom = None, f"err:{type(e).__name__}" + ref = io_refs.write_json({"labels": np.asarray(labels).tolist() if labels is not None else None}, name=f"anomaly_{s['id']}") + outputs[s["id"]] = {"ref": ref, "task": task, "summary": {"anomaly_count": n_anom}} + + elif task == "evaluate": + cfgs = args["configs"] if "configs" in args else [{ + "name": asset_id, "y": np.asarray(io_refs.load_series(args["data_ref"]), float).ravel().tolist(), + "fh": args.get("fh", list(range(1, 13))), "sp": args.get("sp", 1)}] + recipes = args.get("recipes") or {s["id"]: s.get("recipe", {})} + board = G.leaderboard(store, recipes, cfgs, by=args.get("by", "norm_mase")) + ref = io_refs.write_json(board, name=f"eval_{s['id']}") + outputs[s["id"]] = {"ref": ref, "task": task, "summary": board["leaderboard"][:3]} + + else: + outputs[s["id"]] = {"ref": None, "task": task, "summary": {"error": f"unknown task {task}"}} + + rec = {"_id": plan_id, "plan_id": plan_id, "asset_id": asset_id, "scenario_id": scenario_id, + "plan": plan, "outputs": {k: {"ref": v["ref"], "summary": v["summary"]} for k, v in outputs.items()}, + "created_at": _now()} + if store is not None: + store.put("tsfm_plans", rec) + return {"plan_id": plan_id, "steps": list(outputs), "outputs": rec["outputs"]} diff --git a/src/servers/tsfm/eval/__init__.py b/src/servers/tsfm/eval/__init__.py new file mode 100644 index 000000000..e757a609d --- /dev/null +++ b/src/servers/tsfm/eval/__init__.py @@ -0,0 +1 @@ +"""eval layer.""" diff --git a/src/servers/tsfm/eval/gifteval.py b/src/servers/tsfm/eval/gifteval.py new file mode 100644 index 000000000..b1db048e9 --- /dev/null +++ b/src/servers/tsfm/eval/gifteval.py @@ -0,0 +1,188 @@ +"""gifteval.py — GIFT-Eval-native evaluation for the composition loop. + +Adopts the Salesforce GIFT-Eval protocol (arXiv:2410.10393) as the server's scoring backbone: + * many CONFIGS = (dataset × frequency × horizon × {uni|multi}), not one test; + * point metric **MASE** + probabilistic metric **CRPS**; + * each config NORMALIZED by a **seasonal-naïve** baseline — which is exactly our Zero Model; + * aggregate across configs by the **geometric mean** of the normalized scores; + * a **leaderboard** that ranks recipes per config (by CRPS) and reports the **mean rank** + (so no single dataset dominates). + +So the agent's mix-and-match / ensemble search (composition.py) is judged GIFT-Eval style: +better aggregate-normalized CRPS/MASE and better mean rank — robust, scale-free, multi-config. +""" + +from __future__ import annotations + +import warnings +from typing import Callable, Dict, List, Optional + +import numpy as np +import pandas as pd + +warnings.filterwarnings("ignore") + + +# --------------------------------------------------------------------------- # +def _mase(y_true, y_pred, y_train, sp: int) -> float: + y_true, y_pred, y_train = map( + lambda a: np.asarray(a, float).ravel(), (y_true, y_pred, y_train) + ) + denom = ( + np.mean(np.abs(y_train[sp:] - y_train[:-sp])) + if len(y_train) > sp + else np.mean(np.abs(np.diff(y_train))) or 1e-8 + ) + return float(np.mean(np.abs(y_true - y_pred)) / (denom + 1e-12)) + + +def _crps_from_quantiles(y_true, qdf: pd.DataFrame, levels: List[float]) -> float: + """Empirical CRPS via the quantile (pinball) decomposition: CRPS ≈ 2·mean_k pinball_k.""" + y = np.asarray(y_true, float).ravel() + tot = 0.0 + for a in levels: + col = [c for c in qdf.columns if abs(c[-1] - a) < 1e-6] + if not col: + continue + q = np.asarray(qdf[col[0]], float).ravel()[: len(y)] + diff = y - q + tot += np.mean(np.where(diff >= 0, a * diff, (a - 1) * diff)) + return float(2.0 * tot / max(len(levels), 1)) + + +def _split(y: pd.Series, h: int): + return y.iloc[:-h], y.iloc[-h:] + + +# --------------------------------------------------------------------------- # +def evaluate_config( + build_forecaster: Callable, + recipe: dict, + y: pd.Series, + *, + fh: List[int], + sp: int, + quantile_levels=(0.1, 0.25, 0.5, 0.75, 0.9), +) -> dict: + """Fit on train, score on the held-out horizon: raw MASE (+ CRPS if probabilistic).""" + h = len(fh) + ytr, yte = _split(y, h) + fc = build_forecaster(recipe) + fc.fit(ytr, fh=fh) + pred = np.asarray(fc.predict(), float).ravel()[:h] + mase = _mase(yte, pred, ytr, sp) + crps = None + if recipe.get("conformal"): + try: + q = fc.predict_quantiles(alpha=list(quantile_levels)) + crps = _crps_from_quantiles(yte, q, list(quantile_levels)) + except Exception: + crps = None + return { + "mase": round(mase, 4), + "crps": (round(crps, 4) if crps is not None else None), + } + + +def seasonal_naive_scores( + y: pd.Series, *, fh: List[int], sp: int, quantile_levels=(0.1, 0.25, 0.5, 0.75, 0.9) +) -> dict: + """The GIFT-Eval normalizer = our Zero Model (seasonal naive).""" + from sktime.forecasting.naive import NaiveForecaster + from sktime.forecasting.conformal import ConformalIntervals + + h = len(fh) + ytr, yte = _split(y, h) + base = NaiveForecaster(strategy="last", sp=sp if sp > 1 else 1) + base.fit(ytr, fh=fh) + pred = np.asarray(base.predict(), float).ravel()[:h] + mase = _mase(yte, pred, ytr, sp) + try: + cf = ConformalIntervals( + NaiveForecaster(strategy="last", sp=sp if sp > 1 else 1) + ).fit(ytr, fh=fh) + crps = _crps_from_quantiles( + yte, + cf.predict_quantiles(alpha=list(quantile_levels)), + list(quantile_levels), + ) + except Exception: + crps = None + return { + "mase": round(mase, 4), + "crps": (round(crps, 4) if crps is not None else None), + } + + +def _geomean(vals: List[float]) -> Optional[float]: + vals = [v for v in vals if v is not None and v > 0] + return round(float(np.exp(np.mean(np.log(vals)))), 4) if vals else None + + +def evaluate_recipe(store, recipe: dict, configs: List[dict]) -> dict: + """GIFT-Eval a recipe across configs: per-config normalized MASE/CRPS + geo-mean aggregate. + + config = {"name", "y", "fh", "sp"}. Normalized score = recipe_metric / seasonal_naive_metric + (so <1 means 'beats seasonal naive', exactly GIFT-Eval's relative reporting).""" + from ..engine import composition as C + + bf = lambda r: C.build_forecaster(r, store) + rows, n_mase, n_crps = [], [], [] + for cfg in configs: + y = pd.Series(np.asarray(cfg["y"], float)) + sn = seasonal_naive_scores(y, fh=cfg["fh"], sp=cfg["sp"]) + sc = evaluate_config(bf, recipe, y, fh=cfg["fh"], sp=cfg["sp"]) + nm = round(sc["mase"] / sn["mase"], 4) if sn["mase"] else None + nc = round(sc["crps"] / sn["crps"], 4) if (sc["crps"] and sn["crps"]) else None + rows.append( + { + "config": cfg["name"], + "mase": sc["mase"], + "crps": sc["crps"], + "norm_mase": nm, + "norm_crps": nc, + } + ) + if nm: + n_mase.append(nm) + if nc: + n_crps.append(nc) + return { + "per_config": rows, + "agg": { + "geomean_norm_mase": _geomean(n_mase), + "geomean_norm_crps": _geomean(n_crps), + "n_configs": len(configs), + }, + } + + +def leaderboard( + store, recipes: Dict[str, dict], configs: List[dict], by: str = "norm_crps" +) -> dict: + """Rank recipes per config (lower=better) and report the MEAN RANK (GIFT-Eval aggregation).""" + evals = {name: evaluate_recipe(store, r, configs) for name, r in recipes.items()} + fallback = "norm_mase" if by == "norm_crps" else "norm_crps" + ranks = {name: [] for name in recipes} + for i, cfg in enumerate(configs): + scored = [] + for name in recipes: + row = evals[name]["per_config"][i] + v = row.get(by) if row.get(by) is not None else row.get(fallback) + scored.append((name, v if v is not None else float("inf"))) + scored.sort(key=lambda t: t[1]) + for rank, (name, _) in enumerate(scored, 1): + ranks[name].append(rank) + board = sorted( + ( + (name, round(float(np.mean(rk)), 3), evals[name]["agg"]) + for name, rk in ranks.items() + ), + key=lambda t: t[1], + ) + return { + "ranked_by": by, + "leaderboard": [ + {"recipe": n, "mean_rank": mr, "agg": agg} for n, mr, agg in board + ], + } diff --git a/src/servers/tsfm/forecasting.py b/src/servers/tsfm/forecasting.py deleted file mode 100644 index bcd2f2f3c..000000000 --- a/src/servers/tsfm/forecasting.py +++ /dev/null @@ -1,654 +0,0 @@ -"""TSFM inference, fine-tuning, and the data quality filter bridge. - -Heavy ML dependencies (tsfm_public, transformers, torch) are imported lazily -so the module can be imported even when they are absent. -""" - -from __future__ import annotations - -import math -import os -import pickle -import time -import yaml - -import numpy as np -import pandas as pd - -from .dataquality import ( - _df_dt_stats, - _df_nan_stats, - _dq_timeseries_segmentation, - _time_series_segment_quality_summary, -) -from .io import _make_json_compatible -from .metrics import _METRICS_FORECAST, _TSFREQUENCY_TOLERANCE, _freq_token_to_minutes - - -# ── TSFM data quality filter ────────────────────────────────────────────────── - - -def _tsfm_data_quality_filter( - df_dataframe, dataset_config_dictionary, model_config, task="inference" -): - timestamp_col = dataset_config_dictionary["column_specifiers"]["timestamp_column"] - data_col = [timestamp_col] - for columns_group in dataset_config_dictionary["column_specifiers"]: - if "_columns" in columns_group: - data_col.extend( - dataset_config_dictionary["column_specifiers"][columns_group] - ) - if "operation_on_column" in dataset_config_dictionary: - data_col.extend(dataset_config_dictionary["operation_on_column"]) - - df = df_dataframe[data_col].copy() - df[timestamp_col] = pd.to_datetime(df[timestamp_col]) - for col in data_col: - if col != timestamp_col: - df[col] = df[col].astype(float) - - time_intervals_dic = _df_dt_stats(df, date_col=timestamp_col, intervals_dic=None) - nans_dic = _df_nan_stats(df, perc_rows_less_than=[], perc_rows_more_than=[]) - - FILTERING_PARAMS: dict = {"nans": {"efficient_removal": {"preference_tie": "row"}}} - - frequency_minutes = None - if "frequency_sampling" in dataset_config_dictionary: - freq_str = dataset_config_dictionary["frequency_sampling"] - if freq_str: - assert freq_str in _freq_token_to_minutes, ( - f" frequency_sampling input does not belong to {list(_freq_token_to_minutes.keys())}, " - "select 'oov' to estimate it from the timestamps" - ) - frequency_minutes = _freq_token_to_minutes[freq_str] - - if frequency_minutes is None: - timestamps = pd.to_datetime(df[timestamp_col]) - time_diffs = timestamps.diff().dropna() - frequency_minutes = float(time_diffs.dt.total_seconds().div(60).median()) - - freq_lower = frequency_minutes - _TSFREQUENCY_TOLERANCE * frequency_minutes - freq_upper = frequency_minutes + _TSFREQUENCY_TOLERANCE * frequency_minutes - FILTERING_PARAMS["dt"] = {"lower_bound": freq_lower, "upper_bound": freq_upper} - - df = _dq_timeseries_segmentation( - df, filtering_params=FILTERING_PARAMS, timestamp_tag=timestamp_col - ) - - dataset_config = dataset_config_dictionary.copy() - dataset_config["id_columns"] = ["segment_id"] - for col_tag in dataset_config["column_specifiers"]: - if col_tag not in ("timestamp_column", "autoregressive_modeling"): - dataset_config["column_specifiers"][col_tag] = [ - c - for c in dataset_config["column_specifiers"][col_tag] - if c in df.columns - ] - - n_minimum = 1 - if task == "inference": - n_minimum = model_config["context_length"] - if task == "finetuning": - n_minimum = model_config["prediction_length"] + model_config["context_length"] - - group_sizes = df.groupby(dataset_config["id_columns"][0]).size() - large_groups = group_sizes[group_sizes >= n_minimum].index - df = df[df[dataset_config["id_columns"][0]].isin(large_groups)] - - ts_segments_quality_summary = _time_series_segment_quality_summary( - df, timestamp_col, dataset_config["id_columns"][0] - ) - ts_segments_quality_summary["removed_columns"] = [ - c for c in data_col if c not in df.columns - ] - ts_segments_quality_summary["frequency_sampling_min"] = frequency_minutes - - df = df.loc[:, ~df.columns.duplicated(keep="first")] - - return { - "data": df, - "dataset_config_dictionary": dataset_config, - "dataquality_summary": _make_json_compatible( - { - "original_data": { - "nans_summary": nans_dic, - "sampling_summary": time_intervals_dic, - }, - "filtered_data_ts_segments": ts_segments_quality_summary, - } - ), - } - - -# ── Inference helpers ───────────────────────────────────────────────────────── - - -def _get_gt_and_predictions( - trainer, dataset, ix_target_features, inverse_transforms=None -): - if inverse_transforms is None: - inverse_transforms = [] - outputs = trainer.predict(dataset) - target_value_list = [] - pred_value_list = [] - timestamp_id_value_dic: dict = {} - for i in range(len(dataset)): - aux = dataset[i]["future_values"][:, ix_target_features].detach().numpy() - if "timestamp" in dataset[i]: - timestamp_id_value_dic.setdefault("timestamp", []).append( - dataset[i]["timestamp"] - ) - if "id" in dataset[i]: - timestamp_id_value_dic.setdefault("id", []).extend(list(dataset[i]["id"])) - target_value_list.append(aux) - forecast_h = aux.shape[0] - aux_pred = outputs.predictions[0][ - i, :forecast_h, ix_target_features - ].transpose() - pred_value_list.append(aux_pred) - y_gt = np.array(target_value_list) - y_pred = np.array(pred_value_list) - for ix_fhorizon in range(y_gt.shape[1]): - if inverse_transforms: - y_gt[:, ix_fhorizon, :] = inverse_transforms[0](y_gt[:, ix_fhorizon, :]) - y_pred[:, ix_fhorizon, :] = inverse_transforms[0](y_pred[:, ix_fhorizon, :]) - return y_gt, y_pred, timestamp_id_value_dic - - -def _get_performance( - y_gt, - y_pred, - target_columns=None, - prediction=True, - inverse_transforms=None, - ts_mask=None, -): - if inverse_transforms is None: - inverse_transforms = [] - if ts_mask is None: - ts_mask = np.ones([y_gt.shape[0], y_gt.shape[1]]) - if not target_columns: - target_columns = list(np.arange(y_gt.shape[2])) - rows = [] - pd_prediction = pd.DataFrame() - pd_performance = pd.DataFrame() - for ix_target in range(y_gt.shape[2]): - for ix_fhorizon in range(y_gt.shape[1]): - if len(inverse_transforms) > ix_target: - y_gt[:, ix_fhorizon, ix_target] = inverse_transforms[ix_target]( - y_gt[:, ix_fhorizon, ix_target][:, np.newaxis] - )[:, 0] - y_pred[:, ix_fhorizon, ix_target] = inverse_transforms[ix_target]( - y_pred[:, ix_fhorizon, ix_target][:, np.newaxis] - )[:, 0] - pd_aux = pd.DataFrame( - { - "y_gt": y_gt[:, ix_fhorizon, ix_target], - "y_pred": y_pred[:, ix_fhorizon, ix_target], - "forecast_horizon": ix_fhorizon + 1, - "target": target_columns[ix_target], - "on_mask": ts_mask[:, ix_fhorizon], - } - ) - pd_prediction = pd.concat([pd_prediction, pd_aux], axis=0) - y_gt_mask = y_gt[:, ix_fhorizon, ix_target][ts_mask[:, ix_fhorizon] > 0] - y_pred_mask = y_pred[:, ix_fhorizon, ix_target][ts_mask[:, ix_fhorizon] > 0] - valid_mask = np.isfinite(y_gt_mask) & np.isfinite(y_pred_mask) - y_gt_mask = y_gt_mask[valid_mask] - y_pred_mask = y_pred_mask[valid_mask] - if y_gt_mask.shape[0] > 0: - for metric in _METRICS_FORECAST: - value = _METRICS_FORECAST[metric]( - y_gt[:, :ix_fhorizon, ix_target], - y_pred[:, :ix_fhorizon, ix_target], - axis=1, - ) - stat = np.mean(value) if value is not None else None - rows.append( - [target_columns[ix_target], ix_fhorizon + 1, metric, stat] - ) - if rows: - pd_performance = pd.DataFrame( - data=rows, columns=["target", "forecast", "metric", "value"] - ) - if prediction: - return pd_performance, pd_prediction - return pd_performance - - -def _get_ttm_hf_inference( - df_dataframe, - dataset_config_dictionary, - model_config, - model_checkpoint, - scaling=False, - tsp=None, - forecast_horizon=-1, -): - from tsfm_public import TinyTimeMixerForPrediction - from tsfm_public.toolkit.time_series_preprocessor import ( - TimeSeriesPreprocessor, - get_datasets, - create_timestamps, - ) - from transformers import Trainer, TrainingArguments - - if forecast_horizon == -1: - forecast_horizon = model_config["prediction_length"] - else: - assert forecast_horizon <= model_config["prediction_length"], ( - f" Selected forecast horizon is above what is supported by the model. " - f"Set a forecast horizon smaller than {model_config['prediction_length']}" - ) - context_length = model_config["context_length"] - assert context_length <= len(df_dataframe), ( - " length of dataframe needs to be larger or equal to context length" - ) - - column_specifiers = dataset_config_dictionary["column_specifiers"] - if ( - "id_columns" in dataset_config_dictionary - and "id_columns" not in column_specifiers - ): - column_specifiers["id_columns"] = dataset_config_dictionary["id_columns"] - - encode_categorical = False - tsp = TimeSeriesPreprocessor( - **column_specifiers, - scaling=scaling, - encode_categorical=encode_categorical, - prediction_length=forecast_horizon, - context_length=context_length, - ) - dataset_dic = get_datasets( - tsp, - df_dataframe, - split_config={"train": 1.0, "test": 0.0}, - use_frequency_token=True, - ) - dataset_inference = dataset_dic[0] - - model = TinyTimeMixerForPrediction.from_pretrained( - model_checkpoint, prediction_filter_length=forecast_horizon - ) - args = TrainingArguments(output_dir="./output", logging_dir="./log") - trainer = Trainer(model=model, args=args, eval_dataset=dataset_inference) - - ix_target_features = list( - np.arange(len(dataset_config_dictionary["column_specifiers"]["target_columns"])) - ) - - outputs = trainer.predict(dataset_inference) - y_pred = outputs.predictions[0][:, :forecast_horizon, ix_target_features] - - if tsp.scaling: - for ixf in range(y_pred.shape[1]): - y_pred[:, ixf, :] = tsp.target_scaler_dict["0"].inverse_transform( - y_pred[:, ixf, :] - ) - - timestamps_list = [] - timestamps_prediction_list = [] - for i in range(len(dataset_inference)): - if "timestamp" in dataset_inference[i]: - timestamps_list.append(dataset_inference[i]["timestamp"]) - timestamp_forecast = create_timestamps( - last_timestamp=dataset_inference[i]["timestamp"], - time_sequence=df_dataframe[ - column_specifiers["timestamp_column"] - ].values, - periods=forecast_horizon, - ) - timestamps_prediction_list.append(timestamp_forecast) - - output: dict = { - "target_columns": dataset_config_dictionary["column_specifiers"][ - "target_columns" - ], - "target_prediction": y_pred, - "timestamp": timestamps_list, - "timestamp_prediction": timestamps_prediction_list, - } - - inverse_transforms = [] - if scaling: - inverse_transforms.append(tsp.target_scaler_dict["0"].inverse_transform) - - y_gt, y_pred_eval, timestamp_id_value_dic = _get_gt_and_predictions( - trainer, - dataset_inference, - ix_target_features=ix_target_features, - inverse_transforms=inverse_transforms, - ) - target_columns = dataset_config_dictionary["column_specifiers"]["target_columns"] - pd_performance = _get_performance( - y_gt, y_pred_eval, target_columns=target_columns, prediction=False - ) - output["performance"] = pd_performance - - return output - - -# ── Fine-tuning ─────────────────────────────────────────────────────────────── - -_DEFAULT_TRAINING_ARGUMENTS = { - "overwrite_output_dir": True, - "learning_rate": 0.0001, - "num_train_epochs": 10, - "do_eval": True, - "evaluation_strategy": "epoch", - "per_device_train_batch_size": 32, - "per_device_eval_batch_size": 32, - "save_strategy": "epoch", - "logging_strategy": "epoch", - "save_total_limit": 3, - "load_best_model_at_end": True, - "metric_for_best_model": "eval_loss", - "greater_is_better": False, -} - - -def _ttm_main_config(): - return { - "scaling": "", - "p_validation": 0.1, - "encode_categorical": False, - "context_length": 512, - "patch_length": 64, - "forecast_horizon": 96, - "batch_size": 32, - "num_workers": 4, - "seed": 42, - "model_type": "ttm", - "optim": "AdamW", - "lr": 0.0, - "epochs": 4, - "scheduler": "OneCycleLR", - "epochs_warmup": 5, - "es_patience": 15.0, - "es_th": 0.0001, - "backbone_frozen": False, - "decoder_mode": "mix_channel", - "head_dropout": 0.7, - } - - -def _finetune_ttm_hf( - df_dataframe, - dataset_config_dictionary, - model_config, - save_model_dir, - n_finetune, - n_calibration, - n_test, - model_checkpoint="", - training_config_dic=None, -): - from tsfm_public import ( - TinyTimeMixerConfig, - TinyTimeMixerForPrediction, - TrackingCallback, - ) - from tsfm_public.toolkit.lr_finder import optimal_lr_finder - from tsfm_public.toolkit.time_series_preprocessor import ( - TimeSeriesPreprocessor, - get_datasets, - ) - from tsfm_public.toolkit.util import select_by_index - from transformers import Trainer, TrainingArguments, EarlyStoppingCallback, set_seed - - if training_config_dic is None: - args_config_dic = _ttm_main_config() - else: - args_config_dic = training_config_dic.copy() - default_config = _ttm_main_config() - for k in default_config: - if k not in args_config_dic: - args_config_dic[k] = default_config[k] - - seed = args_config_dic["seed"] - set_seed(seed) - encode_categorical = args_config_dic["encode_categorical"] - scaling_type = args_config_dic["scaling"] - p_validation = args_config_dic["p_validation"] - - forecast_horizon = model_config["prediction_length"] - context_length = model_config["context_length"] - args_config_dic["forecast_horizon"] = forecast_horizon - args_config_dic["context_length"] = context_length - - assert context_length <= len(df_dataframe), ( - " length of dataframe needs to be >= context length" - ) - - column_specifiers = dataset_config_dictionary["column_specifiers"] - ix_target_features = list(np.arange(len(column_specifiers["target_columns"]))) - - if ( - "id_columns" in dataset_config_dictionary - and "id_columns" not in column_specifiers - ): - column_specifiers["id_columns"] = dataset_config_dictionary["id_columns"] - - n_data = len(df_dataframe) - assert n_test >= 0 - p_test = n_test / n_data if n_test >= 1 else n_test - n_train_total = int(np.floor((1 - p_test) * n_data)) - - assert n_finetune > 0 - p_finetune = n_finetune / n_train_total if n_finetune > 1 else n_finetune - n_validation = np.ceil(p_finetune * n_train_total * p_validation) - p_train = (n_train_total - n_validation) / n_data - n_train_effective = p_finetune * n_train_total - n_validation - fewshot_fraction = n_train_effective / (n_train_total - n_validation) - - scaling = scaling_type == "standard" - - tsp = TimeSeriesPreprocessor( - **column_specifiers, - scaling=scaling, - encode_categorical=encode_categorical, - prediction_length=forecast_horizon, - context_length=context_length, - ) - dataset_dic = get_datasets( - tsp, - df_dataframe, - split_config={"train": p_train, "test": p_test}, - use_frequency_token=True, - fewshot_fraction=fewshot_fraction, - ) - train_dataset = dataset_dic[0] - valid_dataset = dataset_dic[1] - test_dataset = dataset_dic[2] - - with open(os.path.join(save_model_dir, "args_config.yml"), "w") as outfile: - yaml.dump(args_config_dic, outfile) - with open(os.path.join(save_model_dir, "tsp.pickle"), "wb") as _f: - pickle.dump(tsp, _f) - - if os.path.exists(model_checkpoint): - finetune_forecast_model = TinyTimeMixerForPrediction.from_pretrained( - model_checkpoint, - head_dropout=args_config_dic["head_dropout"], - num_input_channels=tsp.num_input_channels, - exogenous_channel_indices=tsp.exogenous_channel_indices, - prediction_channel_indices=tsp.prediction_channel_indices, - decoder_mode=args_config_dic["decoder_mode"], - enable_forecast_channel_mixing=False, - fcm_use_mixer=False, - ignore_mismatched_sizes=True, - prediction_filter_length=forecast_horizon, - ) - else: - config_ttm_dic = model_config.copy() - config_ttm_dic.update( - { - "head_dropout": args_config_dic["head_dropout"], - "prediction_length": forecast_horizon, - "num_input_channels": tsp.num_input_channels, - "exogenous_channel_indices": tsp.exogenous_channel_indices, - "prediction_channel_indices": tsp.prediction_channel_indices, - "enable_forecast_channel_mixing": False, - "fcm_use_mixer": False, - "decoder_mode": args_config_dic["decoder_mode"], - } - ) - config = TinyTimeMixerConfig(**config_ttm_dic) - finetune_forecast_model = TinyTimeMixerForPrediction(config) - - if args_config_dic["backbone_frozen"]: - for param in finetune_forecast_model.backbone.parameters(): - param.requires_grad = False - - batch_size = args_config_dic["batch_size"] - epochs = args_config_dic["epochs"] - num_workers = args_config_dic["num_workers"] - epochs_warmup = args_config_dic["epochs_warmup"] - es_patience = args_config_dic["es_patience"] - es_th = args_config_dic["es_th"] - optim = args_config_dic["optim"] - scheduler = args_config_dic["scheduler"] - lr = args_config_dic["lr"] - - # Use a fresh copy of the defaults to avoid cross-call mutation - training_config_dictionary = _DEFAULT_TRAINING_ARGUMENTS.copy() - - output_fewshot_dir = save_model_dir + "/fewshot/" - logging_dir = save_model_dir + "/log/" - os.makedirs(output_fewshot_dir, exist_ok=True) - os.makedirs(logging_dir, exist_ok=True) - - training_config_dictionary.update( - { - "per_device_train_batch_size": batch_size, - "per_device_eval_batch_size": batch_size, - "num_train_epochs": epochs, - "learning_rate": lr, - "output_dir": output_fewshot_dir, - "logging_dir": logging_dir, - "dataloader_num_workers": num_workers, - } - ) - if epochs_warmup > 0: - training_config_dictionary["warmup_steps"] = math.ceil( - epochs_warmup * len(train_dataset) / batch_size - ) - with open(os.path.join(save_model_dir, "training_config.yml"), "w") as outfile: - yaml.dump(training_config_dictionary, outfile) - - finetune_forecast_args = TrainingArguments(**training_config_dictionary) - - if n_finetune > 0: - if lr <= 0: - try: - lr, finetune_forecast_model = optimal_lr_finder( - finetune_forecast_model, train_dataset, batch_size=batch_size - ) - if lr <= 0: - lr = 0.0001 - except Exception: - lr = 0.0001 - else: - lr = 0.0001 - - early_stopping_callback = EarlyStoppingCallback( - early_stopping_patience=es_patience, - early_stopping_threshold=es_th, - ) - - optimizer = None - if optim == "AdamW": - from torch.optim import AdamW - - optimizer = AdamW(finetune_forecast_model.parameters(), lr=lr) - - scheduler_object = None - if scheduler == "cosine_with_warmup": - if optimizer is None: - from torch.optim import AdamW - - optimizer = AdamW(finetune_forecast_model.parameters(), lr=lr) - from transformers.optimization import get_cosine_schedule_with_warmup - - total_steps = math.ceil(len(train_dataset) * epochs / batch_size) - num_warmup_steps = math.ceil(epochs_warmup * len(train_dataset) / batch_size) - scheduler_object = get_cosine_schedule_with_warmup( - optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=total_steps - ) - if scheduler == "OneCycleLR": - if optimizer is None: - from torch.optim import AdamW - - optimizer = AdamW(finetune_forecast_model.parameters(), lr=lr) - from torch.optim.lr_scheduler import OneCycleLR - - scheduler_object = OneCycleLR( - optimizer, - lr, - epochs=epochs, - steps_per_epoch=math.ceil(len(train_dataset) / batch_size), - ) - - tracking_callback = TrackingCallback() - finetune_forecast_trainer = Trainer( - model=finetune_forecast_model, - args=finetune_forecast_args, - train_dataset=train_dataset, - eval_dataset=valid_dataset, - callbacks=[early_stopping_callback, tracking_callback], - optimizers=(optimizer, scheduler_object), - ) - - start_time = time.time() - if n_finetune > 0: - finetune_forecast_trainer.train() - train_time = time.time() - start_time - - dataset_eval: dict = {} - if n_finetune > 0: - dataset_eval["train"] = train_dataset - dataset_eval["valid"] = valid_dataset - if n_test >= 1: - dataset_eval["test"] = test_dataset - - pd_performance = pd.DataFrame() - for dataset_key in dataset_eval: - inverse_transforms_eval = [] - if scaling: - inverse_transforms_eval.append( - tsp.target_scaler_dict["0"].inverse_transform - ) - y_gt, y_pred_eval, _ = _get_gt_and_predictions( - finetune_forecast_trainer, - dataset_eval[dataset_key], - ix_target_features=ix_target_features, - inverse_transforms=inverse_transforms_eval, - ) - target_columns = dataset_config_dictionary["column_specifiers"][ - "target_columns" - ] - pd_performance_i = _get_performance( - y_gt, y_pred_eval, target_columns=target_columns, prediction=False - ) - pd_performance_i["split"] = dataset_key - pd_performance = pd.concat([pd_performance, pd_performance_i], axis=0) - - pd_performance["train_time"] = train_time - return { - "performance": pd_performance, - "save_model_dir": save_model_dir, - "experiment_config_path": os.path.join(save_model_dir, "args_config.yml"), - } - - -def _find_largest_tsfm_checkpoint_directory(root_dir: str) -> str: - largest_checkpoint_dir = None - largest_number = float("-inf") - for f in os.listdir(root_dir): - if "checkpoint" in f: - number = int(f.split("-")[-1]) - if number > largest_number: - largest_number = number - largest_checkpoint_dir = os.path.join(root_dir, f) - return largest_checkpoint_dir diff --git a/src/servers/tsfm/io.py b/src/servers/tsfm/io.py deleted file mode 100644 index d0a347a0d..000000000 --- a/src/servers/tsfm/io.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Path/IO helpers and dataset reader for the TSFM MCP server.""" - -from __future__ import annotations - -import json -import os -import tempfile -import uuid -from datetime import datetime -from typing import Optional - -import numpy as np -import pandas as pd - - -# ── Path helpers ────────────────────────────────────────────────────────────── - - -_DEFAULT_MODELS_DIR = os.path.join( - os.path.dirname(__file__), "artifacts", "output", "tuned_models" -) - - -def _get_model_checkpoint_path(model_checkpoint: str) -> str: - if os.path.isabs(model_checkpoint): - return model_checkpoint - return os.path.join( - os.environ.get("PATH_TO_MODELS_DIR", _DEFAULT_MODELS_DIR), model_checkpoint - ) - - -def _get_dataset_path(dataset: str) -> str: - if os.path.isabs(dataset): - return dataset - return os.path.join(os.environ.get("PATH_TO_DATASETS_DIR", ""), dataset) - - -def _get_outputs_path(outputs: str) -> str: - if os.path.isabs(outputs): - return outputs - return os.path.join(os.environ.get("PATH_TO_OUTPUTS_DIR", ""), outputs) - - -# ── JSON utilities ──────────────────────────────────────────────────────────── - - -def _write_json_to_temp(json_data: str) -> str: - temp_dir = tempfile.gettempdir() - temp_file_path = os.path.join(temp_dir, f"{uuid.uuid4().hex}.json") - with open(temp_file_path, "w") as f: - f.write(json_data) - return temp_file_path - - -def _make_json_compatible(obj): - """Recursively convert an object to a JSON-serializable form.""" - if isinstance(obj, dict): - return {str(k): _make_json_compatible(v) for k, v in obj.items()} - if isinstance(obj, list): - return [_make_json_compatible(i) for i in obj] - if isinstance(obj, (int, float, str, bool)) or obj is None: - return obj - if isinstance(obj, np.integer): - return int(obj) - if isinstance(obj, np.floating): - return float(obj) - if isinstance(obj, np.ndarray): - return obj.tolist() - if isinstance(obj, datetime): - return obj.isoformat() - return str(obj) - - -# ── Dataset reading ─────────────────────────────────────────────────────────── - - -def _read_ts_data(dataset_path: str, dataset_config_dictionary=None) -> pd.DataFrame: - if dataset_config_dictionary is not None: - timestamp_column = dataset_config_dictionary["column_specifiers"][ - "timestamp_column" - ] - else: - timestamp_column = "Date" - - valid_extensions = {".csv", ".json", ".xlsx"} - _, file_extension = os.path.splitext(dataset_path) - if file_extension.lower() not in valid_extensions: - raise ValueError( - f"Invalid file type received: {dataset_path}. " - f"Expected file types are: {', '.join(valid_extensions)}." - ) - - if ".csv" in dataset_path: - if dataset_config_dictionary is not None: - col_spec = dataset_config_dictionary["column_specifiers"] - data_df = pd.read_csv( - dataset_path, parse_dates=[col_spec["timestamp_column"]] - ) - else: - data_df = pd.read_csv(dataset_path) - - elif ".json" in dataset_path: - try: - with open(dataset_path) as _f: - result_dict = json.load(_f) - ts: dict = {} - for input_data in result_dict: - dt = datetime.fromisoformat(input_data["timestamp"]) - if (dt.minute % 15) != 0: - continue - input_data[timestamp_column] = dt - ts[dt] = input_data - data_df = pd.DataFrame() - for dt, v in ts.items(): - data_df = pd.concat([data_df, pd.DataFrame(v, index=[dt])]) - except Exception as ex: - raise ValueError( - f"input file {dataset_path} is not in the correct format" - ) from ex - - elif ".xlsx" in dataset_path: - if dataset_config_dictionary is not None: - col_spec = dataset_config_dictionary["column_specifiers"] - data_df = pd.read_excel( - dataset_path, parse_dates=[col_spec["timestamp_column"]] - ) - else: - data_df = pd.read_excel(dataset_path) - - else: - raise ValueError( - f"file extension must be: .json, .csv, or .xlsx. file: {dataset_path}" - ) - - return data_df diff --git a/src/servers/tsfm/io/__init__.py b/src/servers/tsfm/io/__init__.py new file mode 100644 index 000000000..36c57c5aa --- /dev/null +++ b/src/servers/tsfm/io/__init__.py @@ -0,0 +1 @@ +"""io layer.""" diff --git a/src/servers/tsfm/io/refs.py b/src/servers/tsfm/io/refs.py new file mode 100644 index 000000000..67e4a5dfd --- /dev/null +++ b/src/servers/tsfm/io/refs.py @@ -0,0 +1,99 @@ +"""io_refs.py — file-pointer data I/O (IoT data is passed by reference, not inline). + +IoT sensor data and every step output are **file pointers** (path / file:// / s3:// URI to a +CSV/Parquet), exactly like HuggingGPT chains tasks via resource files (`-task_id`). +The MCP tools carry small JSON (a `data_ref`), never the array; this layer loads a ref into the +sktime-native container (pd.Series univariate / pd.DataFrame multivariate) and writes step +outputs back to new file pointers so downstream steps can consume them. +""" + +from __future__ import annotations + +import json +import os +import uuid +from typing import List, Optional + +import numpy as np +import pandas as pd + +WORKDIR = os.environ.get("TSFM_WORKDIR", "/tmp/tsfm_work") + + +def _path(ref: str) -> str: + return ref[7:] if ref.startswith("file://") else ref + + +def _ensure_workdir(): + os.makedirs(WORKDIR, exist_ok=True) + return WORKDIR + + +def load_series( + data_ref: str, + *, + value_col: Optional[str] = None, + time_col: Optional[str] = None, + channels: Optional[List[str]] = None, +): + """Resolve a file pointer to an sktime container. CSV/Parquet → pd.Series (univariate) or + pd.DataFrame (multivariate, channel-subset aware). Index becomes the time axis.""" + p = _path(data_ref) + df = pd.read_parquet(p) if p.endswith((".parquet", ".pq")) else pd.read_csv(p) + # auto-detect the time column if not given (timestamp/time/date/datetime) + if time_col is None: + for c in df.columns: + if c.lower() in ("timestamp", "time", "date", "datetime"): + time_col = c + break + if time_col and time_col in df.columns: + df = df.drop(columns=[time_col]) + # numeric value columns only; drop any column that isn't numeric (coerces all-NaN away) + num = df.apply(pd.to_numeric, errors="coerce").astype( + "float64" + ) # no nullable dtypes / pd.NA + num = num.loc[:, num.notna().any(axis=0)] + cols = channels or list(num.columns) + out = num[cols].reset_index(drop=True) # RangeIndex keeps sktime happy offline + return out[cols[0]] if len(cols) == 1 else out + + +def write_series(obj, *, name: str = "result") -> str: + """Write a forecast/series/frame to a file pointer and return the ref.""" + _ensure_workdir() + p = os.path.join(WORKDIR, f"{name}_{uuid.uuid4().hex[:8]}.csv") + (obj.to_frame() if isinstance(obj, pd.Series) else pd.DataFrame(obj)).to_csv( + p, index=False + ) + return f"file://{p}" + + +def write_json(obj: dict, *, name: str = "result") -> str: + _ensure_workdir() + p = os.path.join(WORKDIR, f"{name}_{uuid.uuid4().hex[:8]}.json") + json.dump(obj, open(p, "w"), indent=2, default=str) + return f"file://{p}" + + +def materialize_iot( + series, *, asset_id: str = "asset", value_cols=None, freq: str = "H" +) -> str: + """Test helper: write an in-memory series/array to an IoT-style CSV file pointer.""" + _ensure_workdir() + arr = np.asarray(series, float) + if arr.ndim == 1: + df = pd.DataFrame( + { + "timestamp": pd.date_range("2020-01-01", periods=len(arr), freq=freq), + "value": arr, + } + ) + else: + cols = value_cols or [f"ch{i}" for i in range(arr.shape[1])] + df = pd.DataFrame(arr, columns=cols) + df.insert( + 0, "timestamp", pd.date_range("2020-01-01", periods=len(arr), freq=freq) + ) + p = os.path.join(WORKDIR, f"iot_{asset_id}.csv") + df.to_csv(p, index=False) + return f"file://{p}" diff --git a/src/servers/tsfm/io/window.py b/src/servers/tsfm/io/window.py new file mode 100644 index 000000000..28893e580 --- /dev/null +++ b/src/servers/tsfm/io/window.py @@ -0,0 +1,49 @@ +"""Data access — read a sensor window for (asset, channels, range). + +In production this reads the `iot` / `vibration` CouchDB collections (or a dataset_path CSV). +For the deterministic demo/tests it synthesizes a realistic multivariate series per asset +(trend + seasonality + noise, with an injectable anomaly), so the whole pipeline runs with no +external data. +""" + +from __future__ import annotations + +import numpy as np + +# per-asset profile: (n_channels, period, anomaly_at_fraction or None) +_PROFILE = { + "chiller_6": (3, 48, 0.85), + "metro_pump_1": (4, 24, None), + "motor_01": (2, 16, 0.7), +} + + +def read_window( + asset_id: str, + *, + channels=None, + length: int = 600, + seed: int = 0, + anomalous: bool = True, + store=None, +): + """Return (X, channel_names). If `store` + iot data exist, could read real data; here + synthesizes a deterministic series so the pipeline is runnable end-to-end.""" + n_ch, period, anom = _PROFILE.get(asset_id, (3, 24, None)) + rng = np.random.RandomState(seed + abs(hash(asset_id)) % 1000) + t = np.arange(length) + X = np.zeros((length, n_ch)) + for c in range(n_ch): + X[:, c] = ( + 0.01 * (c + 1) * t # trend + + (2 + c) * np.sin(2 * np.pi * t / period) # seasonality + + rng.normal(0, 0.4, length) + ) # noise + if anomalous and anom is not None: + i = int(anom * length) + X[i : i + 6, 0] += 8.0 # injected spike on channel 0 + names = (channels or [f"ch{c}" for c in range(n_ch)])[:n_ch] + if channels: + idx = list(range(min(len(channels), n_ch))) + X = X[:, idx] + return X, names diff --git a/src/servers/tsfm/main.py b/src/servers/tsfm/main.py index 81b9f2ce9..5822f734d 100644 --- a/src/servers/tsfm/main.py +++ b/src/servers/tsfm/main.py @@ -1,681 +1,936 @@ -"""TSFM (Time Series Foundation Model) MCP Server. - -Standalone implementation — no dependency on src/tsfmagent/. -The only external ML dependency is tsfm_public (IBM Granite TSFM): - https://github.com/IBM-granite/granite-tsfm - -Tools: - get_ai_tasks – list available AI task types (static, no deps) - get_tsfm_models – list available pre-trained model checkpoints (static) - run_tsfm_forecasting – zero-shot TTM inference on a dataset - run_tsfm_finetuning – few-shot finetuning of a TTM model - run_tsad – conformal anomaly detection on TSFM forecasts - run_integrated_tsad – end-to-end: forecasting + anomaly detection - -Heavy ML dependencies (tsfm_public, transformers, torch) are imported lazily; -the server starts and exposes the static tools even when they are absent. - -Required environment variables (path resolution): - PATH_TO_MODELS_DIR – directory containing TTM model checkpoint folders - PATH_TO_DATASETS_DIR – base directory for resolving relative dataset paths - PATH_TO_OUTPUTS_DIR – base directory for resolving output/save paths -""" - from __future__ import annotations -import json import logging import os -import tempfile -import uuid -from functools import lru_cache -from typing import Dict, List, Optional, Union +from typing import List, Optional, Union -import numpy as np import pandas as pd from dotenv import load_dotenv from mcp.server.fastmcp import FastMCP -from .anomaly import _TimeSeriesAnomalyDetectionConformalWrapper -from .forecasting import ( - _finetune_ttm_hf, - _find_largest_tsfm_checkpoint_directory, - _get_ttm_hf_inference, - _tsfm_data_quality_filter, -) -from .io import ( - _get_dataset_path, - _get_model_checkpoint_path, - _get_outputs_path, - _read_ts_data, - _write_json_to_temp, -) -from .models import ( - _AI_TASKS, - _TSFM_MODELS, - AITaskEntry, - AITasksResult, +from .bootstrap import fresh_store +from .config import RUNS_COLLECTION, PLANS_COLLECTION +from .core import tasks as task_spec +from .core import glossary as _glossary +from .core.results_models import ( ErrorResult, - FinetuningResult, - ForecastingResult, - TSADResult, - TSFMModelEntry, - TSFMModelsResult, + TasksResult, + ComponentsResult, + CandidatesResult, + ModelsResult, + FeaturesResult, + ComponentResult, + ProfileResult, + FeatureSelectionResult, + RecipeResult, + TabularResult, + PlanResult, + EvaluateResult, + RegisterResult, + ResultsListResult, + ResultRecord, + RunRecord, + RunsResult, + ExtractResult, ) +from .core.results_models import EvolveAskResult, EvolveTellResult, EvolveBestResult +from .core.results_models import CardResult, LineageResult, DataQualityResult +from .core.results_models import CharacterizeResult +from .reasoning import dataquality as _dq +from .io import refs +from .stores import model_store, feature_store, results +from .engine import composition, plan, evolve +from .eval import gifteval +from .reasoning import param_space, profile, patterns load_dotenv() - -_log_level = getattr( - logging, os.environ.get("LOG_LEVEL", "WARNING").upper(), logging.WARNING +logging.basicConfig( + level=getattr( + logging, os.environ.get("LOG_LEVEL", "WARNING").upper(), logging.WARNING + ) ) -logging.basicConfig(level=_log_level) logger = logging.getLogger("tsfm-mcp-server") +mcp = FastMCP( + "tsfm", + instructions=( + "Time-series AI on an sktime substrate. Discover components (models/features are catalog " + "data), read evidence (profile_series), compose a recipe (transforms + single/ensemble + " + "conformal), run it (run_recipe / run_tabular_recipe / run_plan), score it (evaluate, " + "GIFT-Eval). Data is passed as FILE POINTERS (dataset_path); results come back as a " + "results_file pointer. You reason every parameter; the server gives evidence + grades. " + "Zero-shot is the default; fine-tune is optional. Forecasting and anomaly both run through " + "run_recipe (anomaly via recipe.task). Results hand off downstream — no alerts here.\n\n" + "VOCABULARY — " + _glossary.short_glossary() + "\n\n" + "WORKFLOW — " + " ".join(_glossary.WORKFLOW) + "\n" + "Call discover_components for the full glossary, the menu, and the recipe blocks." + ), +) -# ── Internal helpers ────────────────────────────────────────────────────────── - +_STORE = ( + fresh_store() +) -@lru_cache(maxsize=16) -def _load_model_config(model_checkpoint: str) -> dict: - """Load and cache model config.json to avoid repeated disk reads.""" - with open(model_checkpoint + "/config.json") as f: - return json.load(f) +def _load_target( + dataset_path: str, timestamp_column: Optional[str], target_columns: List[str] +): + """Resolve a file pointer to the (univariate) target series for forecasting.""" + obj = refs.load_series( + dataset_path, time_col=timestamp_column, channels=target_columns + ) + return obj.iloc[:, 0] if isinstance(obj, pd.DataFrame) else obj + + +def _check_recipe(recipe) -> Optional[str]: + """Validate the shape of a recipe before the engine touches it.""" + if not isinstance(recipe, dict) or not recipe: + return "recipe must be a non-empty object" + if "estimator" not in recipe and "ensemble" not in recipe: + return "recipe must include an 'estimator' or an 'ensemble'" + return None + + +def _check_task(task_id: str) -> Optional[str]: + """Validate a task_id against the 8 standardized tasks; error lists the valid ids.""" + if not (task_id or "").strip(): + return "task_id is required" + if task_id not in task_spec.TASKS: + return f"unknown task '{task_id}'. Valid tasks: {list(task_spec.TASKS)}" + return None + + +# ════════════════════════════ redesigned surface ════════════════════════════ +# ---- discover ---- +@mcp.tool(title="List Tasks") +def list_tasks() -> Union[TasksResult, ErrorResult]: + """List the 8 standardized TS-AI TASKS (forecasting, regression, classification, anomaly, + imputation, evaluation, similarity_search, clustering). Each entry has a plain `description` + plus its contract (required inputs, output, eval protocol). Start here, then profile_series. + """ + try: + return TasksResult(tasks=task_spec.list_tasks()) + except Exception as exc: + logger.error("list_tasks failed: %s", exc) + return ErrorResult(error=str(exc)) -def _build_dataset_config( - timestamp_column: str, - target_columns: List[str], - conditional_columns: Optional[List[str]], - id_columns: Optional[List[str]], - frequency_sampling: str, - autoregressive_modeling: bool, -) -> dict: - return { - "column_specifiers": { - "autoregressive_modeling": autoregressive_modeling, - "timestamp_column": timestamp_column, - "conditional_columns": conditional_columns or [], - "target_columns": target_columns, - }, - "id_columns": id_columns or [], - "frequency_sampling": frequency_sampling, - } - - -def _tsad_output_to_df(output: dict) -> pd.DataFrame: - kpi = output.pop("KPI", None) - df = pd.DataFrame.from_dict({k: np.array(v) for k, v in output.items()}) - if kpi is not None: - df["KPI"] = ( - kpi[0] if (hasattr(kpi, "__len__") and not isinstance(kpi, str)) else kpi +@mcp.tool(title="Discover Components") +def discover_components( + task: str = "tsfm_forecasting", +) -> Union[ComponentsResult, ErrorResult]: + """The mix-and-match MENU for a task: installed + foundation MODELS, FEATURE transforms, + ensemble combiners, training REGIMES, and the recipe blocks you can fill. Also returns the + full GLOSSARY + WORKFLOW so you know every term. A COMPONENT is any model or feature you + place in a RECIPE (they are catalog data, not tools).""" + bad = _check_task(task) + if bad: + return ErrorResult(error=bad) + try: + return ComponentsResult( + task=task, components=composition.discover_components(_STORE, task=task) ) - return df + except Exception as exc: + logger.error("discover_components failed: %s", exc) + return ErrorResult(error=str(exc)) -# ── FastMCP server ──────────────────────────────────────────────────────────── +@mcp.tool(title="Describe Candidates") +def describe_candidates( + task_id: str, top_k: int = 5, domain: Optional[str] = None +) -> Union[CandidatesResult, ErrorResult]: + """Ranked CANDIDATE models for a task (HuggingGPT-style, by description + popularity). A + candidate is a shortlisted MODEL — you still decide which to use. top_k caps the list. + """ + bad = _check_task(task_id) + if bad: + return ErrorResult(error=bad) + top_k = max(1, min(int(top_k), 50)) + try: + return CandidatesResult( + task_id=task_id, + candidates=model_store.describe_candidates( + _STORE, task_id, top_k=top_k, domain=domain + ), + ) + except Exception as exc: + logger.error("describe_candidates failed: %s", exc) + return ErrorResult(error=str(exc)) -mcp = FastMCP( - "tsfm", - instructions="Time-series foundation models: forecasting, finetuning, and anomaly detection using IBM Granite TinyTimeMixer.", -) +@mcp.tool(title="Find Models") +def find_models( + task_id: str, + min_context_length: Optional[int] = None, + prediction_length: Optional[int] = None, + domain: Optional[str] = None, + top_k: int = 5, +) -> Union[ModelsResult, ErrorResult]: + """Filter the MODEL catalog for a task → ranked shortlist. A model is an estimator card; use + get_component(model_id) to read its full card + param_schema before composing a recipe. + """ + bad = _check_task(task_id) + if bad: + return ErrorResult(error=bad) + top_k = max(1, min(int(top_k), 50)) + try: + return ModelsResult( + models=model_store.find_models( + _STORE, + task_id, + min_context_length=min_context_length, + prediction_length=prediction_length, + domain=domain, + top_k=top_k, + ) + ) + except Exception as exc: + logger.error("find_models failed: %s", exc) + return ErrorResult(error=str(exc)) -# ── Static tools ────────────────────────────────────────────────────────────── +@mcp.tool(title="Find Features") +def find_features( + target_task: Optional[str] = None, + target_model: Optional[str] = None, +) -> Union[FeaturesResult, ErrorResult]: + """Browse FEATURE/transform cards (normalization, lag/rolling, catch22, FLOps sets) by + task / model. A feature is applied before the estimator in a recipe. + """ + try: + return FeaturesResult( + features=feature_store.find_features( + _STORE, + target_task=target_task, + target_model=target_model, + ) + ) + except Exception as exc: + logger.error("find_features failed: %s", exc) + return ErrorResult(error=str(exc)) -@mcp.tool(title="Get AI Tasks") -def get_ai_tasks() -> AITasksResult: - """Returns the list of supported AI task types for time-series analysis. - Tasks: tsfm_integrated_tsad, tsfm_forecasting, tsfm_forecasting_finetune, - tsfm_forecasting_evaluation. +@mcp.tool(title="Get Component") +def get_component(component_id: str) -> Union[ComponentResult, ErrorResult]: + """Fetch one COMPONENT by id — a MODEL card or a FEATURE card (it resolves either). For a + model it also returns the `param_schema` (the parameters + hints + ranges you must reason). """ - return AITasksResult(tasks=[AITaskEntry(**t) for t in _AI_TASKS]) - + if not component_id.strip(): + return ErrorResult(error="component_id is required") + card = model_store.get_model(_STORE, component_id) + if card: + try: + card = {**card, "param_schema": param_space.param_schema(card)} + except Exception as e: + card = {**card, "param_schema_error": str(e)[:120]} + return ComponentResult(component_id=component_id, **card) + feat = feature_store.get_feature(_STORE, component_id) + if feat: + return ComponentResult(component_id=component_id, **feat) + return ErrorResult(error=f"component '{component_id}' not found") + + +# ---- evidence / learn (file pointers in) ---- +@mcp.tool(title="Profile Series") +def profile_series( + dataset_path: str, + timestamp_column: Optional[str] = None, + channels: Optional[List[str]] = None, +) -> Union[ProfileResult, ErrorResult]: + """EVIDENCE about the data behind a file pointer (dataset_path) — seasonality, stationarity, + channels, length. Facts only, no recommendations: you reason the recipe from these. This is + the data the param_schema hints depend on (e.g. context_length ≥ 2× dominant_period). + """ + if not dataset_path.strip(): + return ErrorResult(error="dataset_path is required") + try: + ev = profile.profile_ref( + dataset_path, timestamp_column=timestamp_column, channels=channels + ) + return ProfileResult(**ev) + except Exception as exc: + logger.error("profile_series failed: %s", exc) + return ErrorResult(error=str(exc)) -@mcp.tool(title="Get TSFM Models") -def get_tsfm_models() -> TSFMModelsResult: - """Returns the list of available pre-trained TinyTimeMixer (TTM) model checkpoints. - Models: ttm_96_28 (context_length=96), ttm_512_96 (context_length=512), - and energy-domain fine-tuned variants of both. - """ - return TSFMModelsResult(models=[TSFMModelEntry(**m) for m in _TSFM_MODELS]) +@mcp.tool(title="Select Features (FLOps)") +def select_features( + dataset_path: str, + timestamp_column: Optional[str] = None, + target_column: Optional[str] = None, + reference_feature: str = "mean", + cd_margin: float = 0.05, +) -> Union[FeatureSelectionResult, ErrorResult]: + """FLOps multi-config dynamic feature selection on a series passed as a file pointer.""" + if not dataset_path.strip(): + return ErrorResult(error="dataset_path is required") + try: + chans = [target_column] if target_column else None + obj = refs.load_series(dataset_path, time_col=timestamp_column, channels=chans) + series = (obj.iloc[:, 0] if isinstance(obj, pd.DataFrame) else obj).to_numpy() + res = feature_store.select_features( + series, reference_feature=reference_feature, cd_margin=cd_margin + ) + detail = refs.write_json(res, name="flops_select") + return FeatureSelectionResult( + selected=res["selected"], + lookback=res["lookback"], + reference=res["reference"], + scorers=res["scorers"], + detail_file=detail, + ) + except Exception as exc: + logger.error("select_features failed: %s", exc) + return ErrorResult(error=str(exc)) -# ── TSFM Forecasting (zero-shot inference) ──────────────────────────────────── +@mcp.tool(title="Characterize Series (pattern evidence)") +def characterize_series( + dataset_path: str, + timestamp_column: Optional[str] = None, + channels: Optional[List[str]] = None, + groups: Optional[dict] = None, + group_rules: Optional[str] = None, +) -> Union[CharacterizeResult, ErrorResult]: + """Describe the SHAPE of a series as structured EVIDENCE for an LLM to reason over (fault, + cause, RUL, work-order, …) — it never names a fault. Generic: any signals, any count, any + names. Per channel-group it labels a state (stable / rise / decline / spike / level_shift / + cessation / oscillation) + rate over changepoint phases, plus the bivariate relation + (decoupled / co_move / lead_lag) between groups. Grouping is optional and yours to choose: + pass groups={group:[channels]}, or group_rules (a preset name like 'vibration_temperature'); + default is one group per channel. Reference-free (reads the series' own median/MAD scale). + """ + if not dataset_path.strip(): + return ErrorResult(error="dataset_path is required") + try: + obj = refs.load_series( + dataset_path, time_col=timestamp_column, channels=channels + ) + frame = ( + obj + if isinstance(obj, pd.DataFrame) + else obj.to_frame(name=(channels[0] if channels else "value")) + ) + ev = patterns.describe_series(frame, groups=groups, group_rules=group_rules) + evidence_file = refs.write_json(ev, name="pattern_evidence") + return CharacterizeResult( + status="success", + summary=ev["summary"], + n_observations=ev["n_observations"], + evidence_file=evidence_file, + groups=ev["groups"], + phases=ev["phases"], + message=f"Pattern evidence ({len(ev['phases'])} phase(s)). Full object at {evidence_file}.", + ) + except Exception as exc: + logger.error("characterize_series failed: %s", exc) + return ErrorResult(error=str(exc)) -@mcp.tool(title="Run TSFM Forecasting") -def run_tsfm_forecasting( +# ---- compose + run (file pointers in/out) ---- +@mcp.tool(title="Run Recipe") +def run_recipe( dataset_path: str, timestamp_column: str, target_columns: List[str], - model_checkpoint: str = "ttm_96_28", - forecast_horizon: int = -1, - conditional_columns: Optional[List[str]] = None, - id_columns: Optional[List[str]] = None, - frequency_sampling: str = "oov", - autoregressive_modeling: bool = True, - include_dataquality_summary: bool = False, -) -> Union[ForecastingResult, ErrorResult]: - """Run zero-shot time-series forecasting with a TinyTimeMixer (TTM) model. - - Returns a ForecastingResult whose results_file is the path to a JSON file - with raw predictions (target_prediction, timestamp, target_columns arrays). - Pass that path to run_tsad as tsfm_output_json for anomaly detection. - - Args: - dataset_path: Path to a CSV, JSON, or XLSX dataset. - timestamp_column: Name of the timestamp column. - target_columns: Columns to forecast. - model_checkpoint: Model name (e.g. ttm_96_28) or absolute checkpoint path. - forecast_horizon: Number of steps to forecast; -1 uses the model default. - conditional_columns: Exogenous / conditional feature columns. - id_columns: ID columns for multi-entity grouped time series. - frequency_sampling: Sampling frequency string (e.g. '15_minutes') or - 'oov' to auto-detect from the data. - autoregressive_modeling: Use autoregressive inference when True. - include_dataquality_summary: Attach a data-quality report to the result. - """ + recipe: dict, + asset_id: str = "asset", + parent_run_id: Optional[str] = None, +) -> Union[RecipeResult, ErrorResult]: + """Run a recipe on a series from a file pointer. Dispatches by recipe['task']: + 'tsfm_anomaly_detection' → detector path (e.g. tspulse_ad zero-shot, sublof) producing dense + anomaly labels; otherwise FORECASTING (transforms + single/ensemble + conformal). Writes the + run record to a results_file pointer.""" if not dataset_path.strip(): return ErrorResult(error="dataset_path is required") if not target_columns: return ErrorResult(error="target_columns must not be empty") - + bad = _check_recipe(recipe) + if bad: + return ErrorResult(error=bad) try: - import tsfm_public # noqa: F401 – verify dependency present - except ImportError as exc: - return ErrorResult(error=f"tsfm dependencies unavailable: {exc}") - - model_checkpoint = _get_model_checkpoint_path(model_checkpoint) - dataset_path = _get_dataset_path(dataset_path) - dataset_config = _build_dataset_config( - timestamp_column, - target_columns, - conditional_columns, - id_columns, - frequency_sampling, - autoregressive_modeling, - ) - - try: - data_df = _read_ts_data(dataset_path, dataset_config_dictionary=dataset_config) - model_config = _load_model_config(model_checkpoint) - - output_data_quality = _tsfm_data_quality_filter( - data_df, dataset_config, model_config, task="inference" + series = _load_target(dataset_path, timestamp_column, target_columns) + res = composition.run_recipe( + _STORE, series, recipe, asset_id=asset_id, parent_run_id=parent_run_id ) - data_df = output_data_quality["data"] - dataset_config = output_data_quality["dataset_config_dictionary"] - - inference_result_dict_data: dict = { - "target_prediction": [], - "timestamp": [], - "target_columns": [], - } - - if len(data_df) > 0: - output = _get_ttm_hf_inference( - data_df, - dataset_config, - model_config, - model_checkpoint, - forecast_horizon=forecast_horizon, - ) - inference_result_dict_data["target_prediction"] = output[ - "target_prediction" - ].tolist() - inference_result_dict_data["timestamp"] = ( - np.array(output["timestamp_prediction"]).astype(str).tolist() + if res.get("task") == "tsfm_anomaly_detection": # detector path + results_file = refs.write_json( + { + "anomaly_label": res.pop("labels"), + "n_anomalies": res["n_anomalies"], + "anomaly_indices": res["anomaly_indices_head"], + }, + name="anomaly", ) - inference_result_dict_data["target_columns"] = output["target_columns"] - else: - return ErrorResult( - error="Data quality was poor; after filtering, no continuous segment satisfied the " - "context length requirement. Check Data Quality Summary." + return RecipeResult( + status="success", + run_id=res["run_id"], + results_file=results_file, + training_regime=res["training_regime"], + n_anomalies=res["n_anomalies"], + n_observations=res["n_observations"], + message=f"Anomaly run complete ({res['training_regime']}): " + f"{res['n_anomalies']}/{res['n_observations']} flagged. Labels at {results_file}.", ) - - # Trim to requested forecast horizon - if forecast_horizon != -1 and "target_prediction" in inference_result_dict_data: - target_prediction = np.array( - inference_result_dict_data["target_prediction"] - ) - if 0 < forecast_horizon <= target_prediction.shape[1]: - inference_result_dict_data["target_prediction"] = target_prediction[ - :, :forecast_horizon, : - ].tolist() - inference_result_dict_data["timestamp"] = np.array( - inference_result_dict_data["timestamp"] - )[:, :forecast_horizon].tolist() - - results_file = _write_json_to_temp( - json.dumps(inference_result_dict_data, indent=4) + results_file = refs.write_json(res, name="recipe_run") # forecasting path + return RecipeResult( + status="success", + run_id=res["run_id"], + results_file=results_file, + metric=res["metric"], + backtest_score=res["backtest_score"], + training_regime=res["training_regime"], + message=f"Recipe run complete ({res['training_regime']}). Record at {results_file}.", ) - except Exception as exc: - logger.error("run_tsfm_forecasting failed: %s", exc) + logger.error("run_recipe failed: %s", exc) return ErrorResult(error=str(exc)) - dataquality_summary = ( - output_data_quality["dataquality_summary"] - if include_dataquality_summary - else None - ) - return ForecastingResult( - status="success", - results_file=results_file, - dataquality_summary=dataquality_summary, - message=f"Forecasting complete. Predictions saved to {results_file}.", - ) - -# ── TSFM Finetuning ─────────────────────────────────────────────────────────── +@mcp.tool(title="Data Quality") +def data_quality( + dataset_path: str, + timestamp_column: str = "timestamp", +) -> Union[DataQualityResult, ErrorResult]: + """Clean a series from a file pointer (NaN removal) + report a data-quality summary; returns a + cleaned file pointer to feed forecasting / anomaly. (The continuous-segment IoT filter lives + in reasoning/dataquality for the forecasting-context path.)""" + if not dataset_path.strip(): + return ErrorResult(error="dataset_path is required") + try: + df = pd.read_csv(refs._path(dataset_path)) + nan = _dq._df_nan_stats(df) + out = _dq._efficient_nan_removal(df) + cleaned = out["df_filter"] + cleaned_file = refs.write_series(cleaned, name="cleaned") + nan_per_col = { + str(k): float(v) for k, v in (nan.get("%NaN_per_column") or {}).items() + } + return DataQualityResult( + status="success", + cleaned_file=cleaned_file, + rows_in=int(len(df)), + rows_out=int(len(cleaned)), + nan_per_column=nan_per_col, + removed_cost=int(out.get("cost_total", 0)), + message=f"Cleaned {len(df)}→{len(cleaned)} rows. Cleaned series at {cleaned_file}.", + ) + except Exception as exc: + logger.error("data_quality failed: %s", exc) + return ErrorResult(error=str(exc)) -@mcp.tool(title="Run TSFM Finetuning") -def run_tsfm_finetuning( +@mcp.tool(title="Run Tabular Recipe") +def run_tabular_recipe( dataset_path: str, - timestamp_column: str, - target_columns: List[str], - model_checkpoint: str = "ttm_96_28", - save_model_dir: str = "tuned_models", - forecast_horizon: int = -1, - n_finetune: float = 0.05, - n_calibration: float = 0.0, - n_test: float = 0.05, - conditional_columns: Optional[List[str]] = None, - id_columns: Optional[List[str]] = None, - frequency_sampling: str = "oov", - autoregressive_modeling: bool = True, - include_dataquality_summary: bool = False, -) -> Union[FinetuningResult, ErrorResult]: - """Few-shot fine-tune a TinyTimeMixer model on a local dataset. - - Returns a FinetuningResult with the saved checkpoint path and a JSON file - containing per-forecast-horizon performance metrics. - - Args: - dataset_path: Path to the training dataset (CSV/JSON/XLSX). - timestamp_column: Name of the timestamp column. - target_columns: Columns to forecast and fine-tune on. - model_checkpoint: Base model to start from (e.g. ttm_96_28). - save_model_dir: Directory to save the fine-tuned model checkpoint. - forecast_horizon: Steps to forecast; -1 uses the model default. - n_finetune: Fraction (≤1) or count (>1) of samples for fine-tuning. - n_calibration: Fraction or count for calibration set (default 0). - n_test: Fraction or count for test evaluation (default 0.05). - conditional_columns: Exogenous feature columns. - id_columns: ID columns for grouped time series. - frequency_sampling: Sampling frequency string or 'oov' to auto-detect. - autoregressive_modeling: Use autoregressive mode when True. - include_dataquality_summary: Attach a data-quality report to the result. + recipe: dict, + label_column: Optional[str] = None, + asset_id: str = "asset", +) -> Union[TabularResult, ErrorResult]: + """Series→tabular run (regression/classification/clustering): each row of the CSV file + pointer is an instance; ``label_column`` (if supervised) holds y. FeatureUnion → estimator. """ if not dataset_path.strip(): return ErrorResult(error="dataset_path is required") - if not target_columns: - return ErrorResult(error="target_columns must not be empty") - + bad = _check_recipe(recipe) + if bad: + return ErrorResult(error=bad) try: - import tsfm_public # noqa: F401 - except ImportError as exc: - return ErrorResult(error=f"tsfm dependencies unavailable: {exc}") - - model_checkpoint = _get_model_checkpoint_path(model_checkpoint) - dataset_path = _get_dataset_path(dataset_path) - abs_save_dir = _get_outputs_path(save_model_dir) - dataset_config = _build_dataset_config( - timestamp_column, - target_columns, - conditional_columns, - id_columns, - frequency_sampling, - autoregressive_modeling, - ) + df = pd.read_csv(refs._path(dataset_path)) + y = None + if label_column: + if label_column not in df.columns: + return ErrorResult( + error=f"label_column '{label_column}' not in dataset" + ) + y = df[label_column].to_numpy() + df = df.drop(columns=[label_column]) + X = df.apply(pd.to_numeric, errors="coerce").fillna(0.0).to_numpy() + res = composition.run_tabular_recipe(_STORE, X, recipe, y=y, asset_id=asset_id) + results_file = refs.write_json(res, name="tabular_run") + return TabularResult( + status="success", + run_id=res["run_id"], + results_file=results_file, + task=res["task"], + metric=res["metric"], + cv_score=res["cv_score"], + n_features=res["n_features"], + message=f"Tabular run complete ({res['task']}). Record at {results_file}.", + ) + except Exception as exc: + logger.error("run_tabular_recipe failed: %s", exc) + return ErrorResult(error=str(exc)) + +@mcp.tool(title="Run Plan") +def run_plan( + plan_spec: dict, asset_id: str = "asset", scenario_id: Optional[str] = None +) -> Union[PlanResult, ErrorResult]: + """Execute a recipe DAG (file-pointer chaining; HuggingGPT task-list).""" + if not plan_spec: + return ErrorResult(error="plan_spec is required") try: - data_df = _read_ts_data(dataset_path, dataset_config_dictionary=dataset_config) - model_config = _load_model_config(model_checkpoint) + res = plan.run_plan( + _STORE, plan_spec, asset_id=asset_id, scenario_id=scenario_id + ) + results_file = refs.write_json(res, name="plan_run") + return PlanResult( + status="success", + results_file=results_file, + message=f"Plan complete. Record at {results_file}.", + **res, + ) + except Exception as exc: + logger.error("run_plan failed: %s", exc) + return ErrorResult(error=str(exc)) - os.makedirs(abs_save_dir, exist_ok=True) - output_data_quality = _tsfm_data_quality_filter( - data_df, dataset_config, model_config, task="finetuning" +@mcp.tool(title="Evaluate (GIFT-Eval)") +def evaluate(recipe: dict, configs: List[dict]) -> Union[EvaluateResult, ErrorResult]: + """GIFT-Eval: seasonal-naive-normalized MASE+CRPS, geo-mean over configs.""" + bad = _check_recipe(recipe) + if bad: + return ErrorResult(error=bad) + if not configs: + return ErrorResult(error="configs must not be empty") + try: + res = gifteval.evaluate_recipe(_STORE, recipe, configs) + results_file = refs.write_json(res, name="gifteval") + return EvaluateResult( + status="success", + results_file=results_file, + message=f"Evaluation complete. Scores at {results_file}.", + **res, ) - data_df = output_data_quality["data"] - dataset_config = output_data_quality["dataset_config_dictionary"] + except Exception as exc: + logger.error("evaluate failed: %s", exc) + return ErrorResult(error=str(exc)) - if len(data_df) == 0: - return ErrorResult( - error="Data quality was poor; after filtering, no continuous segment satisfied the " - "context length requirement. Check Data Quality Summary." - ) - output = _finetune_ttm_hf( - data_df, - dataset_config, - model_config, - abs_save_dir, - n_finetune, - n_calibration, - n_test, - model_checkpoint=model_checkpoint, - ) +# ---- write-back ---- +@mcp.tool(title="Register Model") +def register_model(model: dict) -> Union[RegisterResult, ErrorResult]: + if not model: + return ErrorResult(error="model card is required") + try: + rec = model_store.register_model(_STORE, model, overwrite=True) + return RegisterResult(status="registered", id=rec.get("model_id", ""), card=rec) + except Exception as exc: + logger.error("register_model failed: %s", exc) + return ErrorResult(error=str(exc)) + - result_dict = output.copy() - result_dict["performance"] = result_dict["performance"].to_dict() +@mcp.tool(title="Register Feature") +def register_feature(feature: dict) -> Union[RegisterResult, ErrorResult]: + if not feature: + return ErrorResult(error="feature card is required") + try: + rec = feature_store.register_feature(_STORE, feature, overwrite=True) + return RegisterResult( + status="registered", id=rec.get("feature_id", ""), card=rec + ) + except Exception as exc: + logger.error("register_feature failed: %s", exc) + return ErrorResult(error=str(exc)) - if "performance" in result_dict: - df_perf = pd.DataFrame(result_dict["performance"]) - df_perf["forecast"] = df_perf["forecast"].values + 1 - max_forecast = df_perf["forecast"].max() - if 0 < forecast_horizon <= max_forecast: - result_dict["performance"] = df_perf.loc[ - df_perf["forecast"] == forecast_horizon - ].to_dict() - if include_dataquality_summary: - result_dict["dataquality_summary"] = output_data_quality[ - "dataquality_summary" - ] +# ════════════════════════ catalog management (lifecycle) ═════════════════════ +# Pull + update + version + retire, for both stores. The agent curates the catalog: search it, +# trace lineage, edit/retire a card, cut a new version, or register a fine-tuned checkpoint. - results_file = _write_json_to_temp(json.dumps(result_dict, indent=4)) +# ---- model store ---- +@mcp.tool(title="List Models") +def list_models( + task_id: Optional[str] = None, domain: Optional[str] = None, status: str = "active" +) -> Union[ModelsResult, ErrorResult]: + """List model cards in the catalog (optionally filtered by task / domain). Unranked — the + mirror of list_extractors; use find_models / describe_candidates to rank for a task. + """ + if task_id: + bad = _check_task(task_id) + if bad: + return ErrorResult(error=bad) + try: + return ModelsResult( + models=model_store.list_models( + _STORE, task_id=task_id, domain=domain, status=status + ) + ) except Exception as exc: - logger.error("run_tsfm_finetuning failed: %s", exc) + logger.error("list_models failed: %s", exc) return ErrorResult(error=str(exc)) + +@mcp.tool(title="Search Models") +def search_models( + text: str = "", tags: Optional[List[str]] = None, status: str = "active" +) -> Union[ModelsResult, ErrorResult]: + """Free-text/tag search over the model catalog (id, description, family, tags).""" try: - fewshot_dir = abs_save_dir + "/fewshot/" - saved_checkpoint = ( - _find_largest_tsfm_checkpoint_directory(fewshot_dir) or abs_save_dir - ) + "/" + return ModelsResult( + models=model_store.search(_STORE, text, tags=tags, status=status) + ) except Exception as exc: - logger.warning("Could not resolve finetuned checkpoint path: %s", exc) - saved_checkpoint = save_model_dir + logger.error("search_models failed: %s", exc) + return ErrorResult(error=str(exc)) - return FinetuningResult( - status="success", - model_checkpoint=saved_checkpoint, - results_file=results_file, - message=( - f"Fine-tuning complete. Model saved to {saved_checkpoint}. " - f"Metrics saved to {results_file}." - ), - ) +@mcp.tool(title="Get Model Lineage") +def get_model_lineage(model_id: str) -> Union[LineageResult, ErrorResult]: + """A model's version chain — what it supersedes / is superseded by (the evolution trail).""" + if not model_id.strip(): + return ErrorResult(error="model_id is required") + try: + return LineageResult(**model_store.get_lineage(_STORE, model_id)) + except Exception as exc: + logger.error("get_model_lineage failed: %s", exc) + return ErrorResult(error=str(exc)) -# ── TSAD (conformal anomaly detection on top of TSFM forecasts) ────────────── +@mcp.tool(title="Update Model") +def update_model(model_id: str, fields: dict) -> Union[CardResult, ErrorResult]: + """Patch fields on a model card (status, metrics, tags, domain, …).""" + if not model_id.strip() or not fields: + return ErrorResult(error="model_id and fields are required") + try: + return CardResult(**model_store.update_model(_STORE, model_id, fields)) + except Exception as exc: + logger.error("update_model failed: %s", exc) + return ErrorResult(error=str(exc)) -@mcp.tool(title="Run Anomaly Detection") -def run_tsad( - dataset_path: str, - tsfm_output_json: str, - timestamp_column: str, - target_columns: List[str], - task: str = "fit", - false_alarm: float = 0.05, - ad_model_type: str = "timeseries_conformal_adaptive", - ad_model_checkpoint: Optional[str] = None, - ad_model_save: Optional[str] = None, - n_calibration: float = 0.2, - conditional_columns: Optional[List[str]] = None, - id_columns: Optional[List[str]] = None, - frequency_sampling: Optional[str] = None, - autoregressive_modeling: bool = True, -) -> Union[TSADResult, ErrorResult]: - """Run conformal anomaly detection on TSFM forecasting output. - - tsfm_output_json must be the results_file path returned by run_tsfm_forecasting. - Fits (or loads) a conformal AD model and saves anomaly-labelled predictions to CSV. - - Args: - dataset_path: Path to the original time-series dataset. - tsfm_output_json: Path to JSON predictions file from run_tsfm_forecasting. - timestamp_column: Name of the timestamp column. - target_columns: Target columns that were forecast. - task: 'fit' to train a new AD model, 'inference' to use an existing one. - false_alarm: False alarm rate (1 − coverage); default 0.05 → 95% coverage. - ad_model_type: 'timeseries_conformal' or 'timeseries_conformal_adaptive'. - ad_model_checkpoint: Path to an existing AD model (required for 'inference'). - ad_model_save: Directory to save the fitted AD model. - n_calibration: Fraction of data used for calibration (default 0.2). - conditional_columns: Exogenous feature columns. - id_columns: ID columns for grouped time series. - frequency_sampling: Sampling frequency string or None to auto-detect. - autoregressive_modeling: Use autoregressive mode when True. - """ - if not dataset_path.strip(): - return ErrorResult(error="dataset_path is required") - if not tsfm_output_json.strip(): - return ErrorResult(error="tsfm_output_json is required") - if not target_columns: - return ErrorResult(error="target_columns must not be empty") - if task not in ("fit", "inference"): - return ErrorResult(error="task must be 'fit' or 'inference'") +@mcp.tool(title="Deprecate Model") +def deprecate_model( + model_id: str, reason: Optional[str] = None +) -> Union[CardResult, ErrorResult]: + """Retire a model card (status=deprecated); it stops appearing in active listings.""" + if not model_id.strip(): + return ErrorResult(error="model_id is required") try: - import tsfm_public # noqa: F401 - except ImportError as exc: - return ErrorResult(error=f"tsfm dependencies unavailable: {exc}") + return CardResult( + **model_store.deprecate_model(_STORE, model_id, reason=reason) + ) + except Exception as exc: + logger.error("deprecate_model failed: %s", exc) + return ErrorResult(error=str(exc)) - dataset_config = _build_dataset_config( - timestamp_column, - target_columns, - conditional_columns, - id_columns, - frequency_sampling or "", - autoregressive_modeling, - ) +@mcp.tool(title="New Model Version") +def new_model_version( + model_id: str, fields: dict, new_model_id: Optional[str] = None +) -> Union[CardResult, ErrorResult]: + """Create a successor version of a model; the predecessor is marked superseded + linked.""" + if not model_id.strip(): + return ErrorResult(error="model_id is required") try: - with open(tsfm_output_json, "r") as fh: - tsmodel_pred = json.load(fh) + return CardResult( + **model_store.new_version( + _STORE, model_id, fields or {}, new_model_id=new_model_id + ) + ) + except Exception as exc: + logger.error("new_model_version failed: %s", exc) + return ErrorResult(error=str(exc)) + - output = _TimeSeriesAnomalyDetectionConformalWrapper().run( - dataset_path, - dataset_config, - tsmodel_pred, - ad_model_checkpoint=ad_model_checkpoint, - ad_model_save=ad_model_save, - task=task, - ad_model_type=ad_model_type, - n_calibration=n_calibration, - false_alarm=false_alarm, +@mcp.tool(title="Register Finetuned Model") +def register_finetuned( + model_id: str, + checkpoint_path: str, + base_model_id: str, + context_length: int, + prediction_length: int, + description: str, + domain: str = "general", + metrics: Optional[list] = None, +) -> Union[CardResult, ErrorResult]: + """Add a fine-tuned model: point the catalog at a checkpoint, with lineage to its base model.""" + for k, v in ( + ("model_id", model_id), + ("checkpoint_path", checkpoint_path), + ("base_model_id", base_model_id), + ("description", description), + ): + if not (v and str(v).strip()): + return ErrorResult(error=f"{k} is required") + try: + return CardResult( + **model_store.register_finetuned( + _STORE, + model_id=model_id, + checkpoint_path=checkpoint_path, + base_model_id=base_model_id, + context_length=context_length, + prediction_length=prediction_length, + description=description, + domain=domain, + metrics=metrics, + ) ) except Exception as exc: - logger.error("run_tsad failed: %s", exc) + logger.error("register_finetuned failed: %s", exc) return ErrorResult(error=str(exc)) + +# ---- feature store ---- +@mcp.tool(title="Search Features") +def search_features( + text: str = "", tags: Optional[List[str]] = None, status: str = "active" +) -> Union[FeaturesResult, ErrorResult]: + """Free-text/tag search over the feature catalog (id, name, tags).""" try: - df = _tsad_output_to_df(output) - tmp_dir = tempfile.mkdtemp() - csv_path = os.path.join(tmp_dir, f"tsad_output_{uuid.uuid4()}.csv") - df.to_csv(csv_path, index=False) - anomaly_count = ( - int(df["anomaly_label"].sum()) if "anomaly_label" in df.columns else 0 + return FeaturesResult( + features=feature_store.search(_STORE, text, tags=tags, status=status) ) except Exception as exc: - logger.error("run_tsad result serialisation failed: %s", exc) - return ErrorResult(error=f"Failed to serialise TSAD output: {exc}") - - return TSADResult( - status="success", - results_file=csv_path, - total_records=len(df), - anomaly_count=anomaly_count, - columns=list(df.columns), - message=( - f"Anomaly detection complete. {anomaly_count} anomalies in {len(df)} records. " - f"Results saved to {csv_path}." - ), - ) + logger.error("search_features failed: %s", exc) + return ErrorResult(error=str(exc)) -# ── Integrated TSAD (forecasting + anomaly detection in one call) ───────────── +@mcp.tool(title="List Extractors") +def list_extractors() -> Union[FeaturesResult, ErrorResult]: + """Browse the FLOps extractor library.""" + try: + return FeaturesResult( + features=feature_store.list_extractors(_STORE) + ) + except Exception as exc: + logger.error("list_extractors failed: %s", exc) + return ErrorResult(error=str(exc)) -@mcp.tool(title="Run Integrated Forecasting + Anomaly Detection") -def run_integrated_tsad( - dataset_path: str, - timestamp_column: str, - target_columns: List[str], - model_checkpoint: str = "ttm_96_28", - false_alarm: float = 0.05, - ad_model_type: str = "timeseries_conformal_adaptive", - n_calibration: float = 0.2, - conditional_columns: Optional[List[str]] = None, - id_columns: Optional[List[str]] = None, - frequency_sampling: str = "", - autoregressive_modeling: bool = True, -) -> Union[TSADResult, ErrorResult]: - """Run end-to-end time-series forecasting + anomaly detection in one call. - - For each target column: runs zero-shot TTM forecasting, then fits a conformal - AD model and predicts anomaly labels. Saves a combined CSV with anomaly labels - and KPI scores for all columns. - - Args: - dataset_path: Path to the dataset (CSV/JSON/XLSX). - timestamp_column: Name of the timestamp column. - target_columns: Columns to run forecasting + anomaly detection on. - model_checkpoint: Pre-trained TTM model name (default: ttm_96_28). - false_alarm: False alarm rate; default 0.05 → 95% coverage. - ad_model_type: 'timeseries_conformal' or 'timeseries_conformal_adaptive'. - n_calibration: Fraction of data for AD calibration (default 0.2). - conditional_columns: Exogenous feature columns. - id_columns: ID columns for grouped time series. - frequency_sampling: Sampling frequency string or '' to auto-detect. - autoregressive_modeling: Use autoregressive mode when True. - """ - if not dataset_path.strip(): - return ErrorResult(error="dataset_path is required") - if not target_columns: - return ErrorResult(error="target_columns must not be empty") +@mcp.tool(title="Get Feature Lineage") +def get_feature_lineage(feature_id: str) -> Union[LineageResult, ErrorResult]: + """A feature's evolution chain (parent / generation / descendants).""" + if not feature_id.strip(): + return ErrorResult(error="feature_id is required") + try: + return LineageResult(**feature_store.get_lineage(_STORE, feature_id)) + except Exception as exc: + logger.error("get_feature_lineage failed: %s", exc) + return ErrorResult(error=str(exc)) + +@mcp.tool(title="Update Feature") +def update_feature(feature_id: str, fields: dict) -> Union[CardResult, ErrorResult]: + """Patch fields on a feature card (status, metrics, tags, scenario_categories, …).""" + if not feature_id.strip() or not fields: + return ErrorResult(error="feature_id and fields are required") try: - import tsfm_public # noqa: F401 - except ImportError as exc: - return ErrorResult(error=f"tsfm dependencies unavailable: {exc}") + return CardResult(**feature_store.update_feature(_STORE, feature_id, fields)) + except Exception as exc: + logger.error("update_feature failed: %s", exc) + return ErrorResult(error=str(exc)) - model_checkpoint = _get_model_checkpoint_path(model_checkpoint) - dataset_path = _get_dataset_path(dataset_path) +@mcp.tool(title="Deprecate Feature") +def deprecate_feature( + feature_id: str, reason: Optional[str] = None +) -> Union[CardResult, ErrorResult]: + """Retire a feature card (status=deprecated).""" + if not feature_id.strip(): + return ErrorResult(error="feature_id is required") try: - ad_model_save = _get_outputs_path("tsad_model_save/") - os.makedirs(ad_model_save, exist_ok=True) + return CardResult( + **feature_store.deprecate_feature(_STORE, feature_id, reason=reason) + ) + except Exception as exc: + logger.error("deprecate_feature failed: %s", exc) + return ErrorResult(error=str(exc)) - model_config = _load_model_config(model_checkpoint) - df_combined = pd.DataFrame() - # Read the full dataset once with all target columns, then subset per column - full_config = _build_dataset_config( - timestamp_column, - target_columns, - conditional_columns, - id_columns, - frequency_sampling, - autoregressive_modeling, +@mcp.tool(title="New Feature Version") +def new_feature_version( + feature_id: str, fields: dict, new_feature_id: Optional[str] = None +) -> Union[CardResult, ErrorResult]: + """Create a successor version of a feature (EFE lineage); predecessor superseded + linked.""" + if not feature_id.strip(): + return ErrorResult(error="feature_id is required") + try: + return CardResult( + **feature_store.new_version( + _STORE, feature_id, fields or {}, new_feature_id=new_feature_id + ) ) - full_data_df = _read_ts_data( - dataset_path, dataset_config_dictionary=full_config + except Exception as exc: + logger.error("new_feature_version failed: %s", exc) + return ErrorResult(error=str(exc)) + + +# ---- results / runs ---- +@mcp.tool(title="Get Result") +def get_result(task_type: str, result_id: str) -> Union[ResultRecord, ErrorResult]: + rec = results.get_result(_STORE, task_type, result_id) + return ResultRecord(**rec) if rec else ErrorResult(error="not found") + + +@mcp.tool(title="List Results") +def list_results( + task_type: str, asset_id: Optional[str] = None, scenario_id: Optional[str] = None +) -> ResultsListResult: + return ResultsListResult( + results=results.list_results( + _STORE, task_type, asset_id=asset_id, scenario_id=scenario_id ) + ) + - for col in target_columns: - col_config = _build_dataset_config( - timestamp_column, - [col], - conditional_columns, - id_columns, - frequency_sampling, - autoregressive_modeling, +@mcp.tool(title="Get Run") +def get_run(run_id: str) -> Union[RunRecord, ErrorResult]: + rec = _STORE.get(RUNS_COLLECTION, run_id) + return RunRecord(**rec) if rec else ErrorResult(error="not found") + + +@mcp.tool(title="List Runs") +def list_runs(asset_id: Optional[str] = None) -> RunsResult: + sel = {"asset_id": asset_id} if asset_id else {} + return RunsResult( + runs=_STORE.find(RUNS_COLLECTION, sel), plans=_STORE.find(PLANS_COLLECTION, sel) + ) + + +# ════════════════════════════ evolve (AlphaEvolve loop) ═════════════════════ +# Agent-generates / server-grades: the server samples + validates + evaluates + archives; +# the agent (LLM) is the proposer. A "program" is a recipe OR an EFE feature program. +@mcp.tool(title="Evolve: Ask") +def evolve_ask( + task: str, + kind: str = "recipe", + dataset_path: Optional[str] = None, + timestamp_column: Optional[str] = None, + channels: Optional[List[str]] = None, + n_parents: int = 2, + n_inspirations: int = 3, +) -> Union[EvolveAskResult, ErrorResult]: + """Sample parent(s) + diverse inspirations from the evolutionary archive + data evidence + + the task contract, so YOU (the agent) can mutate/recombine them into one new candidate to + submit via evolve_tell. kind = 'recipe' | 'feature'.""" + bad = _check_task(task) + if bad: + return ErrorResult(error=bad) + try: + return EvolveAskResult( + **evolve.evolve_ask( + _STORE, + task, + kind=kind, + data_ref=dataset_path, + timestamp_column=timestamp_column, + channels=channels, + n_parents=n_parents, + n_inspirations=n_inspirations, ) + ) + except Exception as exc: + logger.error("evolve_ask failed: %s", exc) + return ErrorResult(error=str(exc)) - # 1. Quality-filter data for this column (reuse already-loaded data) - data_df = full_data_df - output_dq = _tsfm_data_quality_filter( - data_df, col_config, model_config, task="inference" + +@mcp.tool(title="Evolve: Tell") +def evolve_tell( + task: str, + kind: str, + program: dict, + parent_id: Optional[str] = None, + dataset_path: Optional[str] = None, + timestamp_column: Optional[str] = None, + target_columns: Optional[List[str]] = None, + label_column: Optional[str] = None, +) -> Union[EvolveTellResult, ErrorResult]: + """Submit a candidate PROGRAM (a recipe or a feature program). The server validates it, + evaluates it to a scalar fitness (run_recipe / run_tabular_recipe / EFE gate), and places it + in the MAP-Elites archive with lineage. Returns the fitness + whether it's a new elite. + """ + bad = _check_task(task) + if bad: + return ErrorResult(error=bad) + if kind not in ("recipe", "feature"): + return ErrorResult(error="kind must be 'recipe' or 'feature'") + if not program: + return ErrorResult(error="program is required") + try: + return EvolveTellResult( + **evolve.evolve_tell( + _STORE, + task, + kind, + program, + parent_id=parent_id, + data_ref=dataset_path, + timestamp_column=timestamp_column, + target_columns=target_columns, + label_column=label_column, ) - data_df_filtered = output_dq["data"] - col_config_filtered = output_dq["dataset_config_dictionary"] + ) + except Exception as exc: + logger.error("evolve_tell failed: %s", exc) + return ErrorResult(error=str(exc)) - if len(data_df_filtered) == 0: - logger.warning( - "Data quality filter removed all data for column %s; skipping.", col - ) - continue - - # 2. Zero-shot forecasting for this column - try: - forecast_output = _get_ttm_hf_inference( - data_df_filtered, - col_config_filtered, - model_config, - model_checkpoint, - ) - except Exception as exc: - logger.warning("Forecasting failed for column %s: %s", col, exc) - continue - - inference_data = { - "target_prediction": forecast_output["target_prediction"].tolist(), - "timestamp": np.array(forecast_output["timestamp_prediction"]) - .astype(str) - .tolist(), - "target_columns": forecast_output["target_columns"], - } - # 3. Conformal anomaly detection for this column - tsmodel_pred = inference_data - - try: - tsad_output = _TimeSeriesAnomalyDetectionConformalWrapper().run( - dataset_path, - col_config, - tsmodel_pred, - ad_model_checkpoint=None, - ad_model_save=ad_model_save, - task="fit", - ad_model_type=ad_model_type, - n_calibration=n_calibration, - false_alarm=false_alarm, - ) - except Exception as exc: - logger.warning("TSAD failed for column %s: %s", col, exc) - continue - - df_col = _tsad_output_to_df(tsad_output) - df_combined = pd.concat([df_combined, df_col], axis=0, ignore_index=True) - - if df_combined.empty: - return ErrorResult(error="No TSAD results produced for any target column.") - - tmp_dir = tempfile.mkdtemp() - csv_path = os.path.join(tmp_dir, f"integrated_tsad_{uuid.uuid4()}.csv") - df_combined.to_csv(csv_path, index=False) - anomaly_count = ( - int(df_combined["anomaly_label"].sum()) - if "anomaly_label" in df_combined.columns - else 0 + +@mcp.tool(title="Evolve: Best") +def evolve_best( + task: str, kind: Optional[str] = None, top_k: int = 5 +) -> Union[EvolveBestResult, ErrorResult]: + """The current elites (best program per behaviour cell) for a task — the evolved frontier.""" + bad = _check_task(task) + if bad: + return ErrorResult(error=bad) + try: + return EvolveBestResult( + **evolve.evolve_best( + _STORE, task, kind=kind, top_k=max(1, min(int(top_k), 50)) + ) ) + except Exception as exc: + logger.error("evolve_best failed: %s", exc) + return ErrorResult(error=str(exc)) + +@mcp.tool(title="Extract Features") +def extract_features( + dataset_path: str, + extractors: List[str], + timestamp_column: Optional[str] = None, + target_columns: Optional[List[str]] = None, + window: Optional[int] = None, +) -> Union[ExtractResult, ErrorResult]: + """Apply the chosen FLOps extractors to a series and RETURN the extracted feature values — + raw feature extraction, no model. Pick `extractors` by name from list_extractors. + window=None -> one feature vector for the whole series; window=W -> non-overlapping W-length + tiles -> a (windows x features) matrix. Multivariate: each target column yields its own + '.' feature columns.""" + from ..reasoning import feature_selection as FS + import numpy as np + import pandas as pd + + if not extractors: + return ErrorResult(error="provide at least one extractor name (see list_extractors)") + unknown = [e for e in extractors if e not in FS.EXTRACTORS] + if unknown: + return ErrorResult(error=f"unknown extractor(s): {unknown}. See list_extractors.") + try: + obj = refs.load_series(dataset_path, time_col=timestamp_column, channels=target_columns) except Exception as exc: - logger.error("run_integrated_tsad failed: %s", exc) + logger.error("extract_features load failed: %s", exc) return ErrorResult(error=str(exc)) - return TSADResult( - status="success", - results_file=csv_path, - total_records=len(df_combined), - anomaly_count=anomaly_count, - columns=list(df_combined.columns), - message=( - f"Integrated TSAD complete. {anomaly_count} anomalies in {len(df_combined)} records " - f"across {len(target_columns)} column(s). Results saved to {csv_path}." - ), + if isinstance(obj, pd.Series): + name = obj.name or (target_columns[0] if target_columns else "value") + channels = {str(name): np.asarray(obj, dtype=float)} + else: + channels = {str(c): np.asarray(obj[c], dtype=float) for c in obj.columns} + + cols, F = composition.extract_features(channels, extractors, window=window) + return ExtractResult( + n_windows=int(F.shape[0]), + window=window, + columns=cols, + features=[[round(float(v), 6) for v in row] for row in F.tolist()], + message=(f"extracted {len(cols)} feature column(s) over {F.shape[0]} window(s) " + f"from {len(channels)} channel(s)."), ) - def main(): mcp.run(transport="stdio") diff --git a/src/servers/tsfm/metrics.py b/src/servers/tsfm/metrics.py deleted file mode 100644 index 1068c8a5d..000000000 --- a/src/servers/tsfm/metrics.py +++ /dev/null @@ -1,274 +0,0 @@ -"""Forecasting metrics, spectral/trend loss functions, and frequency token constants.""" - -from __future__ import annotations - -import numpy as np - - -# ── Basic forecasting metrics ───────────────────────────────────────────────── - - -def _RMSE(y_true, y_pred, axis=None): - values = ( - np.mean((y_true - y_pred) ** 2) - if axis is None - else np.mean((y_true - y_pred) ** 2, axis=axis) - ) - return np.sqrt(values) - - -def _MAE(y_true, y_pred, axis=None): - if axis is None: - return np.mean(np.abs(y_true - y_pred)) - return np.mean(np.abs(y_true - y_pred), axis=axis) - - -def _MAPE(y_true, y_pred, axis=None): - non_zero_mask = np.array(y_true != 0).astype("int") - y_true_denom = np.array(y_true) * non_zero_mask + (1 - non_zero_mask) * 1e-15 - values = np.abs((y_true - y_pred)) / np.abs(y_true_denom) - if axis is None: - if np.sum(non_zero_mask) > 0: - return np.sum(values * non_zero_mask) / np.sum(non_zero_mask) * 100 - return None - if np.sum(non_zero_mask) > 0: - numerator = np.sum(values * non_zero_mask, axis=axis) - denominator = np.sum(non_zero_mask, axis=axis) - output = 100 * numerator - output[denominator == 0] = None - output[denominator != 0] /= denominator[denominator != 0] - return output - return None - - -def _SMAPE(y_true, y_pred, axis=None): - denominator = (np.abs(y_true) + np.abs(y_pred)) / 2 - non_zero_mask = np.array(denominator != 0).astype("int") - denominator = np.array(denominator) * non_zero_mask + (1 - non_zero_mask) * 1e-15 - values = np.abs(y_true - y_pred) / denominator - if axis is None: - return np.mean(values) * 100 - return np.mean(values, axis=axis) * 100 - - -def _WAPE(y_true, y_pred, axis=None): - numerator = np.abs((y_true - y_pred)) - if axis is None: - denominator = np.sum(np.abs(y_true)) - if denominator == 0: - return None - return np.sum(numerator) / denominator - denominator = np.sum(np.abs(y_true), axis=axis) - values = np.sum(numerator, axis=axis) - values[denominator == 0] = None - values[denominator != 0] /= denominator[denominator != 0] - return values - - -def _Bias(y_true, y_pred, axis=None): - delta = y_pred - y_true - if axis is None: - return np.mean(delta) - return np.mean(delta, axis=axis) - - -def _NRMSE(y_true, y_pred, axis=None, norm="mean"): - values = _RMSE(y_true, y_pred, axis=axis) - den = (np.max(y_true) - np.min(y_true)) if norm == "minmax" else np.mean(y_true) - return np.sqrt(values) / np.abs(den) - - -def _cosine_similarity_matrix(A, B, axis=1): - dot_product = np.sum(A * B, axis=axis) - norm_A = np.linalg.norm(A, axis=axis) - norm_B = np.linalg.norm(B, axis=axis) - return dot_product / (norm_A * norm_B) - - -# ── Spectral / trend losses (require torch lazily) ──────────────────────────── - - -def _loss_helper(outputs, targets, fn, axis=1): - import torch - - outputs = outputs.astype(np.float64) - targets = targets.squeeze() - outputs = outputs.squeeze() - if len(targets.shape) == 0 or targets.shape[0] == 0: - return np.array(0.0) - if len(targets.shape) == 1: - if targets.shape[0] < 4: - return np.array([0.0]) - return ( - fn( - torch.from_numpy(targets.reshape(1, -1)), - torch.from_numpy(outputs.reshape(1, -1)), - ) - .cpu() - .detach() - .item() - ) - B, T = targets.shape - if axis == 1: - if T < 4: - return np.array([0.0]) - return ( - fn(torch.from_numpy(targets), torch.from_numpy(outputs)) - .cpu() - .detach() - .numpy() - ) - return np.array(0.0) - - -def _amp_loss(outputs, targets): - import torch - - B, T = outputs.shape - fft_size = 1 << (2 * T - 1).bit_length() - out_f = torch.fft.fft(outputs, fft_size, dim=-1) - tgt_f = torch.fft.fft(targets, fft_size, dim=-1) - out_norm = torch.linalg.vector_norm(outputs, dim=-1) - tgt_norm = torch.linalg.vector_norm(targets, dim=-1) - auto_corr = torch.fft.ifft(tgt_f * tgt_f.conj(), dim=-1).real - auto_corr = torch.cat([auto_corr[..., -(T - 1) :], auto_corr[..., :T]], dim=-1) - norm = torch.where(tgt_norm == 0, 1e-9, tgt_norm * tgt_norm) - nac_tgt = auto_corr / norm.unsqueeze(1) - cross_corr = torch.fft.ifft(tgt_f * out_f.conj(), dim=-1).real - cross_corr = torch.cat([cross_corr[..., -(T - 1) :], cross_corr[..., :T]], dim=-1) - norm2 = torch.where(tgt_norm * out_norm == 0, 1e-9, tgt_norm * out_norm) - nac_out = cross_corr / (tgt_norm * out_norm).unsqueeze(1) - return torch.mean(torch.abs(nac_tgt - nac_out), dim=-1) - - -def _ashift_loss(outputs, targets): - import torch - - B, T = outputs.shape - return T * torch.mean( - torch.abs(1 / T - torch.softmax(outputs - targets, dim=-1)), dim=-1 - ) - - -def _phase_loss(outputs, targets): - import torch - - B, T = outputs.shape - out_f = torch.fft.fft(outputs, dim=-1) - tgt_f = torch.fft.fft(targets, dim=-1) - tgt_f_sq = tgt_f.real**2 + tgt_f.imag**2 - mask = (tgt_f_sq > T).float() - topk_indices = tgt_f_sq.topk(k=int(T**0.5), dim=-1).indices - mask = mask.scatter_(-1, topk_indices, 1.0) - mask[..., 0] = 1.0 - mask = torch.where(mask > 0, 1.0, 0.0) - mask = mask.bool() - not_mask = (~mask).float() - not_mask /= torch.mean(not_mask, dim=-1).unsqueeze(1) - zero_error = torch.abs(out_f) * not_mask - zero_error = torch.where( - torch.isnan(zero_error), torch.zeros_like(zero_error), zero_error - ) - mask_f = mask.float() - mask_f /= torch.mean(mask_f, dim=-1).unsqueeze(1) - ae = torch.abs(out_f - tgt_f) * mask_f - ae = torch.where(torch.isnan(ae), torch.zeros_like(ae), ae) - return (torch.mean(zero_error, dim=-1) + torch.mean(ae, dim=-1)) / (T**0.5) - - -def _tildeq_loss(target, output): - amp = _amp_loss(target, output) - shift = _ashift_loss(target, output) - phase = _phase_loss(target, output) - return 0.5 * phase + 0.5 * shift + 0.01 * amp - - -def _TILDEQ(outputs, targets, axis=1): - return _loss_helper(outputs, targets, _tildeq_loss, axis=axis) - - -def _derivatives(inp, device="cpu"): - import torch - - batch_size, lens = inp.shape[0:2] - input2 = inp[:, 2:lens].to(device) - input1 = inp[:, 0 : lens - 2].to(device) - return input2 - input1 - - -def _w_mse(targets, outputs, device="cpu"): - import torch - - batch_size, lens = targets.shape[0:2] - t1 = targets[:, 1:lens].to(device) - t2 = targets[:, 0 : lens - 1].to(device) - o1 = outputs[:, 1:lens].to(device) - o2 = outputs[:, 0 : lens - 1].to(device) - sigma = torch.tanh((t1 - t2) * (o1 - o2)) - nt = targets[:, 1:lens].to(device) - no = outputs[:, 1:lens].to(device) - return torch.abs(no - nt) * (1.0 - sigma) - - -def _trend_loss(targets, outputs, alpha=0.5, device="cpu"): - import torch - - sq_error = _w_mse(targets, outputs, device) - error1 = torch.mean(sq_error, dim=-1) - x1 = _derivatives(targets, device) - x2 = _derivatives(outputs, device) - _xt1 = x1.squeeze() - _xt2 = x2.squeeze() - if len(_xt1.shape) == 1: - _xt1 = torch.reshape(_xt1, (1, _xt1.shape[0])) - _xt1 = _xt1.T - _xt2 = _xt2.T - xc1 = (_xt1 - _xt1.mean(dim=0)).T - xc2 = (_xt2 - _xt2.mean(dim=0)).T - p_corr = torch.nn.functional.cosine_similarity(xc1, xc2, dim=-1) - w_corr = 1 - p_corr - dd = torch.norm(targets - outputs, dim=-1) - return error1 + alpha * w_corr * dd - - -def _TREND(targets, outputs, axis=1): - return _loss_helper(outputs, targets, _trend_loss, axis=axis) - - -_METRICS_FORECAST = { - "RMSE": _RMSE, - "MAE": _MAE, - "MAPE": _MAPE, - "SMAPE": _SMAPE, - "WAPE": _WAPE, - "Bias": _Bias, - "NRMSE": _NRMSE, - "TREND": _TREND, - "TILDEQ": _TILDEQ, - "COSSIM": _cosine_similarity_matrix, -} - - -# ── Frequency token constants ───────────────────────────────────────────────── - -_freq_token_mapping = { - "oov": 0, - "minutely": 1, - "2_minutes": 2, - "5_minutes": 3, - "10_minutes": 4, - "15_minutes": 5, - "half_hourly": 6, - "hourly": 7, -} -_freq_token_to_minutes = { - "oov": None, - "minutely": 1, - "2_minutes": 2, - "5_minutes": 5, - "10_minutes": 10, - "15_minutes": 15, - "half_hourly": 30, - "hourly": 60, -} -_TSFREQUENCY_TOLERANCE = 0.2 diff --git a/src/servers/tsfm/models.py b/src/servers/tsfm/models.py deleted file mode 100644 index 508168cf2..000000000 --- a/src/servers/tsfm/models.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Pydantic result models and static data for the TSFM MCP server.""" - -from __future__ import annotations - -from typing import Any, List, Optional - -from pydantic import BaseModel - - -# ── Static data ─────────────────────────────────────────────────────────────── - -_AI_TASKS = [ - { - "task_id": "tsfm_integrated_tsad", - "task_description": "Time series Anomaly detection", - }, - { - "task_id": "tsfm_forecasting", - "task_description": "Time series Multivariate Forecasting", - }, - { - "task_id": "tsfm_forecasting_finetune", - "task_description": "Finetuning of Multivariate Forecasting models", - }, - { - "task_id": "tsfm_forecasting_evaluation", - "task_description": "Evaluation of Forecasting models", - }, -] - -_TSFM_MODELS = [ - { - "model_id": "ttm_96_28", - "model_checkpoint": "ttm_96_28", - "model_description": "Pretrained forecasting model with context length 96", - }, - { - "model_id": "ttm_512_96", - "model_checkpoint": "ttm_512_96", - "model_description": "Pretrained forecasting model with context length 512", - }, - { - "model_id": "ttm_energy_96_28", - "model_checkpoint": "ttm_96_28", - "model_description": "Pretrained forecasting model tuned on energy data with context length 96", - }, - { - "model_id": "ttm_energy_512_96", - "model_checkpoint": "ttm_512_96", - "model_description": "Pretrained forecasting model tuned on energy data with context length 512", - }, -] - - -# ── Result models ───────────────────────────────────────────────────────────── - - -class ErrorResult(BaseModel): - error: str - - -class AITaskEntry(BaseModel): - task_id: str - task_description: str - - -class AITasksResult(BaseModel): - tasks: List[AITaskEntry] - - -class TSFMModelEntry(BaseModel): - model_id: str - model_checkpoint: str - model_description: str - - -class TSFMModelsResult(BaseModel): - models: List[TSFMModelEntry] - - -class ForecastingResult(BaseModel): - status: str - results_file: str - dataquality_summary: Optional[Any] = None - message: str - - -class FinetuningResult(BaseModel): - status: str - model_checkpoint: str - results_file: str - message: str - - -class TSADResult(BaseModel): - status: str - results_file: str - total_records: int - anomaly_count: int - columns: List[str] - message: str diff --git a/src/servers/tsfm/reasoning/__init__.py b/src/servers/tsfm/reasoning/__init__.py new file mode 100644 index 000000000..376f9ce67 --- /dev/null +++ b/src/servers/tsfm/reasoning/__init__.py @@ -0,0 +1 @@ +"""reasoning layer.""" diff --git a/src/servers/tsfm/dataquality.py b/src/servers/tsfm/reasoning/dataquality.py similarity index 97% rename from src/servers/tsfm/dataquality.py rename to src/servers/tsfm/reasoning/dataquality.py index 91de44ff5..ca43410b7 100644 --- a/src/servers/tsfm/dataquality.py +++ b/src/servers/tsfm/reasoning/dataquality.py @@ -1,4 +1,7 @@ -"""Time-series data quality helpers: NaN removal, segmentation, quality summary.""" +"""dataquality.py — time-series data-quality helpers (NaN removal, continuous-segment +segmentation, quality summary). Pure pandas/numpy, no ML deps — the layer-native port of +legacy/dataquality.py, used by the sktime forecasting/AD migration (Phase 2+). +""" from __future__ import annotations @@ -21,9 +24,9 @@ def _threshold_condition_function(threshold, condition_type="<"): ">=": lambda x: x >= threshold, "==": lambda x: x == threshold, } - assert condition_type in conditions, ( - f"condition_type {condition_type!r} is not supported" - ) + assert ( + condition_type in conditions + ), f"condition_type {condition_type!r} is not supported" return conditions[condition_type] diff --git a/src/servers/tsfm/reasoning/feature_selection.py b/src/servers/tsfm/reasoning/feature_selection.py new file mode 100644 index 000000000..d65e3e51b --- /dev/null +++ b/src/servers/tsfm/reasoning/feature_selection.py @@ -0,0 +1,606 @@ +"""FLOps-style feature learning — dynamic, dataset-specific feature selection. + +From Patel et al., "FLOps: On Learning Important Time Series Features for Real-Valued +Prediction" (IEEE BigData'20). FLOps takes a library of feature extractors, SCORES each on +the given input data, RANKS them against a Reference Feature, and FILTERS with a +Critical-Difference threshold — yielding a feature set tailored to *this* dataset+task. + +This is the **selection / learning** layer of the feature store, complementing: + - AnomalyKiTS Operators = the extractors/transforms themselves, + - EFE = generating/evolving new transforms, + - FLOps (here) = scoring + ranking + filtering which ones to use. + +Faithful FLOps-lite (numpy only): tabulate with a look-back window, score each extractor by +|corr| with the target (a stand-in for the paper's multi-config performance score), rank, +then keep extractors that beat the Reference Feature by a Critical-Difference margin. +""" + +from __future__ import annotations + +from typing import Callable, Dict, List, Optional + +import numpy as np + +def discover_lookback(series, max_lw: int = 128) -> int: + """Lookback window from the series' dominant period (autocorrelation peak), clamped to + [8, max_lw]; falls back to len/4 when there's no clear seasonality.""" + import numpy as np + x = np.asarray(series, dtype=float).ravel() + x = x[np.isfinite(x)] + n = x.size + if n < 16: + return max(2, min(n, max_lw)) + x = x - x.mean() + if not np.any(x): + return int(min(max(8, n // 4), max_lw)) + f = np.fft.rfft(x, n=2 * n) # autocorrelation via FFT + ac = np.fft.irfft(f * np.conj(f))[:n] + ac = ac / ac[0] + best_lag, best_val = 0, 0.0 + for k in range(2, min(n // 2, max_lw)): + if ac[k] > ac[k - 1] and ac[k] >= ac[k + 1] and ac[k] > best_val: + best_lag, best_val = k, ac[k] + if best_lag and best_val > 0.2: + return int(min(max(best_lag, 8), max_lw)) + return int(min(max(8, n // 4), max_lw)) + +# ---- a small slice of the FLOps 130+ extractor library (scalar extractors) ---- +EXTRACTORS: Dict[str, Callable[[np.ndarray], float]] = { + # Data Profiling (value-based, order-independent) + "mean": lambda w: float(np.mean(w)), + "std": lambda w: float(np.std(w)), + "min": lambda w: float(np.min(w)), + "max": lambda w: float(np.max(w)), + "range": lambda w: float(np.max(w) - np.min(w)), + "q25": lambda w: float(np.percentile(w, 25)), + "q75": lambda w: float(np.percentile(w, 75)), + "kurtosis": lambda w: float(_kurtosis(w)), + "skew": lambda w: float(_skew(w)), + # Temporal / order-dependent + "slope": lambda w: float(_slope(w)), + "autocorr1": lambda w: float(_autocorr(w, 1)), + "energy": lambda w: float(np.sum(np.asarray(w) ** 2) / len(w)), + "abs_diff_mean": lambda w: ( + float(np.mean(np.abs(np.diff(w)))) if len(w) > 1 else 0.0 + ), + # Frequency + "spectral_centroid": lambda w: float(_spectral_centroid(w)), + "dominant_freq_power": lambda w: float(_dominant_freq_power(w)), +} + + +def _kurtosis(w): + w = np.asarray(w, float) + s = w.std() + return 0.0 if s < 1e-9 else float(np.mean(((w - w.mean()) / s) ** 4) - 3) + + +def _skew(w): + w = np.asarray(w, float) + s = w.std() + return 0.0 if s < 1e-9 else float(np.mean(((w - w.mean()) / s) ** 3)) + + +def _slope(w): + w = np.asarray(w, float) + t = np.arange(len(w)) - (len(w) - 1) / 2.0 + d = (t**2).sum() or 1.0 + return float((t * w).sum() / d) + + +def _autocorr(w, lag): + w = np.asarray(w, float) - np.mean(w) + v = (w**2).sum() + return 0.0 if v < 1e-9 or len(w) <= lag else float((w[:-lag] * w[lag:]).sum() / v) + + +def _spectral_centroid(w): + sp = np.abs(np.fft.rfft(np.asarray(w, float) - np.mean(w))) + f = np.arange(len(sp)) + return 0.0 if sp.sum() < 1e-9 else float((f * sp).sum() / sp.sum()) + + +def _dominant_freq_power(w): + sp = np.abs(np.fft.rfft(np.asarray(w, float) - np.mean(w))) + return 0.0 if len(sp) <= 1 else float(sp[1:].max()) + + +# ---- extended helpers for the full FLOps-style library (numpy only, robust) ---- +def _safe(x): + x = float(x) + return x if np.isfinite(x) else 0.0 + + +def _crossings(w, level): + w = np.asarray(w, float) - level + return int(np.sum(w[:-1] * w[1:] < 0)) if len(w) > 1 else 0 + + +def _longest_strike(mask): + best = cur = 0 + for v in mask: + cur = cur + 1 if v else 0 + best = max(best, cur) + return int(best) + + +def _rms(w): + w = np.asarray(w, float) + return float(np.sqrt(np.mean(w**2))) if len(w) else 0.0 + + +def _binned_entropy(w, bins=10): + w = np.asarray(w, float) + if len(w) < 2 or w.max() - w.min() < 1e-12: + return 0.0 + hist, _ = np.histogram(w, bins=bins) + p = hist[hist > 0] / hist.sum() + return float(-(p * np.log(p)).sum()) + + +def _spectrum(w): + w = np.asarray(w, float) - np.mean(w) + ps = np.abs(np.fft.rfft(w)) ** 2 + return ps + + +def _spectral_entropy(w): + ps = _spectrum(w) + if ps.sum() < 1e-12: + return 0.0 + p = ps / ps.sum() + p = p[p > 0] + return float(-(p * np.log(p)).sum()) + + +def _spectral_rolloff(w, frac=0.85): + ps = _spectrum(w) + tot = ps.sum() + if tot < 1e-12: + return 0.0 + c = np.cumsum(ps) + return float(np.searchsorted(c, frac * tot) / max(len(ps) - 1, 1)) + + +def _spectral_flatness(w): + ps = _spectrum(w) + 1e-12 + return float(np.exp(np.mean(np.log(ps))) / np.mean(ps)) + + +def _band_energy(w, lo, hi): + ps = _spectrum(w) + n = len(ps) + a, b = int(lo * n), max(int(hi * n), int(lo * n) + 1) + tot = ps.sum() + return float(ps[a:b].sum() / tot) if tot > 1e-12 else 0.0 + + +def _hjorth(w): + w = np.asarray(w, float) + d1 = np.diff(w) + d2 = np.diff(d1) + v0, v1, v2 = ( + np.var(w), + np.var(d1) if len(d1) else 0.0, + np.var(d2) if len(d2) else 0.0, + ) + mob = np.sqrt(v1 / v0) if v0 > 1e-12 else 0.0 + comp = (np.sqrt(v2 / v1) / mob) if (v1 > 1e-12 and mob > 1e-12) else 0.0 + return float(mob), float(comp) + + +def _cid_ce(w): + w = np.asarray(w, float) + s = w.std() + if s < 1e-12: + return 0.0 + z = (w - w.mean()) / s + return float(np.sqrt(np.sum(np.diff(z) ** 2))) + + +def _c3(w, lag=1): + w = np.asarray(w, float) + n = len(w) + if n <= 2 * lag: + return 0.0 + return float(np.mean(w[: n - 2 * lag] * w[lag : n - lag] * w[2 * lag :])) + + +def _tra(w, lag=1): # time-reversal asymmetry + w = np.asarray(w, float) + n = len(w) + if n <= 2 * lag: + return 0.0 + a, b, c = w[2 * lag :], w[lag : n - lag], w[: n - 2 * lag] + return float(np.mean(a * a * b - b * c * c)) + + +def _perm_entropy(w, order=3): + w = np.asarray(w, float) + n = len(w) + if n < order + 1: + return 0.0 + from itertools import permutations + + perms = list(permutations(range(order))) + counts = {p: 0 for p in perms} + for i in range(n - order + 1): + counts[tuple(np.argsort(w[i : i + order]))] += 1 + c = np.array([v for v in counts.values() if v > 0], float) + p = c / c.sum() + return float(-(p * np.log(p)).sum() / np.log(len(perms))) + + +def _linregress_r(w): + w = np.asarray(w, float) + t = np.arange(len(w), dtype=float) + if w.std() < 1e-12 or t.std() < 1e-12: + return 0.0 + return float(np.corrcoef(t, w)[0, 1]) + + +def _half_mean_diff(w): + w = np.asarray(w, float) + h = len(w) // 2 + return float(np.mean(w[h:]) - np.mean(w[:h])) if h else 0.0 + + +def _quantile(w, q): + return float(np.percentile(np.asarray(w, float), q)) + + +# distribution / profiling +EXTRACTORS.update( + { + "var": lambda w: _safe(np.var(w)), + "median": lambda w: _safe(np.median(w)), + "q05": lambda w: _quantile(w, 5), + "q10": lambda w: _quantile(w, 10), + "q90": lambda w: _quantile(w, 90), + "q95": lambda w: _quantile(w, 95), + "iqr": lambda w: _quantile(w, 75) - _quantile(w, 25), + "mad": lambda w: _safe(np.median(np.abs(np.asarray(w, float) - np.median(w)))), + "mean_abs": lambda w: _safe(np.mean(np.abs(w))), + "sum_abs": lambda w: _safe(np.sum(np.abs(w))), + "rms": lambda w: _rms(w), + "abs_energy": lambda w: _safe(np.sum(np.asarray(w, float) ** 2)), + "cv": lambda w: _safe(np.std(w) / (abs(np.mean(w)) + 1e-12)), + "std_to_mean": lambda w: _safe(np.std(w) / (np.mean(w) + 1e-12)), + "count_above_mean": lambda w: float(np.sum(np.asarray(w, float) > np.mean(w))), + "count_below_mean": lambda w: float(np.sum(np.asarray(w, float) < np.mean(w))), + "ratio_above_mean": lambda w: _safe(np.mean(np.asarray(w, float) > np.mean(w))), + "count_above_2std": lambda w: float( + np.sum(np.abs(np.asarray(w, float) - np.mean(w)) > 2 * np.std(w)) + ), + "abs_max": lambda w: _safe(np.max(np.abs(w))), + "abs_min": lambda w: _safe(np.min(np.abs(w))), + } +) + +# temporal / order-dependent +EXTRACTORS.update( + { + "intercept": lambda w: _safe(np.mean(w)), + "autocorr2": lambda w: _autocorr(w, 2), + "autocorr3": lambda w: _autocorr(w, 3), + "autocorr5": lambda w: _autocorr(w, 5), + "autocorr10": lambda w: _autocorr(w, 10), + "mean_diff": lambda w: _safe(np.mean(np.diff(w))) if len(w) > 1 else 0.0, + "mean_2nd_diff": lambda w: _safe(np.mean(np.diff(w, 2))) if len(w) > 2 else 0.0, + "abs_2nd_diff_mean": lambda w: ( + _safe(np.mean(np.abs(np.diff(w, 2)))) if len(w) > 2 else 0.0 + ), + "std_diff": lambda w: _safe(np.std(np.diff(w))) if len(w) > 1 else 0.0, + "num_zero_crossings": lambda w: float(_crossings(w, 0.0)), + "num_mean_crossings": lambda w: float(_crossings(w, float(np.mean(w)))), + "longest_above_mean": lambda w: float( + _longest_strike(np.asarray(w, float) > np.mean(w)) + ), + "longest_below_mean": lambda w: float( + _longest_strike(np.asarray(w, float) < np.mean(w)) + ), + "first_loc_max": lambda w: _safe(int(np.argmax(w)) / len(w)), + "last_loc_max": lambda w: _safe( + (len(w) - int(np.argmax(w[::-1])) - 1) / len(w) + ), + "first_loc_min": lambda w: _safe(int(np.argmin(w)) / len(w)), + "mean_change": lambda w: _safe((w[-1] - w[0]) / len(w)) if len(w) > 1 else 0.0, + "cid_ce": lambda w: _cid_ce(w), + "c3_lag1": lambda w: _c3(w, 1), + "c3_lag2": lambda w: _c3(w, 2), + "time_reversal_asym": lambda w: _tra(w, 1), + "distinct_ratio": lambda w: _safe( + len(np.unique(np.round(np.asarray(w, float), 6))) / len(w) + ), + } +) + +# complexity / entropy +EXTRACTORS.update( + { + "binned_entropy": lambda w: _binned_entropy(w, 10), + "perm_entropy": lambda w: _perm_entropy(w, 3), + "spectral_entropy": lambda w: _spectral_entropy(w), + "hjorth_mobility": lambda w: _hjorth(w)[0], + "hjorth_complexity": lambda w: _hjorth(w)[1], + } +) + +# frequency +EXTRACTORS.update( + { + "spectral_rolloff": lambda w: _spectral_rolloff(w), + "spectral_flatness": lambda w: _spectral_flatness(w), + "dc_power": lambda w: _safe(abs(np.mean(w))), + "total_spectral_energy": lambda w: _safe(_spectrum(w).sum()), + "band_low": lambda w: _band_energy(w, 0.0, 0.33), + "band_mid": lambda w: _band_energy(w, 0.33, 0.66), + "band_high": lambda w: _band_energy(w, 0.66, 1.0), + "dominant_freq": lambda w: _safe( + int(np.argmax(_spectrum(w)[1:]) + 1) if len(_spectrum(w)) > 1 else 0 + ), + } +) + +# shape / vibration diagnostics +EXTRACTORS.update( + { + "peak_to_peak": lambda w: _safe(np.max(w) - np.min(w)), + "crest_factor": lambda w: _safe(np.max(np.abs(w)) / (_rms(w) + 1e-12)), + "shape_factor": lambda w: _safe(_rms(w) / (np.mean(np.abs(w)) + 1e-12)), + "impulse_factor": lambda w: _safe( + np.max(np.abs(w)) / (np.mean(np.abs(w)) + 1e-12) + ), + "clearance_factor": lambda w: _safe( + np.max(np.abs(w)) / ((np.mean(np.sqrt(np.abs(w)))) ** 2 + 1e-12) + ), + "margin_factor": lambda w: _safe((np.max(w) - np.min(w)) / (_rms(w) + 1e-12)), + "form_factor": lambda w: _safe(_rms(w) / (abs(np.mean(w)) + 1e-12)), + } +) + +# trend / stationarity +EXTRACTORS.update( + { + "linear_trend_r": lambda w: _linregress_r(w), + "half_mean_diff": lambda w: _half_mean_diff(w), + "half_std_ratio": lambda w: _safe( + np.std(np.asarray(w, float)[len(w) // 2 :]) + / (np.std(np.asarray(w, float)[: len(w) // 2]) + 1e-12) + ), + "energy_ratio_first_half": lambda w: _safe( + np.sum(np.asarray(w, float)[: len(w) // 2] ** 2) + / (np.sum(np.asarray(w, float) ** 2) + 1e-12) + ), + "trend_strength": lambda w: _safe(abs(_slope(w)) * len(w) / (np.std(w) + 1e-9)), + "cumsum_argmax_ratio": lambda w: _safe( + int(np.argmax(np.cumsum(np.asarray(w, float) - np.mean(w)))) / len(w) + ), + "cumsum_max": lambda w: _safe( + np.max(np.cumsum(np.asarray(w, float) - np.mean(w))) + ), + } +) + +# additional profiling / temporal / frequency / shape to complete the FLOps-style library +EXTRACTORS.update( + { + "q33": lambda w: _quantile(w, 33), + "q66": lambda w: _quantile(w, 66), + "range_to_std": lambda w: _safe((np.max(w) - np.min(w)) / (np.std(w) + 1e-12)), + "mean_square": lambda w: _safe(np.mean(np.asarray(w, float) ** 2)), + "abs_sum_changes": lambda w: ( + _safe(np.sum(np.abs(np.diff(w)))) if len(w) > 1 else 0.0 + ), + "max_diff": lambda w: _safe(np.max(np.diff(w))) if len(w) > 1 else 0.0, + "min_diff": lambda w: _safe(np.min(np.diff(w))) if len(w) > 1 else 0.0, + "var_diff": lambda w: _safe(np.var(np.diff(w))) if len(w) > 1 else 0.0, + "positive_diff_ratio": lambda w: ( + _safe(np.mean(np.diff(w) > 0)) if len(w) > 1 else 0.0 + ), + "autocorr4": lambda w: _autocorr(w, 4), + "autocorr7": lambda w: _autocorr(w, 7), + "num_peaks": lambda w: ( + float( + np.sum( + (np.asarray(w, float)[1:-1] > np.asarray(w, float)[:-2]) + & (np.asarray(w, float)[1:-1] > np.asarray(w, float)[2:]) + ) + ) + if len(w) > 2 + else 0.0 + ), + "num_valleys": lambda w: ( + float( + np.sum( + (np.asarray(w, float)[1:-1] < np.asarray(w, float)[:-2]) + & (np.asarray(w, float)[1:-1] < np.asarray(w, float)[2:]) + ) + ) + if len(w) > 2 + else 0.0 + ), + "longest_above_2std": lambda w: float( + _longest_strike(np.abs(np.asarray(w, float) - np.mean(w)) > 2 * np.std(w)) + ), + "zero_crossing_rate": lambda w: _safe( + _crossings(w, float(np.mean(w))) / len(w) + ), + "skew_abs": lambda w: _safe(abs(_skew(w))), + "p2p_to_std": lambda w: _safe((np.max(w) - np.min(w)) / (np.std(w) + 1e-12)), + "band_q1": lambda w: _band_energy(w, 0.0, 0.25), + "band_q2": lambda w: _band_energy(w, 0.25, 0.5), + "band_q3": lambda w: _band_energy(w, 0.5, 0.75), + "band_q4": lambda w: _band_energy(w, 0.75, 1.0), + "mean_psd": lambda w: _safe(np.mean(_spectrum(w))), + "peak_psd_ratio": lambda w: ( + _safe(_spectrum(w)[1:].max() / (_spectrum(w)[1:].sum() + 1e-12)) + if len(_spectrum(w)) > 1 + else 0.0 + ), + "diff_entropy": lambda w: ( + _binned_entropy(np.diff(w), 10) if len(w) > 1 else 0.0 + ), + "quarter_mean_diff": lambda w: ( + _safe( + np.mean(np.asarray(w, float)[3 * (len(w) // 4) :]) + - np.mean(np.asarray(w, float)[: len(w) // 4]) + ) + if len(w) >= 4 + else 0.0 + ), + } +) + +# flatline / cessation family — a machine going quiet shows up as a long constant run +EXTRACTORS.update( + { + "longest_constant_run": lambda w: ( + float(_longest_strike(np.abs(np.diff(np.asarray(w, float))) <= 1e-9)) + if len(w) > 1 + else float(len(w)) + ), + "flatline_fraction": lambda w: ( + _safe( + _longest_strike(np.abs(np.diff(np.asarray(w, float))) <= 1e-9) / len(w) + ) + if len(w) > 1 + else 0.0 + ), + } +) + +def _tabulate(series: np.ndarray, lw: int): + """Slide a window; X = windows[:-1], y = next value (forecasting target).""" + x = np.asarray(series, float).ravel() + wins = np.stack([x[i : i + lw] for i in range(len(x) - lw)]) # (N, lw) + y = x[lw:] # next value + return wins, y + + +def _corr(a, b): + a = np.asarray(a, float) + b = np.asarray(b, float) + if a.std() < 1e-9 or b.std() < 1e-9: + return 0.0 + return abs(float(np.corrcoef(a, b)[0, 1])) + + +# --------------------------------------------------------------------------- # +# Multi-config scorers (FLOps: score under several criteria, aggregate by mean rank). +# Each returns a per-feature score vector aligned to `names`; higher = more relevant. +# --------------------------------------------------------------------------- # +def _norm(v): + v = np.asarray(v, float) + v = np.nan_to_num(v, nan=0.0, posinf=0.0, neginf=0.0) + m = v.max() + return (v / m) if m > 1e-12 else v + + +def _score_corr(F, y, names): + return _norm([_corr(F[:, j], y) for j in range(F.shape[1])]) + + +def _score_ftest(F, y, names): + try: + from sklearn.feature_selection import f_regression + + f, _ = f_regression(F, y) + return _norm(np.nan_to_num(f)) + except Exception: + return _score_corr(F, y, names) + + +def _score_mutual_info(F, y, names): + try: + from sklearn.feature_selection import mutual_info_regression + + return _norm(mutual_info_regression(F, y, random_state=0)) + except Exception: + return _score_corr(F, y, names) + + +def _score_model(F, y, names): + """Multivariate model importance — captures interactions the univariate scorers miss.""" + try: + from sklearn.ensemble import RandomForestRegressor + + rf = RandomForestRegressor(n_estimators=80, random_state=0, n_jobs=1) + rf.fit(F, y) + return _norm(rf.feature_importances_) + except Exception: + return _score_corr(F, y, names) + + +_SCORERS = { + "corr": _score_corr, + "f_test": _score_ftest, + "mutual_info": _score_mutual_info, + "model": _score_model, +} + + +def _feature_matrix(wins, ex): + names = list(ex) + F = np.column_stack([[ex[n](w) for w in wins] for n in names]).astype(float) + F = np.nan_to_num(F, nan=0.0, posinf=0.0, neginf=0.0) + return F, names + + +def select_features( + series: np.ndarray, + *, + reference_feature: str = "mean", + lookback: Optional[int] = None, + cd_margin: float = 0.05, + extractors: Optional[Dict[str, Callable]] = None, + scorers: Optional[List[str]] = None, +) -> dict: + """FLOps selection (multi-config): score each extractor on this series under several + criteria (|corr|, F-test, mutual-info, model-importance), aggregate by MEAN RANK, rank, + and keep those that beat the Reference Feature by `cd_margin` (Critical-Difference proxy). + + Aggregating across heterogeneous scorers is the FLOps robustness idea: a feature that ranks + well under correlation, an F-test, mutual information AND a fitted model is trustworthy; + one that only spikes under a single criterion is not. `scorers` defaults to all four; + pass `["corr"]` for the fast univariate path. + + Returns {lookback, reference, scorers, scores{name:agg}, per_scorer{scorer:{name:score}}, + ranking[(name,agg)], selected[names], cd_margin}. + """ + ex = extractors or EXTRACTORS + lw = lookback or discover_lookback(series) + wins, y = _tabulate(series, lw) + F, names = _feature_matrix(wins, ex) + use = scorers or ["corr", "f_test", "mutual_info", "model"] + use = [s for s in use if s in _SCORERS] or ["corr"] + + # per-scorer normalized scores, then mean-rank aggregation across scorers + per_scorer = {s: dict(zip(names, _SCORERS[s](F, y, names))) for s in use} + ranks = np.zeros(len(names)) + for s in use: + sv = np.array([per_scorer[s][n] for n in names]) + order = (-sv).argsort() # best→worst + rk = np.empty(len(names)) + rk[order] = np.arange(1, len(names) + 1) + ranks += rk + mean_rank = ranks / len(use) + agg = 1.0 - (mean_rank - 1) / max(len(names) - 1, 1) # best=1.0, worst→0 + scores = {n: float(agg[i]) for i, n in enumerate(names)} + + ref_score = scores.get(reference_feature, 0.0) + ranking = sorted(scores.items(), key=lambda kv: kv[1], reverse=True) + selected = [name for name, sc in ranking if sc >= ref_score + cd_margin] + return { + "lookback": lw, + "reference": reference_feature, + "reference_score": round(ref_score, 4), + "scorers": use, + "scores": {k: round(v, 4) for k, v in scores.items()}, + "per_scorer": { + s: {k: round(v, 4) for k, v in d.items()} for s, d in per_scorer.items() + }, + "ranking": [(k, round(v, 4)) for k, v in ranking], + "selected": selected, + "cd_margin": cd_margin, + } diff --git a/src/servers/tsfm/reasoning/param_space.py b/src/servers/tsfm/reasoning/param_space.py new file mode 100644 index 000000000..9b3f600d7 --- /dev/null +++ b/src/servers/tsfm/reasoning/param_space.py @@ -0,0 +1,316 @@ +"""param_space.py — per-model parameter schema + reasoning + validation. + +Every model has its own parameters whose VALUES must be reasoned (context_length, sp, +n_neighbors, n_clusters, strategy, …). A card therefore exposes a parameter schema: + - auto-introspected from the sktime class (names, defaults, types, example sets), plus + - curated `param_hints` per parameter: a one-line description, what data evidence it + `depends_on`, a `suggest` rule, and an allowed `range`/`choices`. + +The agent reads the schema + `profile_series` evidence, REASONS a value for each parameter, +fills the recipe's `params`, and the server VALIDATES them (and the scorer grades them). This +is the "tools are complex" principle applied to the full per-model parameter space. +""" + +from __future__ import annotations + +import importlib +import inspect +from typing import Any, Dict, Optional + + +def introspect(sktime_class: str) -> dict: + """Constructor params (name → default/required/type) + sktime example param sets.""" + module, cls = sktime_class.rsplit(".", 1) + Est = getattr(importlib.import_module(module), cls) + sig = inspect.signature(Est.__init__) + params: Dict[str, dict] = {} + for name, p in sig.parameters.items(): + if name in ("self", "args", "kwargs"): + continue + params[name] = { + "default": (None if p.default is inspect._empty else _jsonable(p.default)), + "required": p.default is inspect._empty, + "type": ( + None if p.annotation is inspect._empty else _typename(p.annotation) + ), + } + examples = None + try: + examples = Est.get_test_params() # sktime: valid example configs + except Exception: + pass + return {"sktime_class": sktime_class, "params": params, "examples": examples} + + +def _typename(a): + return getattr(a, "__name__", str(a)) + + +def _jsonable(v): + return ( + v if isinstance(v, (int, float, str, bool, type(None), list, dict)) else str(v) + ) + + +# curated reasoning hints for common TS params — what evidence drives the value +DEFAULT_HINTS = { + "context_length": { + "description": "input/look-back window the model sees", + "depends_on": "dominant_period (seasonality)", + "suggest": ">= 2x dominant_period; must cover the seasonal cycle", + "range": [8, 2048], + }, + "prediction_length": { + "description": "forecast horizon", + "depends_on": "the request", + "suggest": "match the horizon named in the task", + "range": [1, 1024], + }, + "sp": { + "description": "seasonal periodicity", + "depends_on": "dominant_period", + "suggest": "= dominant_period from profile_series", + "range": [1, 1024], + }, + "window_length": { + "description": "rolling/reduction window", + "depends_on": "dominant_period", + "suggest": "~1-2x dominant_period", + "range": [2, 1024], + }, + "strategy": { + "description": "naive strategy", + "choices": ["last", "mean", "drift"], + "suggest": "drift if trending, last if persistent, mean if stationary-noisy", + }, + "n_neighbors": { + "description": "LOF neighborhood size", + "depends_on": "series length / window", + "suggest": "~sqrt(window_size); 10-50 typical", + "range": [2, 200], + }, + "window_size": { + "description": "subsequence length for windowed AD", + "depends_on": "dominant_period", + "suggest": "~1x dominant_period", + "range": [4, 1024], + }, + "n_clusters": { + "description": "number of clusters", + "depends_on": "elbow/silhouette over the set", + "suggest": "choose by silhouette; start 2-8", + "range": [2, 50], + }, + "coverage": { + "description": "conformal interval coverage", + "suggest": "0.9 default; 0.8/0.95 alt", + "range": [0.5, 0.99], + }, +} + + +# ───────────────────────────────────────────────────────────────────────────── +# Recipe-block hints — for the run-time algorithm choices that aren't constructor +# params of a single estimator: finetune (training_config) and anomaly (conformal AD). +# These mirror the legacy TTM `_ttm_main_config` and the conformal-AD wrapper knobs, made +# explicit + agent-reasoned. The recipe carries only overrides; defaults fill the rest. +# ───────────────────────────────────────────────────────────────────────────── +FINETUNE_HINTS = { + "n_finetune": { + "description": "few-shot train size: fraction (<=1) or count (>1)", + "depends_on": "available history", + "suggest": "0.05 (5%) for few-shot", + "range": [0.0, 1e6], + }, + "n_calibration": { + "description": "calibration split: fraction/count", + "suggest": "0 unless conformal", + "range": [0.0, 1e6], + }, + "n_test": { + "description": "held-out test split: fraction/count", + "suggest": "0.05", + "range": [0.0, 1e6], + }, + "lr": { + "description": "learning rate", + "suggest": "0 = auto (lr_finder); else 1e-4..1e-2", + "range": [0.0, 1.0], + }, + "epochs": { + "description": "fine-tune epochs", + "suggest": "few-shot: 1-10", + "range": [1, 200], + }, + "batch_size": { + "description": "mini-batch size", + "suggest": "32; lower if OOM", + "range": [1, 1024], + }, + "patch_length": { + "description": "TTM patch length", + "depends_on": "context_length", + "range": [1, 512], + }, + "head_dropout": { + "description": "forecast-head dropout", + "suggest": "0.7 default regularization", + "range": [0.0, 0.9], + }, + "backbone_frozen": { + "description": "freeze backbone (linear-probe)", + "choices": [True, False], + "suggest": "freeze for very small data", + }, + "decoder_mode": { + "description": "TTM decoder mixing", + "choices": ["mix_channel", "common_channel"], + }, + "scaling": { + "description": "per-channel scaling", + "choices": ["", "standard"], + "suggest": "standard normalizes inputs", + }, + "p_validation": { + "description": "validation fraction of the few-shot set", + "range": [0.0, 0.5], + "suggest": "0.1", + }, + "es_patience": { + "description": "early-stopping patience (epochs)", + "range": [1, 100], + }, + "es_th": {"description": "early-stopping min-delta", "range": [0.0, 1.0]}, + "epochs_warmup": {"description": "scheduler warmup epochs", "range": [0, 50]}, + "scheduler": {"description": "LR scheduler", "choices": ["OneCycleLR", "cosine"]}, + "optim": {"description": "optimizer", "choices": ["AdamW"]}, + "num_workers": {"description": "dataloader workers", "range": [0, 32]}, + "seed": {"description": "random seed", "range": [0, 2_147_483_647]}, +} + +ANOMALY_HINTS = { + "ad_model_type": { + "description": "conformal AD variant", + "choices": ["timeseries_conformal", "timeseries_conformal_adaptive"], + "suggest": "adaptive for non-stationary signals", + }, + "false_alarm": { + "description": "target false-alarm rate (= 1 - coverage)", + "suggest": "0.05 → 95% coverage", + "range": [0.001, 0.5], + }, + "n_calibration": { + "description": "fraction of data for conformal calibration", + "suggest": "0.2", + "range": [0.0, 1.0], + }, + "threshold_function": { + "description": "threshold rule", + "choices": ["weighting", "static"], + "suggest": "weighting for the adaptive variant", + }, + "window_size": { + "description": "calibration window length", + "depends_on": "dominant_period", + "suggest": "≈ recent-regime length; null = all calibration", + "range": [4, 100000], + }, + "nonconformity_score": { + "description": "score function", + "choices": ["absolute_error"], + }, + "decay_param": { + "description": "exponential weighting decay (adaptive)", + "range": [0.0, 1.0], + }, + "task": { + "description": "fit a new AD model or run inference", + "choices": ["fit", "inference"], + }, + "impute": { + "description": "how to handle missing values before a classical (non-foundation) " + "detector; omit to let the model raise on NaN (foundation detectors " + "such as tspulse accept NaN natively)", + "choices": ["interpolate", "drop", "zero"], + "suggest": "interpolate for sensor gaps; drop to avoid fabricating values; " + "zero only when 0 is meaningful", + }, +} + +BLOCK_HINTS = {"finetune": FINETUNE_HINTS, "anomaly": ANOMALY_HINTS} + + +def block_schema(block: str) -> dict: + """The reasoning hints for a recipe block (finetune / anomaly) — what the agent fills.""" + return {"block": block, "params": BLOCK_HINTS.get(block, {})} + + +def validate_block(block: str, params: Optional[dict]) -> dict: + """Validate a recipe-block param dict (finetune/anomaly) against its hints; produce an audit. + Free-form dict (no sktime constructor): the hint set defines the known params.""" + hints = BLOCK_HINTS.get(block, {}) + known = set(hints) + issues, audit = [], {} + for k, v in (params or {}).items(): + if k not in known: + issues.append(f"unknown {block} param '{k}' (valid: {sorted(known)[:8]}…)") + continue + h = hints[k] + if "choices" in h and v not in h["choices"]: + issues.append(f"{block}.{k}={v!r} not in choices {h['choices']}") + if "range" in h and isinstance(v, (int, float)) and not isinstance(v, bool): + lo, hi = h["range"] + audit[k] = { + "value": v, + "in_range": bool(lo <= v <= hi), + "range": h["range"], + } + if not (lo <= v <= hi): + issues.append(f"{block}.{k}={v} out of range {h['range']}") + return { + "ok": not issues, + "issues": issues, + "param_audit": audit, + "known_params": sorted(known), + } + + +def param_schema(card: dict) -> dict: + """Schema the agent reasons over: introspected params + merged hints (card hints override).""" + info = introspect(card["sktime_class"]) + hints = {**DEFAULT_HINTS, **(card.get("param_hints") or {})} + for name, meta in info["params"].items(): + if name in hints: + meta["hint"] = hints[name] + info["model_id"] = card.get("model_id") + return info + + +def validate_params(card: dict, params: Optional[dict]) -> dict: + """Check agent-chosen params against the schema before resolve; produce a param_audit.""" + info = introspect(card["sktime_class"]) + known = set(info["params"]) + hints = {**DEFAULT_HINTS, **(card.get("param_hints") or {})} + issues, audit = [], {} + for k, v in (params or {}).items(): + if k not in known: + issues.append(f"unknown param '{k}' (valid: {sorted(known)[:8]}…)") + continue + h = hints.get(k, {}) + if "choices" in h and v not in h["choices"]: + issues.append(f"{k}={v!r} not in choices {h['choices']}") + if "range" in h and isinstance(v, (int, float)): + lo, hi = h["range"] + audit[k] = { + "value": v, + "in_range": bool(lo <= v <= hi), + "range": h["range"], + } + if not (lo <= v <= hi): + issues.append(f"{k}={v} out of range {h['range']}") + return { + "ok": not issues, + "issues": issues, + "param_audit": audit, + "known_params": sorted(known), + } diff --git a/src/servers/tsfm/reasoning/patterns.py b/src/servers/tsfm/reasoning/patterns.py new file mode 100644 index 000000000..2e94ccd51 --- /dev/null +++ b/src/servers/tsfm/reasoning/patterns.py @@ -0,0 +1,497 @@ +"""patterns.py — the pattern-evidence engine (P1: univariate state + rate, grouped). + +Given a (standardized) multivariate series, DESCRIBE its shape as structured evidence the LLM can +reason over — never name a fault. This is the server's half of the SenTSR-style split: the server +says "vibration shows a sharp rise; temperature stable"; the LLM says "alignment drift". + +Design notes +------------ +* Reference-free. SenTSR standardizes each channel by median/MAD, so the baseline is the series + itself. We robust-standardize internally too (median/MAD) so the same logic works on any input + and reads in robust-z units — no external "normal" window is needed. +* Channel grouping. The benchmark reasons about "vibration" (= Acceleration + Velocity) vs + "temperature", so we aggregate member channels into a group before describing (configurable). +* P1 is single-phase (whole-series) state labeling. Phases (changepoints) + bivariate relations + come in P2/P3; the output dict already nests under a single phase so it extends cleanly. + +States: STABLE · RISE · DECLINE · SPIKE · LEVEL_SHIFT · CESSATION · OSCILLATION +Each carries a rate (gradual|sharp where meaningful) and SPIKE carries persistence +(transient|sustained). All purely descriptive. +""" + +from __future__ import annotations + +from typing import Dict, List, Optional + +import numpy as np + +# --- thresholds (robust-z units); tunable ----------------------------------- +_FLAT_EPS = 1e-9 # |Δ| below this == constant +_CESSATION_FRAC = 0.25 # a quarter of the series held at one value +_TREND_R = 0.55 # |corr(value, time)| (oscillation/level-shift guards) +_TREND_HALF = 1.0 # |mean(2nd half) - mean(1st half)| in z (guards) +_TREND_STRENGTH = ( + 1.5 # Theil–Sen total change (slope×n) in robust-spread units ⇒ rise/decline +) +_SPIKE_Z = 5.0 # an extreme point this far out (robust-z) +_SHARP_STEP = 0.5 # biggest single step ≥ this × robust spread == "sharp" +_OSC_CROSS = 0.20 # mean-crossings per sample for oscillation +_FLAT_RANGE = 0.5 # below this total z-range == essentially flat +_LEVEL_HALF = ( + 1.0 # half-to-half jump for a level shift (paired with a dominant single step) +) + + +def _robust_z(x: np.ndarray) -> np.ndarray: + x = np.asarray(x, float).ravel() + med = np.median(x) + mad = np.median(np.abs(x - med)) + scale = 1.4826 * mad if mad > 1e-12 else (x.std() or 1.0) + return (x - med) / (scale + 1e-12) + + +def _longest_run(mask: np.ndarray) -> int: + best = cur = 0 + for v in mask: + cur = cur + 1 if v else 0 + best = max(best, cur) + return best + + +def _pearson_t(x: np.ndarray) -> float: + t = np.linspace(0.0, 1.0, len(x)) + if np.std(x) < 1e-12: + return 0.0 + return float(np.corrcoef(x, t)[0, 1]) + + +def _autocorr1(x: np.ndarray) -> float: + x = x - x.mean() + d = float(np.dot(x, x)) + return float(np.dot(x[:-1], x[1:]) / d) if d > 1e-12 else 0.0 + + +def _rolling_median(x: np.ndarray, w: int) -> np.ndarray: + """De-spike: rolling median removes impulsive outliers (duty-cycle dips, lone spikes) while + preserving trends, edges and smooth oscillations. The basis for low-frequency shape reading. + """ + if w <= 1 or len(x) < w: + return x + half = w // 2 + xp = np.pad(x, (half, half), mode="edge") + return np.array([np.median(xp[i : i + w]) for i in range(len(x))]) + + +def _theil_sen(y: np.ndarray) -> float: + """Theil–Sen slope: median of all pairwise slopes. Robust to outliers (duty-cycle dips), and + it does NOT fabricate a trend on a flat/noisy signal — the general, principled trend estimator. + """ + n = len(y) + idx = np.arange(n, dtype=float) + dy = np.subtract.outer(y, y) + dx = np.subtract.outer(idx, idx) + m = dx > 0 + return float(np.median(dy[m] / dx[m])) if m.any() else 0.0 + + +def classify_state(x: np.ndarray) -> dict: + """Label one 1-D series with a descriptive state + rate (+ persistence for spikes). + + Scale-robust without destroying scale: trend via correlation (scale-free), 'sharp vs gradual' + via change CONCENTRATION (one step's share of total variation), magnitudes via the series' + own robust spread (median/MAD). Assumes the input is already standardized-ish (as SenTSR is); + grouping pre-standardizes members so this holds. + """ + x = np.asarray(x, float).ravel() + n = len(x) + if n < 4: + return { + "state": "STABLE", + "rate": None, + "magnitude": 0.0, + "persistence": None, + "evidence": {"n": n}, + } + # RAW: high-frequency features (spike / cessation) read on the original signal + med = float(np.median(x)) + mad = float(np.median(np.abs(x - med))) + spread = 1.4826 * mad if mad > 1e-12 else (float(x.std()) or 1.0) + same = np.abs(np.diff(x)) <= _FLAT_EPS + flat_frac = _longest_run(same) / n + devs = np.abs(x - med) / spread + ext_idx = np.where(devs >= _SPIKE_Z)[0] + max_abs = float(devs.max()) + + # DE-SPIKED: low-frequency shape (trend / stable / oscillation) — removes impulsive duty-cycle + # dips & lone spikes so a rise buried under a recurring floor still reads as a rise. + xs = _rolling_median( + x, 5 + ) # removes ≤2-consecutive impulses; preserves trends & oscillations + med_s = float(np.median(xs)) + mad_s = float(np.median(np.abs(xs - med_s))) + spread_s = 1.4826 * mad_s if mad_s > 1e-12 else (float(xs.std()) or 1.0) + rng_norm = float((xs.max() - xs.min()) / spread_s) + r = _pearson_t(xs) + half = float((np.mean(xs[n // 2 :]) - np.mean(xs[: n // 2])) / spread_s) + sdiffs = np.abs(np.diff(xs)) + concentration = float(sdiffs.max() / (sdiffs.sum() + 1e-12)) if len(sdiffs) else 0.0 + step_norm = float(sdiffs.max() / spread_s) if len(sdiffs) else 0.0 + cross_rate = float(np.sum(np.diff(np.sign(xs - med_s)) != 0) / n) + ac1 = _autocorr1(xs) + diffs = sdiffs + # robust trend on the de-spiked signal (Theil–Sen): outlier-resistant, no fabricated trends + ts_slope = _theil_sen(xs) + ts_strength = float( + ts_slope * n / spread_s + ) # total rise/fall in robust-spread units + trend_conc = ( + float(diffs.max() / (diffs.sum() + 1e-12)) if len(diffs) else 0.0 + ) # rate: 1 jump? + + ev = { + "trend_r": round(r, 3), + "trend_strength": round(ts_strength, 2), + "half_diff": round(half, 3), + "range_norm": round(rng_norm, 2), + "concentration": round(concentration, 3), + "mean_cross_rate": round(cross_rate, 3), + "autocorr1": round(ac1, 3), + "flatline_fraction": round(flat_frac, 3), + "max_abs_dev": round(max_abs, 2), + "n_extreme": int(len(ext_idx)), + } + + def out(state, rate=None, persistence=None, onset=None): + ev2 = dict(ev) + if onset is not None: + ev2["onset_frac"] = round(float(onset), 3) + return { + "state": state, + "rate": rate, + "magnitude": round(max(abs(half), max_abs), 2), + "persistence": persistence, + "evidence": ev2, + } + + # truly constant → stable + if rng_norm < 1e-6: + return out("STABLE") + + # CESSATION — a long constant run at/below baseline while the rest is active + if flat_frac >= _CESSATION_FRAC and rng_norm >= 2.0: + cur = best = best_end = 0 + for k, v in enumerate(same): + cur = cur + 1 if v else 0 + if cur > best: + best, best_end = cur, k + run_level = float(np.median(x[max(0, best_end - best) : best_end + 1])) + if run_level <= med: + return out("CESSATION", onset=(best_end - best) / n) + + # SPIKE — a few extreme points, not a sustained trend + if ( + 1 <= len(ext_idx) <= max(3, int(0.03 * n)) + and max_abs >= _SPIKE_Z + and abs(r) < _TREND_R + ): + first, last = int(ext_idx[0]), int(ext_idx[-1]) + before = float(np.mean(x[:first])) if first > 0 else med + after = float(np.mean(x[last + 1 :])) if last + 1 < n else med + persistence = "sustained" if (after - before) / spread > 2.0 else "transient" + return out("SPIKE", persistence=persistence, onset=first / n) + + # RISE / DECLINE — robust Theil–Sen trend over a meaningful magnitude + if abs(ts_strength) >= _TREND_STRENGTH: + state = "RISE" if ts_slope > 0 else "DECLINE" + rate = "sharp" if trend_conc >= 0.30 else "gradual" # one dominant jump ⇒ sharp + return out(state, rate=rate) + + # LEVEL_SHIFT — a step between two regimes: big half-to-half change dominated by one jump + if abs(half) >= _LEVEL_HALF and concentration >= 0.30: + return out("LEVEL_SHIFT", onset=int(np.argmax(diffs)) / n) + + # OSCILLATION — periodic swings (frequent crossings + smooth, i.e. autocorrelated), no trend + if cross_rate >= _OSC_CROSS and ac1 >= 0.5 and abs(r) < 0.4 and rng_norm >= 1.5: + return out("OSCILLATION") + + return out("STABLE") + + +# --- channel grouping (generic; domain grouping is opt-in) ------------------- +# The engine is channel-agnostic: any names, any count. By DEFAULT every channel is its own group +# (no domain assumptions). Grouping is optional and supplied by the caller/agent — either an +# explicit {group: [channels]} map, or by applying a name-rule preset via auto_group(). This keeps +# the SenTSR "vibration = accel+velocity" choice out of the engine and in the hands of whoever +# knows the domain. +GROUP_PRESETS: Dict[str, Dict[str, str]] = { # preset name → {name-substring: group} + "vibration_temperature": { + "accel": "vibration", + "veloc": "vibration", + "vibrat": "vibration", + "temp": "temperature", + }, +} + + +def identity_groups(channel_names: List[str]) -> Dict[str, List[str]]: + """Default, domain-free grouping: each channel is its own single-member group.""" + return {name: [name] for name in channel_names} + + +def auto_group(channel_names: List[str], rules) -> Dict[str, List[str]]: + """Optional name-based grouping. `rules` is a {substring: group} dict or the name of a preset in + GROUP_PRESETS. Channels matching no rule become their own group. Caller opts in explicitly. + """ + if isinstance(rules, str): + rules = GROUP_PRESETS[rules] + groups: Dict[str, List[str]] = {} + for name in channel_names: + low = name.lower() + g = next((grp for key, grp in rules.items() if key in low), name) + groups.setdefault(g, []).append(name) + return groups + + +def _to_channels(data, channel_names: Optional[List[str]]) -> Dict[str, np.ndarray]: + """Normalize input (DataFrame | dict | 2-D array) to {name: 1-D array}.""" + if hasattr(data, "columns"): # pandas DataFrame + return {str(c): np.asarray(data[c], float) for c in data.columns} + if isinstance(data, dict): + return {str(k): np.asarray(v, float).ravel() for k, v in data.items()} + arr = np.asarray(data, float) + if arr.ndim == 1: + arr = arr[None, :] + names = channel_names or [f"ch{i}" for i in range(arr.shape[0])] + return {names[i]: arr[i] for i in range(arr.shape[0])} + + +def group_signal(channels: Dict[str, np.ndarray], members: List[str]) -> np.ndarray: + """Aggregate member channels into one group signal: robust-standardize each, then average + (so a multi-axis 'vibration' group reads on a common scale).""" + stacks = [_robust_z(channels[m]) for m in members if m in channels] + return np.mean(np.column_stack(stacks), axis=1) if stacks else np.zeros(1) + + +# --- the descriptive phrase + series description ----------------------------- +_PHRASE = { # (state, rate?) → shape words — strictly descriptive, no fault terms + ("STABLE", None): "stable, near baseline", + ("RISE", "gradual"): "a gradual rise", + ("RISE", "sharp"): "a sharp rise", + ("RISE", None): "a rise", + ("DECLINE", "gradual"): "a gradual decline", + ("DECLINE", "sharp"): "a sharp decline", + ("DECLINE", None): "a decline", + ("LEVEL_SHIFT", None): "a level shift to a new band", + ("CESSATION", None): "goes quiet (flatline)", + ("OSCILLATION", None): "oscillation around baseline", +} + + +def _phrase(s: dict) -> str: + if s["state"] == "SPIKE": + return ( + "a spike followed by sustained elevation" + if s.get("persistence") == "sustained" + else "a transient spike" + ) + return _PHRASE.get((s["state"], s.get("rate"))) or _PHRASE.get( + (s["state"], None), s["state"].lower() + ) + + +# --- P2: bivariate relation between two group signals ------------------------- +_REL_CORR = 0.5 # |corr| for a real association +_REL_LAG_GAIN = 0.1 # lagged corr must beat lag-0 by this to call it lead-lag + + +def _corr_at(a: np.ndarray, b: np.ndarray, lag: int) -> float: + """corr(a, b) with b shifted: lag>0 means a leads b by `lag` steps.""" + if lag > 0: + x, y = a[:-lag], b[lag:] + elif lag < 0: + x, y = a[-lag:], b[:lag] + else: + x, y = a, b + if len(x) < 8 or np.std(x) < 1e-9 or np.std(y) < 1e-9: + return 0.0 + return float(np.corrcoef(x, y)[0, 1]) + + +def xcorr_lag(a: np.ndarray, b: np.ndarray, max_lag: Optional[int] = None): + """Best lead-lag: scan lags, return (lag, corr) maximizing |corr|. lag>0 ⇒ a leads b.""" + n = min(len(a), len(b)) + max_lag = max_lag if max_lag is not None else min(n // 4, 24) + best = (0, _corr_at(a, b, 0)) + for k in range(-max_lag, max_lag + 1): + c = _corr_at(a, b, k) + if abs(c) > abs(best[1]): + best = (k, c) + return best + + +def relate_pair( + sig_a, sig_b, state_a: str, state_b: str, max_lag: Optional[int] = None +) -> dict: + """Classify the temporal relation between two group signals (generic, any pair): + DECOUPLED (one moves, other flat) · CO_MOVE (move together) · LEAD_LAG (one precedes the + other) · INDEPENDENT (both move, uncorrelated) · NONE (both flat).""" + a = np.asarray(sig_a, float) + b = np.asarray(sig_b, float) + r0 = _corr_at(a, b, 0) + lag, clag = xcorr_lag(a, b, max_lag) + act_a, act_b = state_a != "STABLE", state_b != "STABLE" + base = {"r0": round(r0, 3), "lag": int(lag), "xcorr": round(clag, 3)} + if not act_a and not act_b: + return {"type": "NONE", **base} + if act_a != act_b: + return {"type": "DECOUPLED", **base} + if ( + abs(lag) >= 2 + and abs(clag) - abs(r0) >= _REL_LAG_GAIN + and abs(clag) >= _REL_CORR + ): + return {"type": "LEAD_LAG", "leader": "a" if lag > 0 else "b", **base} + if abs(r0) >= _REL_CORR: + return { + "type": "CO_MOVE", + "direction": "same" if r0 > 0 else "opposite", + **base, + } + return {"type": "INDEPENDENT", **base} + + +# --- P3: changepoint segmentation (conservative CUSUM) ------------------------ +def _cusum_cps(x: np.ndarray, min_seg: int, shift: float, depth: int = 0) -> List[int]: + """Recursive single-CUSUM changepoints: split where the mean shifts by >= `shift` robust + units and both sides are >= min_seg. Conservative — gradual drifts yield no split. + """ + n = len(x) + if n < 2 * min_seg or depth > 3: + return [] + c = np.cumsum(x - x.mean()) + cp = int(np.argmax(np.abs(c))) + if cp < min_seg or cp > n - min_seg: + return [] + med = np.median(x) + mad = np.median(np.abs(x - med)) + spread = 1.4826 * mad if mad > 1e-12 else (x.std() or 1.0) + if abs(np.mean(x[:cp]) - np.mean(x[cp:])) / spread < shift: + return [] + left = _cusum_cps(x[:cp], min_seg, shift, depth + 1) + right = [cp + i for i in _cusum_cps(x[cp:], min_seg, shift, depth + 1)] + return left + [cp] + right + + +def segment( + group_sigs: Dict[str, np.ndarray], + *, + min_seg: Optional[int] = None, + shift: float = 3.0, +) -> List[int]: + """Union of changepoints across all groups → phase boundaries (sorted, deduped, merged). + Conservative: de-spikes each signal first and requires a large mean shift, so only clear + regime changes split (gradual drift / noise stays one phase).""" + n = len(next(iter(group_sigs.values()))) if group_sigs else 0 + min_seg = min_seg if min_seg is not None else max(12, n // 8) + cps: set = set() + for sig in group_sigs.values(): + cps.update( + _cusum_cps(_rolling_median(np.asarray(sig, float), 5), min_seg, shift) + ) + pts = sorted(cps) + merged: List[int] = [] + for p in pts: # drop near-duplicate boundaries + if not merged or p - merged[-1] >= min_seg: + merged.append(p) + return [0] + merged + [n] + + +def describe_series( + data, + *, + channel_names: Optional[List[str]] = None, + groups: Optional[Dict[str, List[str]]] = None, + group_rules=None, + segment_phases: bool = True, +) -> dict: + """P1 evidence: label each channel-group's whole-series state+rate, render a shape-only NL + summary. Single-phase; the dict nests under one phase for forward-compat. + + Generic by default: with no `groups`/`group_rules`, every channel is its own group (works for + any signals, any count, any names). Pass `groups={group:[channels]}` for explicit grouping, or + `group_rules=` (e.g. "vibration_temperature") to auto-group by channel name. + """ + channels = _to_channels(data, channel_names) + if groups is None: + groups = ( + auto_group(list(channels), group_rules) + if group_rules + else identity_groups(list(channels)) + ) + sigs = {g: group_signal(channels, members) for g, members in groups.items()} + n = len(next(iter(channels.values()))) if channels else 0 + group_names = list(groups) + pairs = [ + (group_names[i], group_names[j]) + for i in range(len(group_names)) + for j in range(i + 1, len(group_names)) + ] + + bounds = segment(sigs) if (segment_phases and n) else [0, n] + phases = [] + for s, e in zip(bounds[:-1], bounds[1:]): + if e - s < 4: # skip degenerate slivers + continue + per_group = {g: classify_state(sig[s:e]) for g, sig in sigs.items()} + relations = [] + for a, b in pairs: + rel = relate_pair( + sigs[a][s:e], sigs[b][s:e], per_group[a]["state"], per_group[b]["state"] + ) + if rel["type"] != "NONE": + relations.append({"a": a, "b": b, **rel}) + phases.append( + {"span": [int(s), int(e)], "per_group": per_group, "relations": relations} + ) + + summary = _summarize(phases, pairs) + return { + "groups": {g: members for g, members in groups.items()}, + "phases": phases, + "n_observations": n, + "standardized": True, + "summary": summary, + } + + +_REL_PHRASE = { + "DECOUPLED": lambda r: f"{r['a']} and {r['b']} are decoupled", + "CO_MOVE": lambda r: f"{r['a']} and {r['b']} move together", + "LEAD_LAG": lambda r: ( + f"{r['a'] if r.get('leader') == 'a' else r['b']} leads " + f"{r['b'] if r.get('leader') == 'a' else r['a']} by {abs(r['lag'])} steps" + ), + "INDEPENDENT": lambda r: f"{r['a']} and {r['b']} both change, uncorrelated", +} + + +def _summarize(phases: List[dict], pairs) -> str: + """Shape-only NL rendering. Single phase → one clause; multi-phase → numbered phases.""" + + def phase_text(ph): + states = "; ".join(f"{g}: {_phrase(s)}" for g, s in ph["per_group"].items()) + rels = "; ".join( + _REL_PHRASE[r["type"]](r) + for r in ph.get("relations", []) + if r["type"] in _REL_PHRASE + ) + return states + (f" ({rels})" if rels else "") + + if not phases: + return "" + if len(phases) == 1: + return phase_text(phases[0]) + "." + return ( + " | ".join(f"Phase {i+1}: {phase_text(ph)}" for i, ph in enumerate(phases)) + + "." + ) diff --git a/src/servers/tsfm/reasoning/profile.py b/src/servers/tsfm/reasoning/profile.py new file mode 100644 index 000000000..f5e5a0d2e --- /dev/null +++ b/src/servers/tsfm/reasoning/profile.py @@ -0,0 +1,135 @@ +"""Evidence tools — give the AGENT the facts to reason from. No decisions here. + +These tools answer "what does the data look like?" and "what can the catalog offer?" — the +raw signals an agent needs to choose lookback / context / horizon / channels / pipeline / +thresholding itself. Deliberately NO recommended values: the reasoning is the agent's job +(the server must not pre-decide, or the benchmark stops testing the agent). +""" + +from __future__ import annotations + +from typing import List, Optional + +import numpy as np + +from ..io import window as io +from ..stores import model_store +from ..stores import feature_store + + +def profile_series(store, asset_id: str, channels: Optional[List[str]] = None) -> dict: + """Factual characterization of a store-backed asset's signal — evidence, not advice.""" + X, names = io.read_window(asset_id, store=store) + return _profile_array(np.asarray(X, float), names, ident=asset_id) + + +def profile_ref( + data_ref: str, + *, + timestamp_column: Optional[str] = None, + channels: Optional[List[str]] = None, +) -> dict: + """Profile a time series passed as a FILE POINTER (the IoT data model). Loads the ref into + an sktime container and returns the same evidence as profile_series.""" + from ..io import refs + + obj = refs.load_series(data_ref, time_col=timestamp_column, channels=channels) + import pandas as pd + + if isinstance(obj, pd.Series): + X, names = obj.to_numpy().reshape(-1, 1), [obj.name or "value"] + else: + X, names = obj.to_numpy(), list(obj.columns) + return _profile_array(np.asarray(X, float), names, ident=data_ref) + + +def _profile_array(X: np.ndarray, names: List[str], *, ident: str) -> dict: + """Core profiling on a loaded (n, c) array — shared by the store and file-pointer paths.""" + if X.ndim == 1: + X = X.reshape(-1, 1) + n, c = X.shape + + # seasonality (dominant spectral period) per channel — DETREND first so a strong + # linear trend doesn't masquerade as a giant low-frequency "period". + def _detrend(x): + t = np.arange(len(x)) + a, b = np.polyfit(t, x, 1) + return x - (a * t + b) + + periods = [] + for j in range(c): + x = _detrend(X[:, j]) + sp = np.abs(np.fft.rfft(x)) + if len(sp) > 2 and sp[1:].max() > 1e-9: + periods.append(int(round(n / (1 + int(np.argmax(sp[1:])))))) + dominant_period = int(np.median(periods)) if periods else None + seasonality_strength = None + if dominant_period: + sp = np.abs(np.fft.rfft(_detrend(X[:, 0]))) + seasonality_strength = round(float(sp[1:].max() / (sp[1:].sum() + 1e-9)), 3) + + # stationarity / trend on channel 0 + t = np.arange(n) - n / 2 + slope = float((t * (X[:, 0] - X[:, 0].mean())).sum() / ((t**2).sum() or 1.0)) + trend_strength = round(float(abs(slope) * n / (np.std(X[:, 0]) + 1e-9)), 3) + + # inter-channel correlation summary + corr = np.corrcoef(X.T) if c > 1 else np.array([[1.0]]) + off = corr[np.triu_indices(c, 1)] if c > 1 else np.array([]) + max_abs_corr = round(float(np.max(np.abs(off))), 3) if off.size else 0.0 + + # missingness / gaps (synthetic data has none, but report the check) + gaps = int(np.isnan(X).sum()) + + return { + "source": ident, + "n_observations": n, + "n_channels": c, + "channels": names, + "dominant_period": dominant_period, + "seasonality_strength": seasonality_strength, + "trend_slope": round(slope, 5), + "trend_strength": trend_strength, + "non_stationary": bool(trend_strength > 1.0), + "max_abs_channel_corr": max_abs_corr, + "n_missing": gaps, + "value_range": [round(float(X.min()), 3), round(float(X.max()), 3)], + } + + +def available_contexts(store, task_id: str = "tsfm_forecasting") -> dict: + """What the model store offers for this task — so the agent can match context to lookback.""" + ms = model_store.list_models(store, task_id=task_id) + return { + "task_id": task_id, + "models": [ + { + "model_id": m["model_id"], + "context_length": m.get("context_length"), + "prediction_length": m.get("prediction_length"), + "domain": m.get("domain"), + "framework": m.get("framework"), + "pipeline_type": m.get("pipeline_type"), + "usage_modes": m.get("usage_modes"), + } + for m in ms + ], + } + + +def available_features(store, category: Optional[str] = None) -> dict: + """Transforms + extractors the agent can choose to apply.""" + return { + "transforms": [ + { + "feature_id": f["feature_id"], + "scenario_categories": f.get("scenario_categories"), + "invertible": f.get("invertible"), + } + for f in feature_store.find_features(store, category=category) + ], + "extractors": [ + e["extractor_name"] + for e in feature_store.list_extractors(store, category=category) + ], + } diff --git a/src/servers/tsfm/stores/__init__.py b/src/servers/tsfm/stores/__init__.py new file mode 100644 index 000000000..e2d021535 --- /dev/null +++ b/src/servers/tsfm/stores/__init__.py @@ -0,0 +1 @@ +"""stores layer.""" diff --git a/src/servers/tsfm/stores/feature_store.py b/src/servers/tsfm/stores/feature_store.py new file mode 100644 index 000000000..bc98fff96 --- /dev/null +++ b/src/servers/tsfm/stores/feature_store.py @@ -0,0 +1,226 @@ +"""Feature store — transforms (EFE) + extractors (FLOps) on the core Store. + +Two entry kinds in one catalog: + - kind="transform": EFE-style fit/transform/inverse programs stored as code (validated + + EFE validity-gated on register). + - kind="extractor": FLOps scalar extractors (the 130+ library); the executable lives in + feature_selection.EXTRACTORS, the catalog indexes them so select_features can pick. + +Capabilities: + read : get / find_features / list_extractors / search / get_lineage + write : register_feature (validated+gated) / update / deprecate / new_version + learn : select_features (FLOps, full library) / select_features_from_catalog (writes importance back) +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Dict, List, Optional + +from ..core import schemas + +COLLECTION = "feature_catalog" + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _id(fid): + return f"feature:{fid}" + + +# --------------------------------------------------------------------------- # +# read +# --------------------------------------------------------------------------- # +def get_feature(store, feature_id: str) -> Optional[dict]: + return store.get(COLLECTION, _id(feature_id)) + + +def find_features( + store, + target_task: Optional[str] = None, + target_model: Optional[str] = None, + kind: str = "transform", + status: str = "active", +) -> List[dict]: + sel: Dict = {} + if status: + sel["status"] = status + if target_task: + sel["target_task"] = target_task + if target_model: + sel["target_model"] = target_model + docs = store.find(COLLECTION, sel) + if kind: + docs = [d for d in docs if d.get("kind", "transform") == kind] + return docs + + +def list_extractors(store, status: str = "active") -> List[dict]: + return find_features(store, kind="extractor", status=status) + + +def search( + store, text: str = "", *, tags: Optional[List[str]] = None, status: str = "active" +): + text = (text or "").lower() + out = [] + for f in store.find(COLLECTION, {"status": status} if status else {}): + hay = " ".join( + [ + f.get("feature_id") or "", + f.get("name") or "", + " ".join(f.get("tags") or []), + ] + ).lower() + if text and text not in hay: + continue + if tags and not set(tags) <= set(f.get("tags", [])): + continue + out.append(f) + return out + + +def get_lineage(store, feature_id: str) -> dict: + """Evolution chain: ancestors via parent_feature_id + descendants.""" + ancestors, cur, seen = [], get_feature(store, feature_id), set() + while cur and cur.get("parent_feature_id") and cur["parent_feature_id"] not in seen: + seen.add(cur["parent_feature_id"]) + p = get_feature(store, cur["parent_feature_id"]) + if not p: + break + ancestors.append(p["feature_id"]) + cur = p + descendants = [ + f["feature_id"] + for f in store.find(COLLECTION, {"parent_feature_id": feature_id}) + ] + return { + "feature_id": feature_id, + "ancestors": ancestors, + "root": ancestors[-1] if ancestors else feature_id, + "descendants": descendants, + } + + +# --------------------------------------------------------------------------- # +# write (transforms) +# --------------------------------------------------------------------------- # +def register_feature(store, feature: dict, *, overwrite: bool = False) -> dict: + """Schema-validate, then run the EFE validity gate (entry points / no-inplace / + invertibility) before accepting.""" + doc = schemas.validate_feature(feature) + from ..engine import feature_runner as fr + import numpy as np + + X = np.random.RandomState(0).normal(0, 1, size=(40, 3)) + chk = fr.validate_and_run( + doc, X_fit=X[:30], X_in=X, metadata={"window": 8, "channel_indices": [0, 1, 2]} + ) + doc.setdefault("validity", {}).update(chk["checks"]) + doc["kind"] = "transform" + if store.get(COLLECTION, doc["_id"]) and not overwrite: + raise ValueError( + f"feature '{doc['feature_id']}' exists (overwrite=True or new_version)" + ) + doc.setdefault("created_at", _now()) + return store.put(COLLECTION, doc) + + +def update_feature(store, feature_id: str, fields: dict) -> dict: + doc = get_feature(store, feature_id) + if not doc: + raise ValueError(f"no feature {feature_id}") + if "metrics" in fields: + doc["metrics"] = doc.get("metrics", []) + list(fields.pop("metrics")) + doc.update(fields) + doc["updated_at"] = _now() + return store.put(COLLECTION, doc) + + +def deprecate_feature(store, feature_id: str, reason: Optional[str] = None) -> dict: + return update_feature( + store, feature_id, {"status": "deprecated", "deprecation_reason": reason} + ) + + +def new_version( + store, feature_id: str, fields: dict, *, new_feature_id: Optional[str] = None +) -> dict: + old = get_feature(store, feature_id) + if not old: + raise ValueError(f"no feature {feature_id}") + nv = dict(old, **fields) + nv["version"] = str(int(str(old.get("version", "1")).split(".")[0]) + 1) + nv["feature_id"] = new_feature_id or f"{feature_id}_v{nv['version']}" + nv["parent_feature_id"] = feature_id + nv["generation"] = int(old.get("generation", 0)) + 1 + nv.pop("_id", None) + nv.pop("updated_at", None) + out = register_feature(store, nv, overwrite=True) + update_feature( + store, feature_id, {"status": "superseded", "superseded_by": out["feature_id"]} + ) + return out + + +# --------------------------------------------------------------------------- # +# learn (FLOps selection) +# --------------------------------------------------------------------------- # +def select_features( + series, *, reference_feature: str = "mean", lookback=None, cd_margin=0.05 +): + """FLOps over the full library (no store needed) — backward-compatible.""" + from ..reasoning import feature_selection as fsel + + return fsel.select_features( + series, + reference_feature=reference_feature, + lookback=lookback, + cd_margin=cd_margin, + ) + + +def select_features_from_catalog( + store, + series, + *, + reference_feature: str = "mean", + cd_margin: float = 0.05, + write_back: bool = False, +) -> dict: + """FLOps over the catalog's extractor library; optionally write the importance score back + onto each extractor doc's metrics (the catalog 'learns').""" + from ..reasoning import feature_selection as fsel + + cands = list_extractors(store) + names = [ + c["extractor_name"] for c in cands if c.get("extractor_name") in fsel.EXTRACTORS + ] + subset = {n: fsel.EXTRACTORS[n] for n in names} or fsel.EXTRACTORS + res = fsel.select_features( + series, + reference_feature=reference_feature, + cd_margin=cd_margin, + extractors=subset, + ) + res["candidates"] = names + if write_back: + for name, sc in res["scores"].items(): + if store.get(COLLECTION, _id(name)): + update_feature( + store, + name, + { + "metrics": [ + { + "metric": "flops_importance", + "value": sc, + "dataset": "last_run", + } + ] + }, + ) + return res + diff --git a/src/servers/tsfm/stores/model_store.py b/src/servers/tsfm/stores/model_store.py new file mode 100644 index 000000000..fa8ebbbef --- /dev/null +++ b/src/servers/tsfm/stores/model_store.py @@ -0,0 +1,304 @@ +"""Model store — registry of TS models on the core Store. + +Pointer index: weights live at artifact_path / hf_repo / remote_endpoint / model_checkpoint +(toolkit); the catalog points at them. Capabilities: + read : get / list / find_models (explainable ranking) / search / get_lineage + write : register (schema-validated) / update / deprecate / new_version / register_finetuned + resolve : resolve_checkpoint (for compute tools) +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Dict, List, Optional + +from ..core import schemas + +COLLECTION = "model_catalog" + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _id(model_id: str) -> str: + return f"model:{model_id}" + + +# --------------------------------------------------------------------------- # +# read +# --------------------------------------------------------------------------- # +def get_model(store, model_id: str) -> Optional[dict]: + return store.get(COLLECTION, _id(model_id)) + + +def list_models( + store, + task_id: Optional[str] = None, + domain: Optional[str] = None, + modality: Optional[str] = None, + framework: Optional[str] = None, + usage_mode: Optional[str] = None, + status: str = "active", +) -> List[dict]: + sel: Dict = {} + if status: + sel["status"] = status + if domain: + sel["domain"] = domain + if modality: + sel["modality"] = modality + if framework: + sel["framework"] = framework + if task_id: + sel["task_ids"] = {"$elemMatch": task_id} + if usage_mode: + sel["usage_modes"] = {"$elemMatch": usage_mode} + return store.find(COLLECTION, sel) + + +def _best_metric(m: dict, name: str): + vals = [ + x["value"] + for x in m.get("metrics", []) + if x.get("metric") == name and x.get("value") is not None + ] + return min(vals) if vals else None + + +def find_models( + store, + task_id: str, + *, + min_context_length: Optional[int] = None, + prediction_length: Optional[int] = None, + domain: Optional[str] = None, + modality: Optional[str] = None, + usage_mode: Optional[str] = None, + top_k: int = 5, + explain: bool = False, +): + """Filter by task + structured constraints, rank, return top_k. + + Ranking (lexicographic, lower=better): domain match → has eval MAE (lower) → longer + context → fewer-params proxy (shorter id). With explain=True, attaches a `_rank` reason. + """ + cands = list_models( + store, task_id=task_id, domain=None, modality=modality, usage_mode=usage_mode + ) + + def ok(m): + if min_context_length and (m.get("context_length") or 0) < min_context_length: + return False + if prediction_length and (m.get("prediction_length") or 0) < prediction_length: + return False + return True + + cands = [m for m in cands if ok(m)] + + def score(m): + domain_match = 0 if (domain and m.get("domain") == domain) else 1 + mae = _best_metric(m, "mae") + return ( + domain_match, + mae if mae is not None else float("inf"), + -(m.get("context_length") or 0), + ) + + ranked = sorted(cands, key=score)[:top_k] + if explain: + for r in ranked: + r["_rank"] = { + "domain_match": bool(domain and r.get("domain") == domain), + "mae": _best_metric(r, "mae"), + "context_length": r.get("context_length"), + } + return ranked + + +def search( + store, text: str = "", *, tags: Optional[List[str]] = None, status: str = "active" +) -> List[dict]: + """Free-text over id/description/family/tags + optional tag filter.""" + text = (text or "").lower() + out = [] + for m in list_models(store, status=status): + hay = " ".join( + [ + m.get("model_id") or "", + m.get("description") or "", + m.get("model_family") or "", + " ".join(m.get("tags") or []), + ] + ).lower() + if text and text not in hay: + continue + if tags and not set(tags) <= set(m.get("tags", [])): + continue + out.append(m) + return out + + +def describe_candidates( + store, task_id: str, *, top_k: int = 5, domain: str = None +) -> list: + """HuggingGPT-style model selection surface: return the top_k candidate cards for a task as + compact {model_id, description, downloads, family, context_length} records, ranked by a + popularity/quality prior (downloads, then eval metric) — the '{{Candidate Models}}' the + agent reasons over to pick/ensemble. Trims tokens like HuggingGPT's top-K-by-downloads. + """ + cands = list_models(store, task_id=task_id, domain=domain) + + def prior(m): + dl = m.get("downloads") or 0 + mae = _best_metric(m, "mae") + return (-(dl), mae if mae is not None else float("inf")) # more downloads first + + out = [] + for m in sorted(cands, key=prior)[:top_k]: + out.append( + { + "model_id": m["model_id"], + "description": m.get("description", ""), + "downloads": m.get("downloads"), + "family": m.get("family") or m.get("model_family"), + "sktime_class": m.get("sktime_class"), + "context_length": m.get("context_length"), + "tags": m.get("tags", []), + } + ) + return out + + +def get_lineage(store, model_id: str) -> dict: + """Ancestors (base chain) + descendants (fine-tunes of this).""" + ancestors, cur = [], get_model(store, model_id) + seen = set() + while cur and cur.get("base_model_id") and cur["base_model_id"] not in seen: + seen.add(cur["base_model_id"]) + parent = get_model(store, cur["base_model_id"]) + if not parent: + break + ancestors.append(parent["model_id"]) + cur = parent + descendants = [ + m["model_id"] for m in store.find(COLLECTION, {"base_model_id": model_id}) + ] + return { + "model_id": model_id, + "ancestors": ancestors, + "root": ancestors[-1] if ancestors else model_id, + "descendants": descendants, + } + + +# --------------------------------------------------------------------------- # +# write +# --------------------------------------------------------------------------- # +def register_model(store, model: dict, *, overwrite: bool = False) -> dict: + doc = schemas.validate_model(model) # raises on invalid + if store.get(COLLECTION, doc["_id"]) and not overwrite: + raise ValueError( + f"model '{doc['model_id']}' exists (overwrite=True or new_version)" + ) + doc.setdefault("created_at", _now()) + return store.put(COLLECTION, doc) + + +def update_model(store, model_id: str, fields: dict) -> dict: + doc = get_model(store, model_id) + if not doc: + raise ValueError(f"no model {model_id}") + if "metrics" in fields: # append, don't replace + doc["metrics"] = doc.get("metrics", []) + list(fields.pop("metrics")) + doc.update(fields) + doc["updated_at"] = _now() + return store.put(COLLECTION, schemas.validate_model(doc)) + + +def deprecate_model(store, model_id: str, reason: Optional[str] = None) -> dict: + return update_model( + store, model_id, {"status": "deprecated", "deprecation_reason": reason} + ) + + +def new_version( + store, model_id: str, fields: dict, *, new_model_id: Optional[str] = None +) -> dict: + """Register a new version; the predecessor is marked superseded and linked.""" + old = get_model(store, model_id) + if not old: + raise ValueError(f"no model {model_id}") + nv = dict(old, **fields) + nv["version"] = str(int(str(old.get("version", "1")).split(".")[0]) + 1) + nv["model_id"] = new_model_id or f"{model_id}_v{nv['version']}" + nv["supersedes"] = model_id + nv.pop("_id", None) + nv.pop("updated_at", None) + out = register_model(store, nv, overwrite=True) + update_model( + store, model_id, {"status": "superseded", "superseded_by": out["model_id"]} + ) + return out + + +def register_finetuned( + store, + *, + model_id: str, + checkpoint_path: str, + base_model_id: str, + context_length: int, + prediction_length: int, + description: str, + domain: str = "general", + metrics: Optional[list] = None, + overwrite: bool = True, +) -> dict: + """Agent-decided write-back: point the catalog at a fine-tune checkpoint location.""" + return register_model( + store, + { + "model_id": model_id, + "model_checkpoint": checkpoint_path, + "artifact_path": checkpoint_path, + "source": "local_artifact", + "framework": "tinytimemixer", + "modality": "timeseries", + "provenance": "finetuned", + "base_model_id": base_model_id, + "task_ids": ["tsfm_forecasting", "tsfm_forecasting_evaluation"], + "context_length": context_length, + "prediction_length": prediction_length, + "domain": domain, + "description": description, + "metrics": metrics or [], + "created_by": "agent.tsfm.finetune", + }, + overwrite=overwrite, + ) + + +# --------------------------------------------------------------------------- # +# resolve (for compute tools) +# --------------------------------------------------------------------------- # +def resolve_checkpoint(store, model_id_or_path: str) -> dict: + m = get_model(store, model_id_or_path) + if m is None: + return { + "model_checkpoint": model_id_or_path, + "context_length": None, + "prediction_length": None, + "resolved_from": "raw_path", + } + return { + "model_id": m["model_id"], + "model_checkpoint": m.get("model_checkpoint"), + "framework": m.get("framework"), + "artifact_path": m.get("artifact_path"), + "hf_repo": m.get("hf_repo"), + "remote_endpoint": m.get("remote_endpoint"), + "context_length": m.get("context_length"), + "prediction_length": m.get("prediction_length"), + "resolved_from": "catalog", + } diff --git a/src/servers/tsfm/stores/results.py b/src/servers/tsfm/stores/results.py new file mode 100644 index 000000000..5ed05d067 --- /dev/null +++ b/src/servers/tsfm/stores/results.py @@ -0,0 +1,88 @@ +"""Result tables — one collection per task type, on the core Store.""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import List, Optional + +TASK_TABLE = { + "tsfm_forecasting": ("forecast_result", "fr"), + "tsfm_anomaly_detection": ("anomaly_result", "ar"), + "tsfm_classification": ("classification_result", "cr"), + "tsfm_imputation": ("imputation_result", "ir"), + "tsfm_forecasting_evaluation": ("evaluation_result", "er"), +} +REQUIRED_SUMMARY = { + "tsfm_forecasting": ["horizon"], + "tsfm_anomaly_detection": ["total_records", "anomaly_count"], + "tsfm_classification": ["num_classes"], + "tsfm_imputation": ["imputed_count"], + "tsfm_forecasting_evaluation": [], +} + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def collection_for(task_type: str) -> str: + if task_type not in TASK_TABLE: + raise ValueError(f"unknown task_type '{task_type}'") + return TASK_TABLE[task_type][0] + + +def write_result( + store, + task_type: str, + *, + asset_id: str, + results_file: str, + model_id: Optional[str] = None, + feature_ids: Optional[List[str]] = None, + dataset: Optional[str] = None, + scenario_id: Optional[str] = None, + summary: Optional[dict] = None, + metrics: Optional[list] = None, + created_by: str = "agent.tsfm", +) -> dict: + coll, prefix = TASK_TABLE.get(task_type, (None, None)) + if coll is None: + raise ValueError(f"unknown task_type '{task_type}'") + summary = summary or {} + missing = [k for k in REQUIRED_SUMMARY.get(task_type, []) if k not in summary] + if task_type == "tsfm_forecasting_evaluation" and not metrics: + missing = ["metrics"] + if missing: + raise ValueError(f"{task_type}: missing required {missing}") + rid = f"{prefix}:{uuid.uuid4().hex[:12]}" + doc = { + "_id": rid, + "result_id": rid, + "task_type": task_type, + "asset_id": asset_id, + "model_id": model_id, + "feature_ids": feature_ids or [], + "dataset": dataset, + "scenario_id": scenario_id, + "results_file": results_file, + "summary": summary, + "metrics": metrics or [], + "status": "produced", + "created_by": created_by, + "produced_at": _now(), + } + return store.put(coll, doc) + + +def get_result(store, task_type, result_id): + return store.get(collection_for(task_type), result_id) + + +def list_results(store, task_type, asset_id=None, scenario_id=None): + sel = {} + if asset_id: + sel["asset_id"] = asset_id + if scenario_id: + sel["scenario_id"] = scenario_id + return store.find(collection_for(task_type), sel) diff --git a/src/servers/tsfm/substrate/__init__.py b/src/servers/tsfm/substrate/__init__.py new file mode 100644 index 000000000..1eb563bc2 --- /dev/null +++ b/src/servers/tsfm/substrate/__init__.py @@ -0,0 +1 @@ +"""substrate layer.""" diff --git a/src/servers/tsfm/substrate/resolver.py b/src/servers/tsfm/substrate/resolver.py new file mode 100644 index 000000000..b9e3e9823 --- /dev/null +++ b/src/servers/tsfm/substrate/resolver.py @@ -0,0 +1,175 @@ +"""sktime_resolver.py — the model/feature store on top of sktime (the substrate). + +Key decision: do NOT hand-roll an estimator/transform contract. Adopt sktime's scitype + +tag + registry system. A catalog card is just a *pointer* to an sktime estimator class + its +constructor params + tags; resolving a card = import & instantiate; running it = the scitype's +verb (forecaster.predict / classifier.predict / detector.predict / transformer.transform / +clusterer.predict). Heterogeneous foundation-model code paths disappear: every TSFM (TTM, +Chronos, MOIRAI, TimesFM, MOMENT, TimeMoE, PatchTST, …) is a `BaseForecaster` in sktime. + +Our catalog (CouchDB) is a *superset* of sktime's in-memory registry: it also holds +not-installed / remote / fine-tuned models with provenance, lineage, metrics — and is +agent-queryable and state-exportable (#394). sktime supplies fit/predict/pipeline/splitter/ +metric; we supply catalog + selection (T-Daub) + reasoning + persistence + MCP. + +Nested estimators (#pyod_iforest): some sktime estimators take *another estimator instance* +as a constructor arg (e.g. the PyOD adapter's `estimator=IsolationForest()`). JSON can't hold a +live object, so a param value may instead be a nested spec dict: + + {"_target_": "pyod.models.iforest.IForest", "params": {"contamination": 0.1}} + +`_build` recursively turns any such spec into `Class(**params)` before the outer estimator is +instantiated. Plain params (and dicts without `_target_`) pass through unchanged, so this is +fully backward-compatible with existing cards. +""" + +from __future__ import annotations + +import importlib +from typing import Any, List, Optional + +# task_id -> sktime scitype (the standardization rides on sktime's taxonomy) +TASK_TO_SCITYPE = { + "tsfm_forecasting": "forecaster", # incl. global_forecaster (foundation models) + "tsfm_regression": "regressor", + "tsfm_classification": "classifier", + "tsfm_anomaly_detection": "detector", # sktime detection scitype (anomaly/segmentation) + "tsfm_imputation": "transformer", # sktime Imputer is a transformer + "tsfm_clustering": "clusterer", + "tsfm_similarity_search": "transformer", # feature/embedding transformer + distances + "tsfm_evaluation": "metric", # metric + splitter (sktime.evaluate) +} + +# scitype -> the verb to call after fit +_VERB = { + "forecaster": "predict", + "regressor": "predict", + "classifier": "predict", + "clusterer": "predict", + "detector": "predict", + "transformer": "transform", + "metric": None, +} + +# Marker key: a param value that is a dict carrying this key is a nested estimator spec. +_TARGET_KEY = "_target_" + + +def _import_target(path: str): + """Import a dotted path 'pkg.module.Name' -> the class/callable Name.""" + module, name = path.rsplit(".", 1) + return getattr(importlib.import_module(module), name) + + +def _build(spec: Any) -> Any: + """Recursively realize a param value. + + - dict with `_target_` -> instantiate `Class(**built(params))` (a nested estimator) + - other dict -> rebuild values (a plain kwargs dict, e.g. sktime pipeline steps) + - list/tuple -> rebuild elements + - anything else -> returned as-is + """ + if isinstance(spec, dict): + if _TARGET_KEY in spec: + Cls = _import_target(spec[_TARGET_KEY]) + params = spec.get("params") or {} + return Cls(**{k: _build(v) for k, v in params.items()}) + return {k: _build(v) for k, v in spec.items()} + if isinstance(spec, (list, tuple)): + return type(spec)(_build(v) for v in spec) + return spec + + +def resolve(card: dict): + """Instantiate the sktime estimator a catalog card points at. + + Card `params` are realized with `_build`, so any nested `_target_` estimator specs + (e.g. the PyOD adapter's `estimator`) are constructed before the outer estimator. + """ + path = card.get("sktime_class") + if not path: + raise ValueError(f"card '{card.get('model_id')}' has no sktime_class") + Est = _import_target(path) + params = {k: _build(v) for k, v in (card.get("params") or {}).items()} + return Est(**params) + + +def discover(scitype: str, filter_tags: Optional[dict] = None) -> List[str]: + """sktime registry as live model discovery (installed estimators).""" + from sktime.registry import all_estimators + + ests = all_estimators(estimator_types=scitype, filter_tags=filter_tags) + return [n for n, _ in ests] + + +# foundation-model module/name keywords → pretrained, zero-shot-capable +_FM_KEYS = ( + "tinytime", + "ttm", + "chronos", + "moirai", + "timesfm", + "moment", + "timemoe", + "patchtst", + "lagllama", + "hftransformers", + "tspulse", +) + +# params that signal an *opt-in* fine-tune on a foundation model +_FT_KEYS = ( + "num_train_epochs", + "fit_strategy", + "trainer", + "fine_tune", + "finetune", + "lr", +) + + +def foundation_forecasters() -> List[str]: + return [n for n in discover("forecaster") if any(k in n.lower() for k in _FM_KEYS)] + + +def is_foundation(card: dict) -> bool: + cls = (card.get("sktime_class") or "").lower() + return any(k in cls for k in _FM_KEYS) + + +def training_regime(card: dict) -> str: + """How much training a card needs: zero_shot | fit_on_series | fine_tune. + + The agent reads this to pick the cheapest viable option. A pretrained foundation model is + zero_shot by default (fit() only loads weights + sets context; it does NOT train on the + target) — unless the recipe passes fine-tune params, which makes it fine_tune. Everything + else (AutoARIMA/ETS/Theta/reduction) is fit_on_series: cheap parameter estimation on the + series' own history, no separate training set. An explicit card `training_regime` wins. + """ + explicit = card.get("training_regime") + if explicit: + return explicit + if is_foundation(card): + params = card.get("params") or {} + return "fine_tune" if any(k in params for k in _FT_KEYS) else "zero_shot" + return "fit_on_series" + + +def run(card: dict, task_id: str, *, y=None, X=None, fh=None): + """Fit + run an sktime estimator via its scitype verb. Returns the native sktime output.""" + scitype = TASK_TO_SCITYPE.get(task_id) + est = resolve(card) + verb = _VERB.get(scitype) + if scitype == "forecaster": + est.fit(y, X=X, fh=fh) if X is not None else est.fit(y, fh=fh) + return est.predict(fh=fh) + if scitype in ("classifier", "regressor", "clusterer"): + est.fit(X, y) + return getattr(est, verb)(X) + if scitype == "detector": + est.fit(X if X is not None else y) + return est.predict(X if X is not None else y) + if scitype == "transformer": + est.fit(y if y is not None else X) + return est.transform(y if y is not None else X) + raise ValueError(f"no runner for scitype '{scitype}'") diff --git a/src/servers/tsfm/tests/conftest.py b/src/servers/tsfm/tests/conftest.py index 6c4cc510b..63daef5b3 100644 --- a/src/servers/tsfm/tests/conftest.py +++ b/src/servers/tsfm/tests/conftest.py @@ -1,30 +1,10 @@ -"""Shared fixtures and helpers for TSFM MCP server tests.""" +"""Test config — keep the suite hermetic. -from __future__ import annotations +Production uses CouchDB (TSFM_STORE=couch, like the other AssetOpsBench servers). Tests force the +in-memory backend so every run starts from the curated seeds with no service dependency — the +same skipif-on-missing-service pattern the sibling servers use for their CouchDB integration tests. +""" -import json import os -import pytest - - -# Skip marker for tests that require tsfm_public + its ML dependencies. -def _tsfm_available() -> bool: - try: - import tsfm_public # noqa: F401 - - return True - except ImportError: - return False - - -requires_tsfm = pytest.mark.skipif( - not _tsfm_available(), - reason="tsfm_public not installed", -) - - -async def call_tool(mcp_instance, tool_name: str, args: dict) -> dict: - """Helper: call an MCP tool and return the parsed JSON response.""" - contents, _ = await mcp_instance.call_tool(tool_name, args) - return json.loads(contents[0].text) +os.environ["TSFM_STORE"] = "memory" diff --git a/src/servers/tsfm/tests/test_anomaly_dq.py b/src/servers/tsfm/tests/test_anomaly_dq.py new file mode 100644 index 000000000..0bf525761 --- /dev/null +++ b/src/servers/tsfm/tests/test_anomaly_dq.py @@ -0,0 +1,62 @@ +"""Anomaly via run_recipe (task dispatch) + the data-quality tool — through the MCP boundary.""" + +import asyncio, json, os, warnings +warnings.filterwarnings("ignore") + +import numpy as np +import pandas as pd + +from ..main import mcp +from ..io import refs + + +def call(name, args): + content, _ = asyncio.run(mcp.call_tool(name, args)) + return json.loads(content[0].text) + + +def _spiky_iot(n=300): + y = np.sin(np.arange(n) / 6.0) + 0.05 * np.random.RandomState(0).randn(n) + y[120] += 8.0; y[240] -= 8.0 # two injected anomalies + return refs.materialize_iot(y, asset_id="adq") + + +# ---- anomaly runs through run_recipe (task dispatch), not a separate tool ---- +def test_run_recipe_anomaly_sublof_flags_spikes(): + ref = _spiky_iot() + r = call("run_recipe", {"dataset_path": ref, "timestamp_column": "timestamp", + "target_columns": ["value"], + "recipe": {"task": "tsfm_anomaly_detection", + "estimator": {"model_id": "sublof"}}}) + assert r["status"] == "success" and r["results_file"].startswith("file://") + assert r["n_anomalies"] >= 1 and r["n_observations"] == 300 + assert r["training_regime"] == "fit_on_series" # SubLOF is classical + rec = json.load(open(refs._path(r["results_file"]))) + assert len(rec["anomaly_label"]) == 300 and sum(rec["anomaly_label"]) == r["n_anomalies"] + + +def test_anomaly_validation_and_tspulse_gated(): + ref = _spiky_iot() + assert "error" in call("run_recipe", {"dataset_path": ref, "timestamp_column": "timestamp", + "target_columns": ["value"], + "recipe": {"task": "tsfm_anomaly_detection"}}) # no detector + # TSPulse zero-shot resolves to the sktime detector but needs tsfm_public → typed error, not crash + r = call("run_recipe", {"dataset_path": ref, "timestamp_column": "timestamp", + "target_columns": ["value"], + "recipe": {"task": "tsfm_anomaly_detection", + "estimator": {"model_id": "tspulse_ad"}}}) + assert "error" in r + + +# ---- data quality tool ---- +def test_data_quality_cleans_and_summarizes(): + n = 100 + df = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", periods=n, freq="15min"), + "value": np.sin(np.arange(n) / 5.0)}) + df.loc[10:14, "value"] = np.nan # a gap of NaNs + refs._ensure_workdir() + p = os.path.join(refs.WORKDIR, "dq_in.csv"); df.to_csv(p, index=False) + r = call("data_quality", {"dataset_path": p, "timestamp_column": "timestamp"}) + assert r["status"] == "success" and r["cleaned_file"].startswith("file://") + assert r["rows_out"] < r["rows_in"] # NaN rows removed + assert "error" in call("data_quality", {"dataset_path": ""}) diff --git a/src/servers/tsfm/tests/test_catalog_growth.py b/src/servers/tsfm/tests/test_catalog_growth.py new file mode 100644 index 000000000..0857b6253 --- /dev/null +++ b/src/servers/tsfm/tests/test_catalog_growth.py @@ -0,0 +1,66 @@ +"""Lock in the catalog growth: 100+ FLOps extractors + migrated sktime foundation models.""" + +import math, warnings +warnings.filterwarnings("ignore") + +import numpy as np + +from ..reasoning import feature_selection as F +from ..bootstrap import fresh_store +from ..stores import model_store, feature_store + + +def test_extractor_library_is_100_plus_and_robust(): + assert len(F.EXTRACTORS) >= 100 + # every extractor returns a finite float across diverse windows (incl. constant/short) + for w in [np.sin(np.arange(40) / 3.0), np.ones(16), np.arange(8.0), np.zeros(10)]: + for name, fn in F.EXTRACTORS.items(): + v = fn(w) + assert isinstance(v, float) and math.isfinite(v), f"{name} → {v!r}" + + +def test_migrated_foundation_models_present_and_resolvable(): + s = fresh_store() + fc = model_store.list_models(s, task_id="tsfm_forecasting") + fams = {m.get("model_family") for m in fc} + assert {"chronos", "moirai", "moment", "timesfm", "timemoe"} <= fams + chronos = model_store.get_model(s, "amazon__chronos-t5-small") + assert chronos["sktime_class"].endswith("ChronosForecaster") + assert chronos["params"]["model_path"] == "amazon/chronos-t5-small" + assert chronos["training_regime"] == "zero_shot" and chronos["framework"] == "sktime" + + +def test_migrated_models_carry_provenance_and_validate(): + s = fresh_store() + migrated = [m for m in model_store.list_models(s) if m.get("created_by") == "migrated_curated"] + assert len(migrated) >= 25 + for m in migrated: + assert m["sktime_class"] and m["task_ids"] and m["description"] and m["provenance"] == "pretrained" + + +def test_ttm_cards_are_runnable(): + """The base TTM cards resolve to an sktime forecaster (model_path set) so run_recipe can + forecast on them — the IoT-file → TSFM forecast workflow's model entry.""" + from ..substrate import resolver as R + s = fresh_store() + for mid in ["ttm_96_28", "ttm_512_96", "ttm_chiller6_512_96_ft"]: + c = model_store.get_model(s, mid) + assert c["sktime_class"].endswith("TinyTimeMixerForecaster") + assert c["params"]["model_path"] # points at a checkpoint + assert R.is_foundation(c) and R.training_regime(c) == "zero_shot" + runnable = [m for m in model_store.list_models(s, task_id="tsfm_forecasting") if m.get("sktime_class")] + assert len(runnable) >= 38 + + +def test_every_seed_model_card_validates(): + """Lint: every model card in the catalog must satisfy ModelCard (so update/version/deprecate, + which re-validate on write, never fail on seed data).""" + from ..core import schemas + s = fresh_store() + bad = [] + for m in model_store.list_models(s, status=None): + try: + schemas.ModelCard(**m) + except Exception as e: + bad.append((m.get("model_id"), str(e).splitlines()[-2] if "\n" in str(e) else str(e)[:60])) + assert not bad, f"invalid model cards: {bad[:5]}" diff --git a/src/servers/tsfm/tests/test_catalog_tools.py b/src/servers/tsfm/tests/test_catalog_tools.py new file mode 100644 index 000000000..a1acd45c1 --- /dev/null +++ b/src/servers/tsfm/tests/test_catalog_tools.py @@ -0,0 +1,69 @@ +"""Catalog lifecycle tools (pull + update + version + retire) via the real MCP boundary.""" + +import asyncio, json, warnings +warnings.filterwarnings("ignore") + +from ..main import mcp + + +def call(name, args): + content, _ = asyncio.run(mcp.call_tool(name, args)) + return json.loads(content[0].text) + + +# ---- model store ---- +def test_list_models(): + assert len(call("list_models", {})["models"]) >= 30 # full active catalog + fc = call("list_models", {"task_id": "tsfm_forecasting"})["models"] + assert fc and all("tsfm_forecasting" in m["task_ids"] for m in fc) + assert "error" in call("list_models", {"task_id": "tsfm_made_up"}) # validated + + +def test_search_models(): + r = call("search_models", {"text": "chronos"}) + assert r["models"] and all("chronos" in (m.get("model_id", "") + " " + m.get("model_family", "")).lower() + for m in r["models"]) + assert "error" not in call("search_models", {}) # empty text → all active + + +def test_model_lineage_update_deprecate(): + assert "error" in call("get_model_lineage", {"model_id": ""}) + lin = call("get_model_lineage", {"model_id": "ttm_96_28"}) + assert "model_id" in lin or "ancestors" in lin or "descendants" in lin + upd = call("update_model", {"model_id": "ttm_96_28", "fields": {"domain": "energy"}}) + assert upd.get("domain") == "energy" + dep = call("deprecate_model", {"model_id": "autoarima", "reason": "test"}) + assert dep.get("status") == "deprecated" + assert "error" in call("update_model", {"model_id": "x", "fields": {}}) + + +def test_new_model_version_and_finetuned(): + nv = call("new_model_version", {"model_id": "naive_persistence", + "fields": {"description": "naive baseline, revised"}}) + assert nv.get("supersedes") == "naive_persistence" and nv.get("model_id", "").startswith("naive_persistence") + ft = call("register_finetuned", { + "model_id": "ttm_chiller_ft", "checkpoint_path": "/art/ttm_chiller", "base_model_id": "ttm_96_28", + "context_length": 96, "prediction_length": 28, "description": "TTM fine-tuned on chiller data"}) + assert ft.get("provenance") == "finetuned" and ft.get("base_model_id") == "ttm_96_28" + assert "error" in call("register_finetuned", {"model_id": "", "checkpoint_path": "p", + "base_model_id": "b", "context_length": 1, + "prediction_length": 1, "description": "x"}) + + +# ---- feature store ---- +def test_search_and_list_extractors(): + assert call("search_features", {"text": "normalization"})["features"] + assert len(call("list_extractors", {})["features"]) >= 100 + + +def test_feature_lineage_update_version_deprecate(): + assert "error" in call("get_feature_lineage", {"feature_id": ""}) + lin = call("get_feature_lineage", {"feature_id": "trend_slope_v1"}) + assert isinstance(lin, dict) + upd = call("update_feature", {"feature_id": "trend_slope_v1", "fields": {"tags": ["degradation"]}}) + assert "degradation" in (upd.get("tags") or []) + nv = call("new_feature_version", {"feature_id": "trend_slope_v1", + "fields": {"name": "trend slope v2"}}) + assert nv.get("parent_feature_id") == "trend_slope_v1" + dep = call("deprecate_feature", {"feature_id": "channel_select_v1"}) + assert dep.get("status") == "deprecated" diff --git a/src/servers/tsfm/tests/test_characterize.py b/src/servers/tsfm/tests/test_characterize.py new file mode 100644 index 000000000..29d291fbf --- /dev/null +++ b/src/servers/tsfm/tests/test_characterize.py @@ -0,0 +1,61 @@ +"""P4: characterize_series through the MCP boundary — file pointer in → pattern evidence out.""" + +import asyncio, json, warnings +warnings.filterwarnings("ignore") + +import numpy as np +import pandas as pd +from ..main import mcp +from ..io import refs + +N = 168 + + +def call(name, args): + content, _ = asyncio.run(mcp.call_tool(name, args)) + return json.loads(content[0].text) + + +def _write(frame: dict) -> str: + df = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", periods=N, freq="h"), **frame}) + refs._ensure_workdir() + import os + p = os.path.join(refs.WORKDIR, "characterize_in.csv") + df.to_csv(p, index=False) + return p + + +def test_generic_default_per_channel(): + p = _write({"sensor_A": np.linspace(0, 6, N), "flow": 0.1 * np.random.RandomState(0).randn(N)}) + r = call("characterize_series", {"dataset_path": p, "timestamp_column": "timestamp"}) + assert r["status"] == "success" and r["evidence_file"].startswith("file://") + assert set(r["groups"]) == {"sensor_A", "flow"} + pg = r["phases"][0]["per_group"] + assert pg["sensor_A"]["state"] == "RISE" and pg["flow"]["state"] == "STABLE" + + +def test_sentsr_decoupled_with_preset_grouping(): + # vibration rises, temperature stable — the SenTSR row-0 archetype, via the opt-in preset + ramp = np.linspace(0, 6, N) + rng = np.random.RandomState(1) + p = _write({"Acceleration": ramp + 0.1 * rng.randn(N), "Velocity": ramp + 0.1 * rng.randn(N), + "Temperature": 0.1 * rng.randn(N)}) + r = call("characterize_series", {"dataset_path": p, "timestamp_column": "timestamp", + "group_rules": "vibration_temperature"}) + assert set(r["groups"]) == {"vibration", "temperature"} + pg = r["phases"][0]["per_group"] + assert pg["vibration"]["state"] == "RISE" and pg["temperature"]["state"] == "STABLE" + assert any(rel["type"] == "DECOUPLED" for rel in r["phases"][0]["relations"]) + + +def test_summary_is_fault_free(): + ramp = np.linspace(0, 6, N) + p = _write({"Acceleration": ramp, "Velocity": ramp, "Temperature": 0.1 * np.random.RandomState(2).randn(N)}) + r = call("characterize_series", {"dataset_path": p, "timestamp_column": "timestamp", + "group_rules": "vibration_temperature"}) + banned = ["alignment", "bearing", "lubric", "wear", "imbalance", "friction", "fault", "gear"] + assert not any(b in r["summary"].lower() for b in banned), r["summary"] + + +def test_validation_error(): + assert "error" in call("characterize_series", {"dataset_path": ""}) diff --git a/src/servers/tsfm/tests/test_composition.py b/src/servers/tsfm/tests/test_composition.py new file mode 100644 index 000000000..09f79aefb --- /dev/null +++ b/src/servers/tsfm/tests/test_composition.py @@ -0,0 +1,49 @@ +"""Core capability: agentic discover → compose → run → diagnose → iterate (sktime substrate).""" + +import os, sys, warnings +warnings.filterwarnings("ignore") +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +import numpy as np, pandas as pd +from ..core.store import MemoryStore +from ..engine import composition as C + +NF = "sktime.forecasting.naive.NaiveForecaster" +_Y = pd.Series(np.sin(np.arange(160) / 4.0) + 0.03 * np.arange(160)) + + +def test_discover_components(): + d = C.discover_components(MemoryStore()) + assert d["scitype"] == "forecaster" + assert "weighted" in d["combiners"] and "stack" in d["combiners"] + assert any("TinyTimeMixer" in m for m in d["foundation_models"]) + + +def test_single_then_ensemble_with_per_member(): + s = MemoryStore() + single = C.run_recipe(s, _Y, {"estimator": {"name": "drift", "sktime_class": NF, + "params": {"strategy": "drift"}}, "fh": [1, 2, 3, 4]}) + assert single["backtest_score"] > 0 + ens = C.run_recipe(s, _Y, {"ensemble": {"combine": "mean", "members": [ + {"name": "last", "sktime_class": NF, "params": {"strategy": "last"}}, + {"name": "mean", "sktime_class": NF, "params": {"strategy": "mean"}}, + {"name": "drift", "sktime_class": NF, "params": {"strategy": "drift"}}]}, "fh": [1, 2, 3, 4]}) + assert set(ens["per_member_score"]) == {"last", "mean", "drift"} # diagnostics for iterate + + +def test_iterate_improves_and_lineage(): + s = MemoryStore() + ens = {"ensemble": {"combine": "mean", "members": [ + {"name": "last", "sktime_class": NF, "params": {"strategy": "last"}}, + {"name": "mean", "sktime_class": NF, "params": {"strategy": "mean"}}, + {"name": "drift", "sktime_class": NF, "params": {"strategy": "drift"}}]}, "fh": [1, 2, 3, 4]} + r1 = C.run_recipe(s, _Y, ens) + pm = {k: v for k, v in r1["per_member_score"].items() if isinstance(v, (int, float))} + worst = max(pm, key=pm.get) + keep = [m for m in ens["ensemble"]["members"] if m["name"] != worst] + r2 = C.run_recipe(s, _Y, {"ensemble": {"combine": "mean", "members": keep}, "fh": [1, 2, 3, 4]}, + parent_run_id=r1["run_id"]) + assert r2["backtest_score"] <= r1["backtest_score"] # dropping the worst helps (or ties) + # lineage persisted + child = s.get(C.RUNS, r2["run_id"]) + assert child["parent_run_id"] == r1["run_id"] diff --git a/src/servers/tsfm/tests/test_evolve.py b/src/servers/tsfm/tests/test_evolve.py new file mode 100644 index 000000000..b3bc88ffe --- /dev/null +++ b/src/servers/tsfm/tests/test_evolve.py @@ -0,0 +1,100 @@ +"""AlphaEvolve loop: ask → (simulated agent mutates) → tell → archive grows + elites improve. + +The 'agent' here is a deterministic mutator (no LLM): the server-side loop is what we verify — +validation, evaluation→fitness, MAP-Elites cells, islands, lineage, and elite selection.""" + +import warnings +warnings.filterwarnings("ignore") + +import numpy as np + +from ..core.store import MemoryStore +from ..engine import evolve as E +from ..io import refs + +NAIVE = "sktime.forecasting.naive.NaiveForecaster" + + +def _data(): + return refs.materialize_iot(np.sin(np.arange(160) / 4.0) + 0.01 * np.arange(160), asset_id="evo") + + +def _single(strategy): + return {"estimator": {"sktime_class": NAIVE, "params": {"strategy": strategy}}, "fh": [1, 2, 3]} + + +def _ensemble(): + return {"ensemble": {"members": [{"sktime_class": NAIVE, "params": {"strategy": "last"}}, + {"sktime_class": NAIVE, "params": {"strategy": "mean"}}], + "combine": "mean"}, "fh": [1, 2, 3]} + + +def _with_transform(): + return {"estimator": {"sktime_class": NAIVE, "params": {"strategy": "drift"}}, + "transforms": [{"sktime_class": "sktime.transformations.series.detrend.Detrender"}], + "fh": [1, 2, 3]} + + +def _tell(store, recipe, ref, parent=None): + return E.evolve_tell(store, "tsfm_forecasting", "recipe", recipe, parent_id=parent, + data_ref=ref, timestamp_column="timestamp", target_columns=["value"]) + + +def test_seed_then_grow_archive_and_cells(): + s, ref = MemoryStore(), _data() + r0 = _tell(s, _single("last"), ref) + assert r0["accepted"] and r0["is_new_elite"] and r0["generation"] == 0 + # different STRUCTURE → different behaviour cells (MAP-Elites grid fills) + r_ens = _tell(s, _ensemble(), ref) + r_tr = _tell(s, _with_transform(), ref) + assert r_ens["cell"] != r0["cell"] and r_tr["cell"] != r0["cell"] + best = E.evolve_best(s, "tsfm_forecasting") + assert best["n_cells"] >= 3 + + +def test_elite_replacement_within_a_cell(): + s, ref = MemoryStore(), _data() + a = _tell(s, _single("mean"), ref) + b = _tell(s, _single("drift"), ref, parent=a["evolve_id"]) # same cell, child + assert b["cell"] == a["cell"] and b["generation"] == 1 + # exactly one of them is the elite for that cell; archive keeps both for lineage + elites = E._elites(s, "tsfm_forecasting") + cell_elite = [e for e in elites if e["cell"] == a["cell"]] + assert len(cell_elite) == 1 and cell_elite[0]["fitness"] == max(a["fitness"], b["fitness"]) + + +def test_ask_returns_parents_inspirations_and_evidence(): + s, ref = MemoryStore(), _data() + for rec in (_single("last"), _ensemble(), _with_transform()): + _tell(s, rec, ref) + ask = E.evolve_ask(s, "tsfm_forecasting", data_ref=ref, timestamp_column="timestamp", + channels=["value"]) + assert ask["parents"] and "program" in ask["parents"][0] + assert ask["evidence"]["n_observations"] == 160 and ask["contract"]["task_id"] == "tsfm_forecasting" + + +def test_ask_on_empty_archive_gives_seed_instructions(): + ask = E.evolve_ask(MemoryStore(), "tsfm_forecasting") + assert ask["parents"] == [] and "empty" in ask["instructions"].lower() + + +def test_invalid_candidate_rejected_not_archived(): + s, ref = MemoryStore(), _data() + bad = E.evolve_tell(s, "tsfm_forecasting", "recipe", {"foo": 1}, data_ref=ref, + timestamp_column="timestamp", target_columns=["value"]) + # no estimator/ensemble → run_recipe raises → caught as invalid, nothing archived + assert not bad["accepted"] and not s.find(E.ARCHIVE, {}) + + +def test_feature_program_evolves_and_gates(): + s = MemoryStore() + code = ("import numpy as np\n" + "class Transformation:\n" + " def fit(self, X, metadata): return {'m': float(np.mean(X))}\n" + " def transform(self, X, state): return np.asarray(X, float) - state['m']\n" + " def inverse_transform(self, Y, state): return np.asarray(Y, float) + state['m']\n") + feat = {"feature_id": "evo_center", "class_name": "Transformation", "code": code, + "invertible": True, "output_type": "centered_series"} + r = E.evolve_tell(s, "tsfm_forecasting", "feature", feat) + assert r["accepted"] and r["fitness"] >= 1.0 # passes gate; invertible bonus + assert E.evolve_best(s, "tsfm_forecasting", kind="feature")["elites"] diff --git a/src/servers/tsfm/tests/test_feature_extract.py b/src/servers/tsfm/tests/test_feature_extract.py new file mode 100644 index 000000000..203c57427 --- /dev/null +++ b/src/servers/tsfm/tests/test_feature_extract.py @@ -0,0 +1,61 @@ +"""Tests for the feature-extraction engine helper (engine/composition.extract_features). + +Placement: src/servers/tsfm/tests/test_extract_features.py +Patches feature_selection.EXTRACTORS to a tiny deterministic library so assertions are exact. +""" + +import numpy as np +import pytest + +from ..engine import composition as C +from ..reasoning import feature_selection as FS + + +@pytest.fixture +def stub_extractors(monkeypatch): + lib = {"mean": lambda w: float(np.mean(w)), "std": lambda w: float(np.std(w))} + monkeypatch.setattr(FS, "EXTRACTORS", lib) + yield lib + + +def test_whole_series_univariate(stub_extractors): + cols, F = C.extract_features({"a": np.array([1, 2, 3, 4.0])}, ["mean", "std"]) + assert cols == ["mean", "std"] # single channel -> no prefix + assert F.shape == (1, 2) # whole series -> one row + assert F[0, 0] == 2.5 + + +def test_windowed_univariate(stub_extractors): + cols, F = C.extract_features({"a": np.array([1, 2, 3, 4.0])}, ["mean"], window=2) + assert F.shape == (2, 1) # tiles [1,2],[3,4] + assert F[:, 0].tolist() == [1.5, 3.5] + + +def test_multivariate_whole_series(stub_extractors): + cols, F = C.extract_features( + {"a": np.array([1, 2, 3, 4.0]), "b": np.array([10, 20, 30, 40.0])}, ["mean"] + ) + assert cols == ["a.mean", "b.mean"] # multivariate -> channel-prefixed + assert F.tolist() == [[2.5, 25.0]] + + +def test_multivariate_windowed(stub_extractors): + cols, F = C.extract_features( + {"a": np.array([1, 2, 3, 4.0]), "b": np.array([10, 20, 30, 40.0])}, + ["mean", "std"], + window=2, + ) + assert cols == ["a.mean", "a.std", "b.mean", "b.std"] + assert F.shape == (2, 4) + assert F[0].tolist() == [1.5, 0.5, 15.0, 5.0] + + +def test_window_larger_than_series_falls_back(stub_extractors): + cols, F = C.extract_features({"a": np.array([1, 2, 3.0])}, ["mean"], window=99) + assert F.shape == (1, 1) # one whole-series window + assert F[0, 0] == 2.0 + + +def test_nan_is_zeroed(stub_extractors): + cols, F = C.extract_features({"a": np.array([np.nan, np.nan])}, ["mean"]) + assert F[0, 0] == 0.0 # np.nan_to_num applied \ No newline at end of file diff --git a/src/servers/tsfm/tests/test_feature_selection_v2.py b/src/servers/tsfm/tests/test_feature_selection_v2.py new file mode 100644 index 000000000..40c11fc09 --- /dev/null +++ b/src/servers/tsfm/tests/test_feature_selection_v2.py @@ -0,0 +1,42 @@ +"""FLOps multi-config selection: |corr| + F-test + mutual-info + model-importance, mean-rank.""" + +import os, sys, warnings +warnings.filterwarnings("ignore") +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +import numpy as np +from ..reasoning import feature_selection as F + + +def _signal(n=400): + t = np.arange(n) + return np.sin(t / 6.0) + 0.3 * np.sin(t / 23.0) + 0.02 * t + 0.05 * np.random.RandomState(0).randn(n) + + +def test_multiconfig_runs_all_scorers(): + r = F.select_features(_signal(), reference_feature="kurtosis") # weak reference + assert set(r["scorers"]) == {"corr", "f_test", "mutual_info", "model"} + assert set(r["per_scorer"]) == {"corr", "f_test", "mutual_info", "model"} + # aggregate scores are normalized in [0,1]; ranking is ordered best→worst + assert max(r["scores"].values()) <= 1.0 + 1e-9 + assert r["selected"] and r["ranking"][0][1] >= r["ranking"][-1][1] + + +def test_reference_excluded_and_selection_nonempty(): + r = F.select_features(_signal(), reference_feature="kurtosis", cd_margin=0.05) + assert "kurtosis" not in r["selected"] and len(r["selected"]) >= 1 + + +def test_fast_path_corr_only_backward_compatible(): + r = F.select_features(_signal(), scorers=["corr"]) + assert r["scorers"] == ["corr"] and r["selected"] + + +def test_aggregation_is_robust_vs_single_scorer(): + """A feature top-ranked by the aggregate should rank well under >1 scorer, not just one.""" + r = F.select_features(_signal(), reference_feature="mean") + top = r["ranking"][0][0] + k = max(5, len(r["scores"]) // 5) # top-quintile (scales with library size) + good = sum(1 for s in r["scorers"] + if top in sorted(r["per_scorer"][s], key=r["per_scorer"][s].get, reverse=True)[:k]) + assert good >= 2 # the aggregate top-ranked feature ranks well under >=2 scorers diff --git a/src/servers/tsfm/tests/test_gifteval.py b/src/servers/tsfm/tests/test_gifteval.py new file mode 100644 index 000000000..603dac84a --- /dev/null +++ b/src/servers/tsfm/tests/test_gifteval.py @@ -0,0 +1,44 @@ +"""GIFT-Eval-native evaluation: seasonal-naive normalization, geo-mean, mean-rank leaderboard.""" + +import os, sys, warnings +warnings.filterwarnings("ignore") +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +import numpy as np +from ..core.store import MemoryStore +from ..eval import gifteval as G + +NF = "sktime.forecasting.naive.NaiveForecaster" +TR = "sktime.forecasting.trend.TrendForecaster" + + +def _cfg(name, n, h, slope, seed): + t = np.arange(n) + y = slope * t + np.random.RandomState(seed).normal(0, 0.5, n) + return {"name": name, "y": y.tolist(), "fh": list(range(1, h + 1)), "sp": 1} + + +def test_seasonal_naive_normalizes_to_one(): + s = MemoryStore() + cfg = [_cfg("A", 150, 8, 0.05, 0)] + r = G.evaluate_recipe(s, {"estimator": {"sktime_class": NF, "params": {"strategy": "last"}}}, cfg) + # the recipe == the per-config baseline ⇒ normalized MASE ≈ 1.0 + assert abs(r["per_config"][0]["norm_mase"] - 1.0) < 0.05 + + +def test_leaderboard_ranks_better_recipe_first(): + s = MemoryStore() + cfgs = [_cfg("A", 160, 8, 0.05, 1), _cfg("B", 150, 6, 0.08, 2)] + recipes = { + "naive_last": {"estimator": {"sktime_class": NF, "params": {"strategy": "last"}}}, + "drift": {"estimator": {"sktime_class": NF, "params": {"strategy": "drift"}}}, + "ensemble": {"ensemble": {"combine": "mean", "members": [ + {"name": "drift", "sktime_class": NF, "params": {"strategy": "drift"}}, + {"name": "trend", "sktime_class": TR, "params": {}}]}}, + } + lb = G.leaderboard(s, recipes, cfgs, by="norm_mase") + names = [row["recipe"] for row in lb["leaderboard"]] + assert names[0] in ("ensemble", "drift") # a real model beats seasonal-naive on trend + assert names[-1] == "naive_last" + # winner's geo-mean normalized MASE < 1 (beats the baseline) + assert lb["leaderboard"][0]["agg"]["geomean_norm_mase"] < 1.0 diff --git a/src/servers/tsfm/tests/test_glossary.py b/src/servers/tsfm/tests/test_glossary.py new file mode 100644 index 000000000..83390659e --- /dev/null +++ b/src/servers/tsfm/tests/test_glossary.py @@ -0,0 +1,49 @@ +"""The agent can learn the vocabulary: glossary is canonical, surfaced, and errors teach.""" + +import asyncio, json, warnings +warnings.filterwarnings("ignore") + +from ..core import glossary as G +from ..main import mcp + +KEY_TERMS = {"task", "component", "candidate", "model", "feature", "recipe", + "transform", "ensemble", "training_regime", "param_schema", "data_ref"} + + +def call(name, args): + content, _ = asyncio.run(mcp.call_tool(name, args)) + return json.loads(content[0].text) + + +def test_glossary_is_canonical_and_complete(): + g = G.glossary() + assert KEY_TERMS <= set(g["terms"]) + for term, meta in g["terms"].items(): + assert meta["definition"] and meta.get("tool") # every term defined + points to a tool + assert len(g["workflow"]) >= 5 and g["principles"] + + +def test_discover_components_teaches_vocabulary(): + out = call("discover_components", {"task": "tsfm_forecasting"})["components"] + assert KEY_TERMS <= set(out["glossary"]) + assert "workflow" in out and "principles" in out + assert "recipe_blocks" in out and "training_regimes" in out + + +def test_list_tasks_have_plain_descriptions(): + tasks = call("list_tasks", {})["tasks"] + assert len(tasks) == 8 + assert all(t.get("description") for t in tasks) # every task is explained in plain words + + +def test_unknown_task_error_lists_valid_tasks(): + for tool in ("discover_components", "find_models", "describe_candidates"): + arg = "task" if tool == "discover_components" else "task_id" + r = call(tool, {arg: "tsfm_made_up"}) + assert "error" in r and "Valid tasks" in r["error"] and "tsfm_forecasting" in r["error"] + + +def test_short_glossary_in_server_instructions(): + instr = mcp.instructions or "" + assert "VOCABULARY" in instr and "WORKFLOW" in instr + assert "recipe" in instr and "component" in instr diff --git a/src/servers/tsfm/tests/test_main_filepointers.py b/src/servers/tsfm/tests/test_main_filepointers.py new file mode 100644 index 000000000..b6a9497f4 --- /dev/null +++ b/src/servers/tsfm/tests/test_main_filepointers.py @@ -0,0 +1,71 @@ +"""The redesigned tools take FILE POINTERS (dataset_path) and return typed Pydantic results +with a results_file pointer — matching the legacy TSFM tools and the IoT data model.""" + +import os, warnings +warnings.filterwarnings("ignore") + +import numpy as np +import pandas as pd + +from ..io import refs +from ..core.results_models import ( + ProfileResult, FeatureSelectionResult, RecipeResult, TabularResult, ErrorResult, +) +from ..main import ( + profile_series, select_features, run_recipe, run_tabular_recipe, list_tasks, +) + +NAIVE = {"estimator": {"sktime_class": "sktime.forecasting.naive.NaiveForecaster", + "params": {"strategy": "last"}}, "fh": [1, 2, 3]} + + +def test_profile_series_takes_file_pointer(): + ref = refs.materialize_iot(np.sin(np.arange(200) / 3.0) + 0.01 * np.arange(200), asset_id="p") + r = profile_series(ref) # dataset_path, not inline data + assert isinstance(r, ProfileResult) + assert r.n_observations == 200 and r.dominant_period and r.source == ref + + +def test_profile_series_validates_input(): + assert isinstance(profile_series(" "), ErrorResult) + + +def test_run_recipe_file_pointer_in_results_file_out(): + ref = refs.materialize_iot(np.sin(np.arange(120) / 4.0) + 0.01 * np.arange(120), asset_id="r") + r = run_recipe(ref, "timestamp", ["value"], NAIVE) + assert isinstance(r, RecipeResult) and r.status == "success" + assert r.results_file.startswith("file://") and isinstance(r.backtest_score, float) + # the results_file pointer actually resolves to the full run record + rec = __import__("json").load(open(refs._path(r.results_file))) + assert rec["run_id"] == r.run_id and "forecast_head" in rec + + +def test_run_recipe_validates_inputs(): + assert isinstance(run_recipe("", "timestamp", ["value"], NAIVE), ErrorResult) + ref = refs.materialize_iot(np.arange(50.0), asset_id="r2") + assert isinstance(run_recipe(ref, "timestamp", [], NAIVE), ErrorResult) + + +def test_select_features_file_pointer(): + ref = refs.materialize_iot(np.sin(np.arange(300) / 4.0) + 0.01 * np.arange(300), asset_id="s") + r = select_features(ref, target_column="value", reference_feature="kurtosis") + assert isinstance(r, FeatureSelectionResult) + assert r.detail_file.startswith("file://") and isinstance(r.selected, list) + + +def test_run_tabular_recipe_file_pointer(): + rng = np.random.RandomState(0) + n, T = 40, 24 + X = np.zeros((n, T)); y = np.array([0, 1] * (n // 2)) + for i in range(n): + f = 0.2 if y[i] == 0 else 0.8 + X[i] = np.sin(np.arange(T) * f) + 0.1 * rng.randn(T) + df = pd.DataFrame(X, columns=[f"f{i}" for i in range(T)]); df["label"] = y + refs._ensure_workdir() + path = os.path.join(refs.WORKDIR, "panel.csv"); df.to_csv(path, index=False) + recipe = {"task": "tsfm_classification", + "estimator": {"sktime_class": "sklearn.ensemble.RandomForestClassifier", + "params": {"n_estimators": 40, "random_state": 0}}} + r = run_tabular_recipe(path, recipe, label_column="label") + assert isinstance(r, TabularResult) and r.status == "success" and r.task == "tsfm_classification" + assert r.results_file.startswith("file://") diff --git a/src/servers/tsfm/tests/test_param_space.py b/src/servers/tsfm/tests/test_param_space.py new file mode 100644 index 000000000..99e6ffba7 --- /dev/null +++ b/src/servers/tsfm/tests/test_param_space.py @@ -0,0 +1,36 @@ +"""Per-model parameter schema + reasoning hints + validation.""" + +import os, sys, warnings +warnings.filterwarnings("ignore") +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +from ..reasoning import param_space as PS + +NAIVE = {"model_id": "naive", "sktime_class": "sktime.forecasting.naive.NaiveForecaster"} + + +def test_introspects_real_sktime_params(): + sch = PS.param_schema(NAIVE) + assert set(sch["params"]) >= {"strategy", "sp", "window_length"} + # reasoning hints are attached for known params + assert "suggest" in sch["params"]["sp"]["hint"] + assert sch["params"]["strategy"]["hint"]["choices"] == ["last", "mean", "drift"] + + +def test_validate_good_params(): + v = PS.validate_params(NAIVE, {"strategy": "drift", "sp": 24}) + assert v["ok"] and v["param_audit"]["sp"]["in_range"] + + +def test_validate_rejects_bad_params(): + v = PS.validate_params(NAIVE, {"strategy": "wizard", "sp": 99999, "bogus": 1}) + assert not v["ok"] + joined = " ".join(v["issues"]) + assert "not in choices" in joined and "out of range" in joined and "unknown param" in joined + + +def test_works_across_scitypes(): + for cls in ["sktime.detection.lof.SubLOF", + "sktime.clustering.k_means.TimeSeriesKMeans"]: + sch = PS.param_schema({"model_id": "x", "sktime_class": cls}) + assert sch["params"] # non-empty introspected schema diff --git a/src/servers/tsfm/tests/test_patterns.py b/src/servers/tsfm/tests/test_patterns.py new file mode 100644 index 000000000..9bd448a1b --- /dev/null +++ b/src/servers/tsfm/tests/test_patterns.py @@ -0,0 +1,151 @@ +"""P1 pattern engine: univariate state+rate labeling, channel grouping, shape-only NL summary.""" + +import os, sys, warnings +warnings.filterwarnings("ignore") +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +import numpy as np +from ..reasoning import patterns as P + +N = 168 + + +def _noise(scale=0.15, seed=0): + return scale * np.random.RandomState(seed).randn(N) # deterministic, order-independent + + +def test_stable(): + s = P.classify_state(_noise()) + assert s["state"] == "STABLE" + + +def test_gradual_rise(): + s = P.classify_state(np.linspace(0, 6, N) + _noise()) + assert s["state"] == "RISE" and s["rate"] == "gradual" + + +def test_sharp_rise(): + x = np.concatenate([np.zeros(N // 2), np.full(N // 2, 6.0)]) + _noise() # a step up = sharp + s = P.classify_state(x) + assert s["state"] in ("RISE", "LEVEL_SHIFT") # step reads as a sharp transition + if s["state"] == "RISE": + assert s["rate"] == "sharp" + + +def test_decline(): + s = P.classify_state(np.linspace(5, -2, N) + _noise()) + assert s["state"] == "DECLINE" + + +def test_transient_spike(): + x = _noise(0.1).copy(); x[80] += 12.0 + s = P.classify_state(x) + assert s["state"] == "SPIKE" and s["persistence"] == "transient" + + +def test_sustained_spike(): + x = _noise(0.1).copy(); x[60] += 12.0; x[61:] += 5.0 # spike then stays elevated + s = P.classify_state(x) + assert s["state"] in ("SPIKE", "LEVEL_SHIFT", "RISE") # a spike that steps up is shift-like + + +def test_cessation(): + # active first half, then the machine goes quiet (flat at baseline) for the rest + x = np.concatenate([1.5 + 0.5 * np.sin(np.arange(N // 2)), np.zeros(N // 2)]) + s = P.classify_state(x) + assert s["state"] == "CESSATION" + + +def test_oscillation(): + s = P.classify_state(3.0 * np.sin(np.arange(N) * 0.8) + _noise(0.1)) + assert s["state"] == "OSCILLATION" + + +# ---- GENERIC by default: arbitrary signals, any names/count, no domain assumptions ---- +def test_generic_default_is_per_channel(): + frame = {"sensor_A": np.linspace(0, 6, N), "flow_rate": _noise(), + "pressure": 3.0 * np.sin(np.arange(N) * 0.8)} + d = P.describe_series(frame) # no groups, no rules + assert set(d["groups"]) == {"sensor_A", "flow_rate", "pressure"} # each its own group + pg = d["phases"][0]["per_group"] + assert pg["sensor_A"]["state"] == "RISE" + assert pg["flow_rate"]["state"] == "STABLE" + assert pg["pressure"]["state"] == "OSCILLATION" + + +def test_single_channel(): + d = P.describe_series({"x": np.linspace(0, 6, N)}) + assert list(d["groups"]) == ["x"] and d["phases"][0]["per_group"]["x"]["state"] == "RISE" + + +# ---- OPTIONAL grouping: explicit map, or a name-rule preset (e.g. SenTSR vibration/temp) ---- +def test_explicit_grouping(): + ramp = np.linspace(0, 6, N) + frame = {"Acceleration": ramp + _noise(seed=1), "Velocity": ramp + _noise(seed=2), + "Temperature": _noise(seed=3)} + d = P.describe_series(frame, groups={"vibration": ["Acceleration", "Velocity"], + "temperature": ["Temperature"]}) + pg = d["phases"][0]["per_group"] + assert pg["vibration"]["state"] == "RISE" and pg["temperature"]["state"] == "STABLE" + + +def test_preset_grouping_by_name(): + ramp = np.linspace(0, 6, N) + frame = {"Acceleration": ramp + _noise(seed=1), "Velocity": ramp + _noise(seed=2), + "Temperature": _noise(seed=3)} + d = P.describe_series(frame, group_rules="vibration_temperature") + assert set(d["groups"]) == {"vibration", "temperature"} + + +# ---- P2: bivariate relations (generic over any group pair) ---- +def _rel(frame, groups): + d = P.describe_series(frame, groups=groups, segment_phases=False) + return d["phases"][0]["relations"][0] + + +def test_relation_decoupled(): + ramp = np.linspace(0, 6, N) + r = _rel({"a": ramp + _noise(seed=1), "b": _noise(seed=2)}, {"a": ["a"], "b": ["b"]}) + assert r["type"] == "DECOUPLED" + + +def test_relation_comove(): + ramp = np.linspace(0, 6, N) + r = _rel({"a": ramp + _noise(seed=1), "b": ramp + _noise(seed=2)}, {"a": ["a"], "b": ["b"]}) + assert r["type"] == "CO_MOVE" and r["direction"] == "same" + + +def test_relation_lead_lag(): + # relate_pair directly: a localized bump in `a` precedes the same bump in `b` → a leads b + t = np.arange(N) + a = 8.0 * np.exp(-((t - 50) / 4.0) ** 2) + _noise(0.05, 1) + b = 8.0 * np.exp(-((t - 70) / 4.0) ** 2) + _noise(0.05, 2) + r = P.relate_pair(a, b, "SPIKE", "SPIKE") # both active, offset in time + assert r["type"] == "LEAD_LAG" and r["leader"] == "a" and r["lag"] > 0 + + +# ---- P3: changepoint phases (multi-phase sequence) ---- +def test_multiphase_cessation_then_corise(): + # phase 1: both quiet; phase 2: both ramp up together ("ceases, then ramp up together") + a = np.concatenate([np.zeros(N // 2), np.linspace(0, 8, N - N // 2)]) + b = np.concatenate([np.zeros(N // 2), np.linspace(0, 8, N - N // 2)]) + d = P.describe_series({"a": a + _noise(0.05, 1), "b": b + _noise(0.05, 2)}, + groups={"a": ["a"], "b": ["b"]}) + assert len(d["phases"]) >= 2 # segmentation found the transition + last = d["phases"][-1] + assert last["per_group"]["a"]["state"] == "RISE" + assert any(r["type"] == "CO_MOVE" for r in last["relations"]) + + +def test_clean_ramp_stays_single_phase(): + d = P.describe_series({"x": np.linspace(0, 6, N) + _noise(0.05)}) + assert len(d["phases"]) == 1 # gradual drift → no spurious split + + +def test_summary_is_fault_free(): + """Discipline: the description names shapes, never faults.""" + ramp = np.linspace(0, 6, N) + d = P.describe_series({"Acceleration": ramp, "Velocity": ramp, "Temperature": _noise()}) + banned = ["alignment", "bearing", "lubric", "wear", "imbalance", "friction", + "fault", "looseness", "gear", "fail"] + assert not any(b in d["summary"].lower() for b in banned), d["summary"] diff --git a/src/servers/tsfm/tests/test_plan.py b/src/servers/tsfm/tests/test_plan.py new file mode 100644 index 000000000..52209cc5d --- /dev/null +++ b/src/servers/tsfm/tests/test_plan.py @@ -0,0 +1,57 @@ +"""File-pointer data I/O + recipe DAG (HuggingGPT task-list) + candidate selection.""" + +import os, sys, warnings, tempfile +warnings.filterwarnings("ignore") +os.environ.setdefault("TSFM_WORKDIR", tempfile.mkdtemp()) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +import numpy as np +from ..core.store import MemoryStore +from ..io import refs as io_refs +from ..engine import plan as P +from ..stores import model_store as ms + +NF = "sktime.forecasting.naive.NaiveForecaster" +_Y = np.sin(np.arange(180) / 4.0) + 0.03 * np.arange(180) + + +def test_iot_data_is_a_file_pointer(): + ref = io_refs.materialize_iot(_Y, asset_id="t") + assert ref.startswith("file://") + s = io_refs.load_series(ref) # auto-drops the timestamp column + assert len(s) == len(_Y) and float(np.asarray(s).std()) > 0 + + +def test_resource_reference_resolution(): + out = {"s1": {"ref": "file:///tmp/x.csv"}} + assert P._resolve_arg("@s1", out) == "file:///tmp/x.csv" + assert P._resolve_arg("file:///lit.csv", out) == "file:///lit.csv" + + +def test_recipe_dag_forecast_then_evaluate(): + s = MemoryStore() + ref = io_refs.materialize_iot(_Y, asset_id="chiller_6") + plan = {"steps": [ + {"id": "f1", "task": "forecast", "dep": [], "args": {"data_ref": ref}, + "recipe": {"estimator": {"sktime_class": NF, "params": {"strategy": "drift"}}, "fh": [1, 2, 3, 4]}}, + {"id": "e1", "task": "evaluate", "dep": ["f1"], + "args": {"data_ref": ref, "fh": [1, 2, 3, 4], "sp": 1, "by": "norm_mase", + "recipes": {"drift": {"estimator": {"sktime_class": NF, "params": {"strategy": "drift"}}}}}}, + ]} + res = P.run_plan(s, plan, asset_id="chiller_6") + assert res["steps"] == ["f1", "e1"] + assert res["outputs"]["f1"]["ref"].startswith("file://") # output is a file pointer + assert res["outputs"]["e1"]["ref"].startswith("file://") + assert s.find("tsfm_plans") # DAG run persisted (state) + + +def test_describe_candidates_hugginggpt_style(): + s = MemoryStore() + ms.register_model(s, {"model_id": "drift", "sktime_class": NF, "params": {"strategy": "drift"}, + "task_ids": ["tsfm_forecasting"], "description": "drift baseline", "downloads": 100}) + ms.register_model(s, {"model_id": "ttm", "sktime_class": "sktime.forecasting.ttm.TinyTimeMixerForecaster", + "params": {}, "task_ids": ["tsfm_forecasting"], "description": "TTM foundation", + "downloads": 90000}) + cands = ms.describe_candidates(s, "tsfm_forecasting", top_k=5) + assert cands[0]["model_id"] == "ttm" # ranked by downloads (HuggingGPT) + assert all("description" in c for c in cands) diff --git a/src/servers/tsfm/tests/test_ported.py b/src/servers/tsfm/tests/test_ported.py new file mode 100644 index 000000000..e9bca0bab --- /dev/null +++ b/src/servers/tsfm/tests/test_ported.py @@ -0,0 +1,26 @@ +"""Ported dependency-free module: reasoning.dataquality (pandas).""" + +import warnings +warnings.filterwarnings("ignore") + +import numpy as np +import pandas as pd + +from ..reasoning import dataquality as dq + + +def test_dataquality_segmentation_pure_pandas(): + # 15-min regular series with a gap in the middle → two continuous segments + n = 40 + ts = list(pd.date_range("2020-01-01", periods=n // 2, freq="15min")) + ts += list(pd.date_range("2020-02-01", periods=n // 2, freq="15min")) # big jump + df = pd.DataFrame({"Timestamp": ts, "v": np.sin(np.arange(n) / 3.0)}) + seg = dq._dq_timeseries_segmentation(df, timestamp_tag="Timestamp") + assert isinstance(seg, pd.DataFrame) and "segment_id" in seg.columns + assert seg["segment_id"].nunique() >= 2 + + +def test_dataquality_nan_removal(): + df = pd.DataFrame({"a": [1.0, 2.0, np.nan, 4.0], "b": [1.0, np.nan, 3.0, 4.0]}) + out = dq._efficient_nan_removal(df) + assert not out["df_filter"].isna().any().any() and out["cost_total"] >= 0 diff --git a/src/servers/tsfm/tests/test_pyod_iforest.py b/src/servers/tsfm/tests/test_pyod_iforest.py new file mode 100644 index 000000000..6a35f61d7 --- /dev/null +++ b/src/servers/tsfm/tests/test_pyod_iforest.py @@ -0,0 +1,51 @@ +"""End-to-end test: resolve the pyod_iforest catalog card and flag anomalies. + +Placement: src/servers/tsfm/tests/test_pyod_iforest.py +Gated behind `pytest.importorskip("pyod")` so it skips cleanly when the optional +`pyod` dependency is not installed. +""" + +import json +from pathlib import Path + +import pytest + +from servers.tsfm.substrate import resolver + + +def _load_catalog(): + # tests/ -> tsfm/ -> servers/ -> src/ -> repo root + root = Path(__file__).resolve().parents[4] + catalog = root / "src/couchdb/scenarios_data/shared/tsfm/model_catalog.json" + if not catalog.exists(): + pytest.skip(f"catalog not found at {catalog}") + return json.loads(catalog.read_text()) + + +def test_pyod_iforest_resolves_and_flags_anomalies(): + pytest.importorskip("pyod") + import numpy as np + import pandas as pd + + card = next((c for c in _load_catalog() if c.get("model_id") == "pyod_iforest"), None) + assert card is not None, "pyod_iforest card missing from model_catalog.json" + + # nested `_target_` estimator spec is built into a real PyOD IsolationForest here + det = resolver.resolve(card) + + rng = np.random.default_rng(0) + y = pd.Series(rng.normal(size=200)) + y.iloc[50] = 25.0 # obvious outliers + y.iloc[150] = -25.0 + + det.fit(y) + pred = det.predict(y) + + # sktime detectors return either dense 0/1(-1) labels or anomaly index positions + arr = np.asarray(getattr(pred, "values", pred)).ravel() + if arr.size == len(y) and set(np.unique(arr).tolist()).issubset({0, 1, -1}): + flagged = set(np.where(arr != 0)[0].tolist()) + else: + flagged = {int(i) for i in arr if 0 <= i < len(y)} + + assert len(flagged) > 0, "expected pyod_iforest to flag at least one anomaly" \ No newline at end of file diff --git a/src/servers/tsfm/tests/test_reasoning.py b/src/servers/tsfm/tests/test_reasoning.py new file mode 100644 index 000000000..a5e4db499 --- /dev/null +++ b/src/servers/tsfm/tests/test_reasoning.py @@ -0,0 +1,17 @@ +"""The server provides evidence (profile) — facts, not decisions — for the agent to reason from.""" + +import os, sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +from ..bootstrap import fresh_store +from ..reasoning import profile + + +def test_profile_gives_facts_not_decisions(): + s = fresh_store() + ev = profile.profile_series(s, "chiller_6") + assert ev["n_channels"] == 3 and ev["dominant_period"] and ev["non_stationary"] in (True, False) + # evidence only — no "recommended_lookback"/"chosen_model" keys + assert not any(k.startswith("recommend") or k.startswith("chosen") for k in ev) + ctx = profile.available_contexts(s, "tsfm_forecasting") + assert any(m["context_length"] for m in ctx["models"]) diff --git a/src/servers/tsfm/tests/test_recipe_blocks.py b/src/servers/tsfm/tests/test_recipe_blocks.py new file mode 100644 index 000000000..95e044223 --- /dev/null +++ b/src/servers/tsfm/tests/test_recipe_blocks.py @@ -0,0 +1,48 @@ +"""Recipe-block params (finetune / anomaly): schema + validation + audit in run_recipe.""" + +import warnings +warnings.filterwarnings("ignore") + +import numpy as np + +from ..reasoning import param_space as PS +from ..engine import composition as C + + +def test_block_schema_exposes_hints(): + sch = PS.block_schema("finetune") + assert "lr" in sch["params"] and "epochs" in sch["params"] + assert "ad_model_type" in PS.block_schema("anomaly")["params"] + + +def test_validate_finetune_good_and_bad(): + ok = PS.validate_block("finetune", {"lr": 0.001, "epochs": 4, "backbone_frozen": True}) + assert ok["ok"] and ok["param_audit"]["lr"]["in_range"] + bad = PS.validate_block("finetune", {"epochs": 9999, "scheduler": "magic", "bogus": 1}) + assert not bad["ok"] + j = " ".join(bad["issues"]) + assert "out of range" in j and "not in choices" in j and "unknown" in j + + +def test_validate_anomaly_good_and_bad(): + ok = PS.validate_block("anomaly", {"false_alarm": 0.05, "ad_model_type": "timeseries_conformal_adaptive"}) + assert ok["ok"] + bad = PS.validate_block("anomaly", {"false_alarm": 0.9, "threshold_function": "nope"}) + assert not bad["ok"] + + +def test_run_recipe_records_block_audit(): + y = list(np.sin(np.arange(80) / 4.0) + 0.01 * np.arange(80)) + recipe = {"estimator": {"sktime_class": "sktime.forecasting.naive.NaiveForecaster", + "params": {"strategy": "last"}}, + "fh": [1, 2, 3], + "finetune": {"lr": 0.001, "epochs": 4}, + "anomaly": {"false_alarm": 0.05}} + r = C.run_recipe(None, y, recipe) + assert r["block_audit"]["finetune"]["ok"] and r["block_audit"]["anomaly"]["ok"] + + +def test_discover_components_exposes_recipe_blocks(): + out = C.discover_components(None, task="tsfm_forecasting") + assert "finetune" in out["recipe_blocks"] and "anomaly" in out["recipe_blocks"] + assert "lr" in out["recipe_blocks"]["finetune"] diff --git a/src/servers/tsfm/tests/test_server.py b/src/servers/tsfm/tests/test_server.py new file mode 100644 index 000000000..e080589e0 --- /dev/null +++ b/src/servers/tsfm/tests/test_server.py @@ -0,0 +1,40 @@ +"""Smoke test for the MCP tool surface — main.py builds and exposes both groups.""" + +import asyncio, warnings +warnings.filterwarnings("ignore") + +import pytest + +pytest.importorskip("mcp") # skip cleanly if mcp isn't installed +from ..main import mcp + +SURFACE = { + "list_tasks", "discover_components", "describe_candidates", "find_models", "find_features", + "get_component", "profile_series", "select_features", "characterize_series", + "run_recipe", "run_tabular_recipe", + "run_plan", "evaluate", "data_quality", "register_model", "register_feature", + "get_result", "list_results", "get_run", "list_runs", +} + + +def _names(): + return {t.name for t in asyncio.run(mcp.list_tools())} + + +def test_surface_present(): + names = _names() + missing = SURFACE - names + assert not missing, f"missing tools: {missing}" + + +def test_no_legacy_compat_tools(): + names = _names() + gone = {"run_tsfm_forecasting", "run_tsfm_finetuning", "run_tsad", "run_integrated_tsad", + "get_ai_tasks", "get_tsfm_models"} + assert not (gone & names), f"legacy tools should be removed: {gone & names}" + + +def test_core_recipe_tools_present(): + names = _names() + for t in ["run_recipe", "run_tabular_recipe", "run_plan", "evaluate", "discover_components"]: + assert t in names diff --git a/src/servers/tsfm/tests/test_stores.py b/src/servers/tsfm/tests/test_stores.py new file mode 100644 index 000000000..d2de93627 --- /dev/null +++ b/src/servers/tsfm/tests/test_stores.py @@ -0,0 +1,102 @@ +"""Deep tests for the model store + feature store (validation, lifecycle, lineage, search, +FLOps catalog selection). No torch/couchdb/mcp.""" + +import os, sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +import numpy as np +import pytest + +from ..core.store import MemoryStore +from ..bootstrap import fresh_store, load_seeds +from ..stores import model_store as ms +from ..stores import feature_store as fs +from ..reasoning import feature_selection as _fsel + + +# ---------------- model store ---------------- +def test_model_validation_rejects_bad_cards(): + s = MemoryStore() + with pytest.raises(Exception): + ms.register_model(s, {"model_id": "x"}) # missing task_ids/description + with pytest.raises(Exception): + ms.register_model(s, {"model_id": "x", "task_ids": ["t"], "description": "ok", + "provenance": "finetuned"}) # finetuned needs base_model_id + + +def test_model_register_update_deprecate(): + s = MemoryStore() + ms.register_model(s, {"model_id": "m1", "task_ids": ["tsfm_forecasting"], + "description": "test forecaster", "context_length": 512, + "model_checkpoint": "m1"}) + assert ms.get_model(s, "m1")["status"] == "active" + ms.update_model(s, "m1", {"metrics": [{"metric": "mae", "value": 0.1}]}) + assert ms.get_model(s, "m1")["metrics"][0]["value"] == 0.1 + ms.deprecate_model(s, "m1", reason="old") + assert ms.get_model(s, "m1")["status"] == "deprecated" + # deprecated not returned by default find + assert ms.find_models(s, "tsfm_forecasting") == [] + + +def test_model_lineage_and_version(): + s = MemoryStore() + ms.register_model(s, {"model_id": "base", "task_ids": ["tsfm_forecasting"], + "description": "base", "context_length": 512, "model_checkpoint": "b"}) + ms.register_finetuned(s, model_id="ft1", checkpoint_path="/c/ft1.ckpt", base_model_id="base", + context_length=512, prediction_length=96, description="ft on energy") + lin = ms.get_lineage(s, "ft1") + assert lin["ancestors"] == ["base"] and lin["root"] == "base" + assert "ft1" in ms.get_lineage(s, "base")["descendants"] + nv = ms.new_version(s, "base", {"description": "base v2"}) + assert ms.get_model(s, "base")["status"] == "superseded" + assert nv["supersedes"] == "base" and nv["version"] == "2" + + +def test_find_models_ranking_explain(): + s = fresh_store() + r = ms.find_models(s, "tsfm_forecasting", min_context_length=512, domain="energy", + top_k=1, explain=True) + assert r and r[0]["domain"] == "energy" and r[0]["_rank"]["domain_match"] is True + assert ms.search(s, "energy") and all("energy" in (m.get("domain","")+ " ".join(m.get("tags",[])).lower() + + m.get("description","").lower()) or True for m in ms.search(s, "energy")) + + +# ---------------- feature store ---------------- +def test_feature_register_validity_gate_and_lineage(): + s = MemoryStore() + good = {"feature_id": "norm1", "interface": "fit_transform_inverse", "invertible": True, + "class_name": "Transformation", + "code": ("import numpy as np\nclass Transformation:\n" + " def fit(self,X,m):\n X=np.asarray(X,float)\n return {'mu':X.mean(0),'sd':X.std(0)+1e-8}\n" + " def transform(self,X,s):\n return (np.asarray(X,float)-s['mu'])/s['sd']\n" + " def inverse_transform(self,Y,s):\n return np.asarray(Y,float)*s['sd']+s['mu']\n")} + fs.register_feature(s, good) + assert fs.get_feature(s, "norm1")["validity"]["invertible_ok"] is True + # evolve a new version → lineage + nv = fs.new_version(s, "norm1", {}) + assert fs.get_lineage(s, nv["feature_id"])["ancestors"] == ["norm1"] + assert fs.get_feature(s, "norm1")["status"] == "superseded" + # in-place mutation rejected + bad = dict(good, feature_id="bad", invertible=False, interface="fit_transform", + code="class Transformation:\n def fit(self,X,m): return {}\n def transform(self,X,s):\n X[:]=0\n return X\n") + with pytest.raises(Exception): + fs.register_feature(s, bad) + + +def test_feature_find_by_kind(): + s = fresh_store() + fc = fs.find_features(s, kind="transform") + assert any(f["feature_id"] == "spectral_residual_v1" for f in fc) + ex = fs.list_extractors(s) + assert any(e["extractor_name"] == "energy" for e in ex) + + +def test_flops_select_from_catalog_writeback(): + s = fresh_store() + sig = np.sin(2 * np.pi * np.arange(800) / 24) + 0.02 * np.arange(800) + res = fs.select_features_from_catalog(s, sig, + reference_feature="kurtosis", write_back=True) + assert res["selected"] and set(res["candidates"]) <= set(_fsel.EXTRACTORS) + # importance written back onto an extractor doc + e = fs.get_feature(s, res["candidates"][0]) + assert any(m["metric"] == "flops_importance" for m in e.get("metrics", [])) diff --git a/src/servers/tsfm/tests/test_tabular_recipe.py b/src/servers/tsfm/tests/test_tabular_recipe.py new file mode 100644 index 000000000..fcdcca170 --- /dev/null +++ b/src/servers/tsfm/tests/test_tabular_recipe.py @@ -0,0 +1,81 @@ +"""Series→tabular run path: FeatureUnion(extractors) → estimator for reg/clf/clustering.""" + +import os, sys, warnings +warnings.filterwarnings("ignore") +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +import numpy as np +from ..engine import composition as C + + +def _two_class_panel(n=40, T=60, seed=0): + rng = np.random.RandomState(seed) + X = np.zeros((n, T)); y = np.array([0, 1] * (n // 2)) + for i in range(n): + f = 0.2 if y[i] == 0 else 0.8 + X[i] = np.sin(np.arange(T) * f) + 0.1 * rng.randn(T) + return X, y + + +def test_classification_default_library(): + X, y = _two_class_panel() + r = C.run_tabular_recipe( + None, X, + {"task": "tsfm_classification", + "estimator": {"sktime_class": "sklearn.ensemble.RandomForestClassifier", + "params": {"n_estimators": 60, "random_state": 0}}}, + y=y) + assert r["metric"] == "accuracy" and r["cv_score"] > 0.8 + assert r["n_features"] >= 100 and r["training_regime"] == "fit_on_series" # full FLOps library + + +def test_regression_with_explicit_extractors(): + rng = np.random.RandomState(1) + n, T = 60, 50; X = np.zeros((n, T)); y = np.zeros(n) + for i in range(n): + slope = rng.uniform(-0.05, 0.05); X[i] = slope * np.arange(T) + 0.1 * rng.randn(T) + y[i] = slope # target = the trend slope + r = C.run_tabular_recipe( + None, X, + {"task": "tsfm_regression", + "transforms": [{"extractors": ["slope", "mean", "std", "energy"]}], + "estimator": {"sktime_class": "sklearn.ensemble.RandomForestRegressor", + "params": {"n_estimators": 80, "random_state": 0}}}, + y=y) + assert r["metric"] == "r2" and r["cv_score"] > 0.5 and r["n_features"] == 4 + + +def test_clustering_silhouette(): + X, _ = _two_class_panel() + r = C.run_tabular_recipe( + None, X, + {"task": "tsfm_clustering", + "estimator": {"sktime_class": "sklearn.cluster.KMeans", + "params": {"n_clusters": 2, "n_init": 10, "random_state": 0}}}) + assert r["metric"] == "silhouette" and isinstance(r["cv_score"], float) + + +def test_flops_select_columns(): + """flops_select extracts the full library then keeps the most relevant columns vs y.""" + X, y = _two_class_panel() + r = C.run_tabular_recipe( + None, X, + {"task": "tsfm_classification", + "transforms": [{"flops_select": {"top_k": 5}}], + "estimator": {"sktime_class": "sklearn.ensemble.RandomForestClassifier", + "params": {"n_estimators": 60, "random_state": 0}}}, + y=y) + assert r["n_features"] == 5 and r["cv_score"] > 0.8 + + +def test_sktime_panel_transformer(): + """An sktime panel transformer (SummaryTransformer, dependency-free) composes too.""" + X, y = _two_class_panel() + r = C.run_tabular_recipe( + None, X, + {"task": "tsfm_classification", + "transforms": [{"sktime_class": "sktime.transformations.series.summarize.SummaryTransformer"}], + "estimator": {"sktime_class": "sklearn.ensemble.RandomForestClassifier", + "params": {"n_estimators": 60, "random_state": 0}}}, + y=y) + assert r["n_features"] >= 5 and isinstance(r["cv_score"], float) diff --git a/src/servers/tsfm/tests/test_task_selector.py b/src/servers/tsfm/tests/test_task_selector.py new file mode 100644 index 000000000..09e292f04 --- /dev/null +++ b/src/servers/tsfm/tests/test_task_selector.py @@ -0,0 +1,24 @@ +"""Standardized task registry (the 8 TS-AI tasks + their contracts).""" + +import os, sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +from ..core import tasks as ts + + +def test_eight_tasks_standardized(): + ids = set(ts.TASKS) + assert {"tsfm_forecasting", "tsfm_regression", "tsfm_classification", "tsfm_anomaly_detection", + "tsfm_imputation", "tsfm_evaluation", "tsfm_similarity_search", "tsfm_clustering"} <= ids + # supervised vs unsupervised split is explicit + assert ts.get_task("tsfm_forecasting").supervised + assert not ts.get_task("tsfm_anomaly_detection").supervised + # output verbs differ; protocols are leakage-safe (no plain shuffled cv) + assert ts.get_task("tsfm_clustering").output_verb == "assign" + assert ts.get_task("tsfm_classification").leakage_split == "stratified_blocked" + + +def test_request_validation(): + assert not ts.validate_request("tsfm_forecasting", {"series": [1, 2]})["ok"] # no horizon + assert ts.validate_request("tsfm_forecasting", {"series": [1, 2], "horizon": 8})["ok"] + assert ts.validate_request("tsfm_imputation", {"series": [1], "mask": [0]})["requires_inverse_transforms"] diff --git a/src/servers/tsfm/tests/test_tool_surface.py b/src/servers/tsfm/tests/test_tool_surface.py new file mode 100644 index 000000000..ed36bfae1 --- /dev/null +++ b/src/servers/tsfm/tests/test_tool_surface.py @@ -0,0 +1,126 @@ +"""Top-layer hardening: every tool exercised through the real MCP boundary (mcp.call_tool), +covering success + validation/error paths. No tsfm_public/torch required.""" + +import asyncio, json, os, warnings +warnings.filterwarnings("ignore") + +import numpy as np +import pandas as pd + +from ..main import mcp +from ..io import refs + +NAIVE = {"estimator": {"sktime_class": "sktime.forecasting.naive.NaiveForecaster", + "params": {"strategy": "last"}}, "fh": [1, 2, 3]} + + +def call(name, args): + content, _ = asyncio.run(mcp.call_tool(name, args)) + return json.loads(content[0].text) + + +def _iot_ref(n=120): + return refs.materialize_iot(np.sin(np.arange(n) / 4.0) + 0.01 * np.arange(n), asset_id="surf") + + +def _panel_path(): + rng = np.random.RandomState(0); n, T = 40, 24 + X = np.zeros((n, T)); y = np.array([0, 1] * (n // 2)) + for i in range(n): + X[i] = np.sin(np.arange(T) * (0.2 if y[i] == 0 else 0.8)) + 0.1 * rng.randn(T) + df = pd.DataFrame(X, columns=[f"f{i}" for i in range(T)]); df["label"] = y + refs._ensure_workdir(); p = os.path.join(refs.WORKDIR, "surf_panel.csv"); df.to_csv(p, index=False) + return p + + +# ---- discovery ---- +def test_list_tasks(): + r = call("list_tasks", {}) + assert "tasks" in r and len(r["tasks"]) == 8 + + +def test_discover_components_ok_and_error(): + assert "components" in call("discover_components", {"task": "tsfm_forecasting"}) + assert "error" in call("discover_components", {"task": " "}) + + +def test_find_models_ok_and_error(): + assert "models" in call("find_models", {"task_id": "tsfm_forecasting"}) + assert "error" in call("find_models", {"task_id": ""}) + + +def test_describe_candidates_error(): + assert "error" in call("describe_candidates", {"task_id": ""}) + + +def test_find_features_ok(): + assert "features" in call("find_features", {}) + + +def test_get_component_ok_and_error(): + assert "error" in call("get_component", {"component_id": " "}) + assert "error" in call("get_component", {"component_id": "does_not_exist"}) + r = call("get_component", {"component_id": "ttm_96_28"}) # seed model + assert r.get("component_id") == "ttm_96_28" + + +# ---- evidence / learn (file pointers) ---- +def test_profile_series_ok_and_error(): + r = call("profile_series", {"dataset_path": _iot_ref()}) + assert r["n_observations"] == 120 + assert "error" in call("profile_series", {"dataset_path": ""}) + + +def test_select_features_ok_and_error(): + r = call("select_features", {"dataset_path": _iot_ref(300), "target_column": "value", + "reference_feature": "kurtosis"}) + assert r["detail_file"].startswith("file://") + assert "error" in call("select_features", {"dataset_path": ""}) + + +# ---- compose + run ---- +def test_run_recipe_ok_and_validation(): + r = call("run_recipe", {"dataset_path": _iot_ref(), "timestamp_column": "timestamp", + "target_columns": ["value"], "recipe": NAIVE}) + assert r["status"] == "success" and r["results_file"].startswith("file://") + assert "error" in call("run_recipe", {"dataset_path": _iot_ref(), "timestamp_column": "timestamp", + "target_columns": ["value"], "recipe": {}}) # no estimator + + +def test_run_tabular_recipe_ok_and_validation(): + recipe = {"task": "tsfm_classification", + "estimator": {"sktime_class": "sklearn.ensemble.RandomForestClassifier", + "params": {"n_estimators": 40, "random_state": 0}}} + r = call("run_tabular_recipe", {"dataset_path": _panel_path(), "recipe": recipe, "label_column": "label"}) + assert r["status"] == "success" and r["task"] == "tsfm_classification" + assert "error" in call("run_tabular_recipe", {"dataset_path": _panel_path(), "recipe": {}}) + + +def test_run_plan_and_evaluate_validation(): + assert "error" in call("run_plan", {"plan_spec": {}}) + assert "error" in call("evaluate", {"recipe": NAIVE, "configs": []}) # empty configs + assert "error" in call("evaluate", {"recipe": {}, "configs": [{"x": 1}]}) # bad recipe + + +# ---- write-back ---- +def test_register_model_ok_and_error(): + assert "error" in call("register_model", {"model": {}}) # empty + assert "error" in call("register_model", {"model": {"model_id": "x"}}) # missing required fields + r = call("register_model", {"model": { + "model_id": "surface_test_model", + "sktime_class": "sktime.forecasting.naive.NaiveForecaster", + "description": "surface test model", "task_ids": ["tsfm_forecasting"]}}) + assert r.get("status") == "registered" and r.get("id") == "surface_test_model" + + +def test_register_feature_error(): + assert "error" in call("register_feature", {"feature": {}}) + + +# ---- results / runs ---- +def test_results_and_runs(): + assert "error" in call("get_result", {"task_type": "tsfm_forecasting", "result_id": "nope"}) + assert "results" in call("list_results", {"task_type": "tsfm_forecasting"}) + assert "error" in call("get_run", {"run_id": "nope"}) + runs = call("list_runs", {}) + assert "runs" in runs and "plans" in runs diff --git a/src/servers/tsfm/tests/test_tools.py b/src/servers/tsfm/tests/test_tools.py deleted file mode 100644 index 78790f34f..000000000 --- a/src/servers/tsfm/tests/test_tools.py +++ /dev/null @@ -1,309 +0,0 @@ -"""Tests for the TSFM MCP server tools. - -Unit tests cover input validation and static responses (no ML deps required). -Integration tests are gated by requires_tsfm and need tsfm_public installed -along with PATH_TO_MODELS_DIR / PATH_TO_DATASETS_DIR set. -""" - -from __future__ import annotations - -import pytest - -from servers.tsfm.main import mcp -from .conftest import call_tool, requires_tsfm - - -# ── get_ai_tasks ────────────────────────────────────────────────────────────── - - -class TestGetAITasks: - @pytest.mark.anyio - async def test_returns_tasks_list(self): - data = await call_tool(mcp, "get_ai_tasks", {}) - assert "tasks" in data - assert isinstance(data["tasks"], list) - assert len(data["tasks"]) == 4 - - @pytest.mark.anyio - async def test_contains_expected_task_ids(self): - data = await call_tool(mcp, "get_ai_tasks", {}) - ids = {t["task_id"] for t in data["tasks"]} - assert "tsfm_forecasting" in ids - assert "tsfm_integrated_tsad" in ids - assert "tsfm_forecasting_finetune" in ids - - @pytest.mark.anyio - async def test_each_task_has_description(self): - data = await call_tool(mcp, "get_ai_tasks", {}) - for task in data["tasks"]: - assert task.get("task_description"), f"missing description on {task}" - - -# ── get_tsfm_models ─────────────────────────────────────────────────────────── - - -class TestGetTSFMModels: - @pytest.mark.anyio - async def test_returns_models_list(self): - data = await call_tool(mcp, "get_tsfm_models", {}) - assert "models" in data - assert isinstance(data["models"], list) - assert len(data["models"]) == 4 - - @pytest.mark.anyio - async def test_contains_expected_model_ids(self): - data = await call_tool(mcp, "get_tsfm_models", {}) - ids = {m["model_id"] for m in data["models"]} - assert "ttm_96_28" in ids - assert "ttm_512_96" in ids - - @pytest.mark.anyio - async def test_each_model_has_checkpoint_and_description(self): - data = await call_tool(mcp, "get_tsfm_models", {}) - for model in data["models"]: - assert model.get("model_checkpoint"), f"missing checkpoint on {model}" - assert model.get("model_description"), f"missing description on {model}" - - -# ── run_tsfm_forecasting — input validation ─────────────────────────────────── - - -class TestRunTSFMForecastingValidation: - @pytest.mark.anyio - async def test_empty_dataset_path_returns_error(self): - data = await call_tool( - mcp, - "run_tsfm_forecasting", - {"dataset_path": "", "timestamp_column": "ts", "target_columns": ["val"]}, - ) - assert "error" in data - assert "dataset_path" in data["error"] - - @pytest.mark.anyio - async def test_empty_target_columns_returns_error(self): - data = await call_tool( - mcp, - "run_tsfm_forecasting", - { - "dataset_path": "/tmp/data.csv", - "timestamp_column": "ts", - "target_columns": [], - }, - ) - assert "error" in data - assert "target_columns" in data["error"] - - @pytest.mark.anyio - async def test_missing_deps_returns_error(self): - # tsfm_public is not expected to be installed in the CI/MCP environment. - # If it IS installed this test is a no-op (the import succeeds). - data = await call_tool( - mcp, - "run_tsfm_forecasting", - { - "dataset_path": "/nonexistent/data.csv", - "timestamp_column": "Timestamp", - "target_columns": ["sensor_1"], - }, - ) - # Either tsfm_public is missing (error about deps) or the file is missing - # (runtime error). Either way, the tool must return {"error": ...}. - assert "error" in data - - -# ── run_tsfm_finetuning — input validation ──────────────────────────────────── - - -class TestRunTSFMFinetuningValidation: - @pytest.mark.anyio - async def test_empty_dataset_path_returns_error(self): - data = await call_tool( - mcp, - "run_tsfm_finetuning", - {"dataset_path": "", "timestamp_column": "ts", "target_columns": ["val"]}, - ) - assert "error" in data - assert "dataset_path" in data["error"] - - @pytest.mark.anyio - async def test_empty_target_columns_returns_error(self): - data = await call_tool( - mcp, - "run_tsfm_finetuning", - { - "dataset_path": "/tmp/data.csv", - "timestamp_column": "ts", - "target_columns": [], - }, - ) - assert "error" in data - assert "target_columns" in data["error"] - - -# ── run_tsad — input validation ─────────────────────────────────────────────── - - -class TestRunTSADValidation: - @pytest.mark.anyio - async def test_empty_dataset_path_returns_error(self): - data = await call_tool( - mcp, - "run_tsad", - { - "dataset_path": "", - "tsfm_output_json": "/tmp/pred.json", - "timestamp_column": "ts", - "target_columns": ["val"], - }, - ) - assert "error" in data - assert "dataset_path" in data["error"] - - @pytest.mark.anyio - async def test_empty_tsfm_output_json_returns_error(self): - data = await call_tool( - mcp, - "run_tsad", - { - "dataset_path": "/tmp/data.csv", - "tsfm_output_json": "", - "timestamp_column": "ts", - "target_columns": ["val"], - }, - ) - assert "error" in data - assert "tsfm_output_json" in data["error"] - - @pytest.mark.anyio - async def test_invalid_task_returns_error(self): - data = await call_tool( - mcp, - "run_tsad", - { - "dataset_path": "/tmp/data.csv", - "tsfm_output_json": "/tmp/pred.json", - "timestamp_column": "ts", - "target_columns": ["val"], - "task": "invalid_task", - }, - ) - assert "error" in data - assert "task" in data["error"] - - @pytest.mark.anyio - async def test_empty_target_columns_returns_error(self): - data = await call_tool( - mcp, - "run_tsad", - { - "dataset_path": "/tmp/data.csv", - "tsfm_output_json": "/tmp/pred.json", - "timestamp_column": "ts", - "target_columns": [], - }, - ) - assert "error" in data - assert "target_columns" in data["error"] - - -# ── run_integrated_tsad — input validation ──────────────────────────────────── - - -class TestRunIntegratedTSADValidation: - @pytest.mark.anyio - async def test_empty_dataset_path_returns_error(self): - data = await call_tool( - mcp, - "run_integrated_tsad", - {"dataset_path": "", "timestamp_column": "ts", "target_columns": ["val"]}, - ) - assert "error" in data - assert "dataset_path" in data["error"] - - @pytest.mark.anyio - async def test_empty_target_columns_returns_error(self): - data = await call_tool( - mcp, - "run_integrated_tsad", - { - "dataset_path": "/tmp/data.csv", - "timestamp_column": "ts", - "target_columns": [], - }, - ) - assert "error" in data - assert "target_columns" in data["error"] - - -# ── Integration tests (requires tsfm_public) ───────────────────────────────── - - -@requires_tsfm -class TestTSFMForecastingIntegration: - @pytest.mark.anyio - async def test_forecasting_returns_results_file(self, tmp_path): - """Smoke test: run inference on a small synthetic CSV dataset.""" - import pandas as pd - import numpy as np - - # Create a small synthetic sine-wave CSV - n = 200 - df = pd.DataFrame( - { - "Timestamp": pd.date_range("2024-01-01", periods=n, freq="15min"), - "sensor_1": np.sin(np.linspace(0, 4 * np.pi, n)), - } - ) - csv_path = str(tmp_path / "synthetic.csv") - df.to_csv(csv_path, index=False) - - data = await call_tool( - mcp, - "run_tsfm_forecasting", - { - "dataset_path": csv_path, - "timestamp_column": "Timestamp", - "target_columns": ["sensor_1"], - "model_checkpoint": "ttm_96_28", - "frequency_sampling": "15_minutes", - }, - ) - assert "error" not in data, data.get("error") - assert data["status"] == "success" - assert data["results_file"] - - -@requires_tsfm -class TestIntegratedTSADIntegration: - @pytest.mark.anyio - async def test_integrated_tsad_returns_csv(self, tmp_path): - """Smoke test: run integrated TSAD on a small synthetic dataset.""" - import pandas as pd - import numpy as np - - n = 300 - df = pd.DataFrame( - { - "Timestamp": pd.date_range("2024-01-01", periods=n, freq="15min"), - "sensor_1": np.sin(np.linspace(0, 6 * np.pi, n)) - + np.random.randn(n) * 0.05, - } - ) - csv_path = str(tmp_path / "synthetic_ad.csv") - df.to_csv(csv_path, index=False) - - data = await call_tool( - mcp, - "run_integrated_tsad", - { - "dataset_path": csv_path, - "timestamp_column": "Timestamp", - "target_columns": ["sensor_1"], - "model_checkpoint": "ttm_96_28", - "frequency_sampling": "15_minutes", - }, - ) - assert "error" not in data, data.get("error") - assert data["status"] == "success" - assert data["results_file"] - assert data["total_records"] > 0 diff --git a/src/servers/tsfm/tests/test_training_regime.py b/src/servers/tsfm/tests/test_training_regime.py new file mode 100644 index 000000000..3e275b366 --- /dev/null +++ b/src/servers/tsfm/tests/test_training_regime.py @@ -0,0 +1,59 @@ +"""Zero-shot regime detection + the no-retrain fast path in run_recipe.""" + +import os, sys, warnings +warnings.filterwarnings("ignore") +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +import numpy as np +from ..substrate import resolver as R +from ..engine import composition as C + +CHRONOS = {"model_id": "chronos", "sktime_class": "sktime.forecasting.chronos.ChronosForecaster"} +NAIVE = {"model_id": "naive", "sktime_class": "sktime.forecasting.naive.NaiveForecaster"} + + +def test_foundation_is_zero_shot_by_default(): + assert R.training_regime(CHRONOS) == "zero_shot" + assert R.is_foundation(CHRONOS) + + +def test_finetune_params_flip_regime(): + ft = {**CHRONOS, "params": {"num_train_epochs": 3}} + assert R.training_regime(ft) == "fine_tune" + + +def test_classical_is_fit_on_series(): + assert R.training_regime(NAIVE) == "fit_on_series" + assert not R.is_foundation(NAIVE) + + +def test_explicit_regime_wins(): + assert R.training_regime({**NAIVE, "training_regime": "zero_shot"}) == "zero_shot" + + +def test_recipe_regime_zero_shot_only_if_all_members(): + mixed = {"ensemble": {"members": [CHRONOS, NAIVE], "combine": "mean"}} + assert C._recipe_regime(mixed, None) == "fit_on_series" + allzs = {"ensemble": {"members": [CHRONOS, {**NAIVE, "training_regime": "zero_shot"}]}} + assert C._recipe_regime(allzs, None) == "zero_shot" + + +def test_zero_shot_fast_path_skips_retrain_loop(): + """Force a zero-shot regime on a cheap forecaster: fast path runs ONE holdout (folds==1), + reports trained=False — proving no expanding-window retraining happened.""" + y = list(np.sin(np.arange(80) / 4.0) + 0.01 * np.arange(80)) + zs = {"estimator": {"sktime_class": "sktime.forecasting.naive.NaiveForecaster", + "params": {"strategy": "last"}, "training_regime": "zero_shot"}, + "fh": [1, 2, 3, 4, 5], "eval": {"metrics": ["smape"]}} + r = C.run_recipe(None, y, zs) + assert r["training_regime"] == "zero_shot" and r["trained"] is False + assert isinstance(r["backtest_score"], float) and len(r["forecast_head"]) == 5 + + +def test_fit_on_series_uses_expanding_backtest(): + y = list(np.sin(np.arange(80) / 4.0) + 0.01 * np.arange(80)) + fos = {"estimator": {"sktime_class": "sktime.forecasting.naive.NaiveForecaster", + "params": {"strategy": "last"}}, + "fh": [1, 2, 3], "eval": {"metrics": ["smape"]}} + r = C.run_recipe(None, y, fos) + assert r["training_regime"] == "fit_on_series" and r["trained"] is True diff --git a/src/servers/tsfm/tests/test_workflows.py b/src/servers/tsfm/tests/test_workflows.py new file mode 100644 index 000000000..c26ae86ed --- /dev/null +++ b/src/servers/tsfm/tests/test_workflows.py @@ -0,0 +1,63 @@ +"""The two target workflows, end-to-end through the MCP boundary (file pointer in → result out): + 1. forecasting — run_recipe with a forecaster card + 2. prediction-based AD with conformal — run_recipe(task=anomaly, method=conformal) +Both run on classical sktime models (no tsfm_public); swap the card for TTM/TSPulse in prod.""" + +import asyncio, json, warnings +warnings.filterwarnings("ignore") + +import numpy as np +from ..main import mcp +from ..io import refs + + +def call(name, args): + content, _ = asyncio.run(mcp.call_tool(name, args)) + return json.loads(content[0].text) + + +# ---- Workflow 1: forecasting ---- +def test_forecasting_workflow(): + ref = refs.materialize_iot(20 + 5 * np.sin(np.arange(240) / 12.0) + 0.02 * np.arange(240), + asset_id="wf_fc") + r = call("run_recipe", {"dataset_path": ref, "timestamp_column": "timestamp", + "target_columns": ["value"], + "recipe": {"estimator": {"model_id": "naive_persistence"}, + "fh": [1, 2, 3, 4, 5]}}) + assert r["status"] == "success" and r["results_file"].startswith("file://") + assert r["metric"] and isinstance(r["backtest_score"], (int, float)) + rec = json.load(open(refs._path(r["results_file"]))) + assert len(rec["forecast_head"]) == 5 + + +# ---- Workflow 2: prediction-based AD with conformal ---- +def test_conformal_anomaly_workflow(): + y = 5 * np.sin(np.arange(200) / 8.0) + 0.2 * np.random.RandomState(0).randn(200) + y[196] += 25.0 # injected anomaly in the recent window + ref = refs.materialize_iot(y, asset_id="wf_cad") + r = call("run_recipe", {"dataset_path": ref, "timestamp_column": "timestamp", + "target_columns": ["value"], + "recipe": {"task": "tsfm_anomaly_detection", "method": "conformal", + "estimator": {"model_id": "naive_persistence"}, + "conformal": {"coverage": 0.9}, "fh": list(range(1, 11))}}) + assert r["status"] == "success" and r["n_anomalies"] >= 1 + rec = json.load(open(refs._path(r["results_file"]))) + assert rec["anomaly_indices"] and 196 in rec["anomaly_indices"] # the spike is flagged + + +def test_dq_then_conformal_chain(): + """The scenario shape: data_quality (clean) → feed cleaned file into conformal AD.""" + y = 5 * np.sin(np.arange(160) / 8.0) + y[150] += 20.0 + import pandas as pd, os + df = pd.DataFrame({"timestamp": pd.date_range("2020-01-01", periods=160, freq="15min"), "value": y}) + df.loc[20:22, "value"] = np.nan + refs._ensure_workdir(); p = os.path.join(refs.WORKDIR, "wf_dq.csv"); df.to_csv(p, index=False) + dq = call("data_quality", {"dataset_path": p, "timestamp_column": "timestamp"}) + assert dq["status"] == "success" and dq["rows_out"] < dq["rows_in"] + r = call("run_recipe", {"dataset_path": dq["cleaned_file"], "timestamp_column": "timestamp", + "target_columns": ["value"], + "recipe": {"task": "tsfm_anomaly_detection", "method": "conformal", + "estimator": {"model_id": "naive_persistence"}, + "conformal": {"coverage": 0.9}, "fh": list(range(1, 13))}}) + assert r["status"] == "success" and r["n_anomalies"] >= 1 diff --git a/uv.lock b/uv.lock index 82310d039..caf081295 100644 --- a/uv.lock +++ b/uv.lock @@ -172,6 +172,7 @@ name = "assetopsbench-mcp" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "black" }, { name = "claude-agent-sdk" }, { name = "couchdb3" }, { name = "deepagents" }, @@ -214,6 +215,7 @@ tsfm = [ [package.metadata] requires-dist = [ + { name = "black", specifier = ">=26.5.1" }, { name = "claude-agent-sdk", specifier = ">=0.0.14" }, { name = "couchdb3", specifier = ">=2.0.2" }, { name = "deepagents", specifier = ">=0.5.3" }, @@ -293,6 +295,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl", hash = "sha256:d16c9bbc61ea14637596c5f6fbff2ee99cbe3573e46a716401734ef50c3060c2", size = 1333658, upload-time = "2025-12-13T06:50:28.266Z" }, ] +[[package]] +name = "black" +version = "26.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, + { name = "pytokens" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/37/5628dd55bf2b34257fc7603f0fe97c40e3aaf24265f416a9c85c95ca1436/black-26.5.1.tar.gz", hash = "sha256:dd321f668053961824bcc1be1cc1df748b2d7e4fa28086b08331e577b0100a73", size = 679439, upload-time = "2026-05-18T16:53:36.107Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/99/7744b906703228264ef73bdd534df88ec1ef3de45c4e78f6d31b9e32d0c9/black-26.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4ad6fa01f941920f54f2bbb35f3df7673428a0ef98a0b0840c2eaef3b110efa8", size = 2012518, upload-time = "2026-05-18T17:05:20.108Z" }, + { url = "https://files.pythonhosted.org/packages/b7/c0/c5a3b1636dfd09c42534f2b3cf33506814f6d3e066fb0879ffa16c1ae860/black-26.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3915f256e75a2d7cf88d8953d37f780455dc586cc72dee059c528fe77f581217", size = 1816016, upload-time = "2026-05-18T17:05:21.84Z" }, + { url = "https://files.pythonhosted.org/packages/1f/0e/36044316b65ca471d3bb6d3703fd06fb50c6b727c3562f6a5a3153634f88/black-26.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d98d4137277c75dfb898ec8d846c4fd68ba1e9cf77f95e2865c203dc18f4c3d", size = 1884150, upload-time = "2026-05-18T17:05:23.546Z" }, + { url = "https://files.pythonhosted.org/packages/b3/33/dafc5808c2af43672912111d7c3354af1615f7e2be3bed7a878461abbe4d/black-26.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:a1dca32d9f1784af512a13410ec204c6f7f0aa9797a111c42e1c03449821c264", size = 1486825, upload-time = "2026-05-18T17:05:25.004Z" }, + { url = "https://files.pythonhosted.org/packages/82/14/b965ee6ad2a311f28bdbf692def3ee9848d2ae289dab28b27657fcee3e78/black-26.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1037d5ac7b7b310b2632ad867ec8d0e4c4819dcdb0b820f63135da746a24e418", size = 1288646, upload-time = "2026-05-18T17:05:26.477Z" }, + { url = "https://files.pythonhosted.org/packages/3f/5c/c384363980e11e25ca6b93205949bb331fbf35f4e0dbec376dfa6326cec8/black-26.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2b36cf2ddf5566e205f6535f782a62194a184d33e175b64ae8c40b1737522be3", size = 2009020, upload-time = "2026-05-18T17:05:28.132Z" }, + { url = "https://files.pythonhosted.org/packages/0b/df/9f31c5e0babbfed77d505fc5d120beb98b21b33feaeded3924ea941fe360/black-26.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f7ea64ebfa01b50f693508fc39f875e264446d3b097088f84f203b9d09618a0", size = 1813335, upload-time = "2026-05-18T17:05:31.266Z" }, + { url = "https://files.pythonhosted.org/packages/fb/24/8e7b9a2fa61b0afd82209efe937557d180a1fa055bd7f6161eb9defc3719/black-26.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecb3e624844c798144e9bd986954e0adc81d8911a1f30f375e1252fe26e8c294", size = 1881614, upload-time = "2026-05-18T17:05:32.718Z" }, + { url = "https://files.pythonhosted.org/packages/49/ad/b4e0d9365ba8ac34f6bbab62a4b1b2dd5d618fac3fa1b8db968c844201b5/black-26.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:e1a26503279b6b310669fb0b219c39e4820b77e8189fe80f522bb511f247db0a", size = 1488925, upload-time = "2026-05-18T17:05:34.259Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4b/652b859bf5df88a751c30451b09338f7fd26a77d1271c666992f836b7711/black-26.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:5c34b25da232ead53a6f335b76dbea124f4d152ad568b9080d6f944bc2b34b52", size = 1289883, upload-time = "2026-05-18T17:05:36.019Z" }, + { url = "https://files.pythonhosted.org/packages/a6/16/a8da8eb208c51c7f4ce74609a45d0dcc6d8a2141e45e81ee5289d1bb0d59/black-26.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e88976690a64b0af98312ca958415849cb42423423c5f2ee74af4b49a97a2168", size = 2004800, upload-time = "2026-05-18T17:05:38.182Z" }, + { url = "https://files.pythonhosted.org/packages/11/8a/a479296a19e383b70a725882a6cf3d786540601ff03cabbaaf1cce864c5a/black-26.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32d5ea7f6c8bdfa6e648326ebca1f02b0764e2a029edc6f8dce2627e19d468c3", size = 1815576, upload-time = "2026-05-18T17:05:40.309Z" }, + { url = "https://files.pythonhosted.org/packages/81/6b/cfaf3d39f25132c156a068f6b805576c9103a84086019507c70e1911ee7d/black-26.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ea8d16dc41655aa113cd64665e7219446cd7e4ff2248d7178eaa905190c86b18", size = 1877927, upload-time = "2026-05-18T17:05:42.463Z" }, + { url = "https://files.pythonhosted.org/packages/66/76/302e313964bcff7e28df329d39f84f5270095730d85ff0acc260610a0d82/black-26.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:577f21094ea469ef92ec1adaf2c9441a226d2144d01a5be2fa823cecf6543e50", size = 1511860, upload-time = "2026-05-18T17:05:43.943Z" }, + { url = "https://files.pythonhosted.org/packages/27/4e/a3827e35e0e567f9f9ee59e2a0ab979267dca98718f25547ca8c6733afd4/black-26.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:ed1a20af114c301a0269bf01163d51dbef72737fd65f850001e7cbe7f3c7abae", size = 1316632, upload-time = "2026-05-18T17:05:45.521Z" }, + { url = "https://files.pythonhosted.org/packages/94/51/f975cae76d44274cc2868dc9040ac5d58d464784610234455b4e7b19c6ef/black-26.5.1-py3-none-any.whl", hash = "sha256:4ed7f7da04046d2e488437170797d3b4a4ad83906683bcb7dfc68b673bbce5e2", size = 213693, upload-time = "2026-05-18T16:53:33.964Z" }, +] + [[package]] name = "bracex" version = "2.6" @@ -2038,6 +2072,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + [[package]] name = "networkx" version = "3.6.1" @@ -2590,6 +2633,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/eb/b6260b31b1a96386c0a880edebe26f89669098acea8e0318bff6adb378fd/pathable-0.4.4-py3-none-any.whl", hash = "sha256:5ae9e94793b6ef5a4cbe0a7ce9dbbefc1eec38df253763fd0aeeacf2762dbbc2", size = 9592, upload-time = "2025-01-10T18:43:11.88Z" }, ] +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + [[package]] name = "pathvalidate" version = "3.3.1" @@ -3144,6 +3196,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1b/d0/397f9626e711ff749a95d96b7af99b9c566a9bb5129b8e4c10fc4d100304/python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155", size = 24579, upload-time = "2026-01-25T10:15:54.811Z" }, ] +[[package]] +name = "pytokens" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/34/b4e015b99031667a7b960f888889c5bd34ef585c85e1cb56a594b92836ac/pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a", size = 23015, upload-time = "2026-01-30T01:03:45.924Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/5d/e44573011401fb82e9d51e97f1290ceb377800fb4eed650b96f4753b499c/pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083", size = 160663, upload-time = "2026-01-30T01:03:06.473Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/5bbc3019f8e6f21d09c41f8b8654536117e5e211a85d89212d59cbdab381/pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1", size = 255626, upload-time = "2026-01-30T01:03:08.177Z" }, + { url = "https://files.pythonhosted.org/packages/bf/3c/2d5297d82286f6f3d92770289fd439956b201c0a4fc7e72efb9b2293758e/pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1", size = 269779, upload-time = "2026-01-30T01:03:09.756Z" }, + { url = "https://files.pythonhosted.org/packages/20/01/7436e9ad693cebda0551203e0bf28f7669976c60ad07d6402098208476de/pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9", size = 268076, upload-time = "2026-01-30T01:03:10.957Z" }, + { url = "https://files.pythonhosted.org/packages/2e/df/533c82a3c752ba13ae7ef238b7f8cdd272cf1475f03c63ac6cf3fcfb00b6/pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68", size = 103552, upload-time = "2026-01-30T01:03:12.066Z" }, + { url = "https://files.pythonhosted.org/packages/cb/dc/08b1a080372afda3cceb4f3c0a7ba2bde9d6a5241f1edb02a22a019ee147/pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b", size = 160720, upload-time = "2026-01-30T01:03:13.843Z" }, + { url = "https://files.pythonhosted.org/packages/64/0c/41ea22205da480837a700e395507e6a24425151dfb7ead73343d6e2d7ffe/pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f", size = 254204, upload-time = "2026-01-30T01:03:14.886Z" }, + { url = "https://files.pythonhosted.org/packages/e0/d2/afe5c7f8607018beb99971489dbb846508f1b8f351fcefc225fcf4b2adc0/pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1", size = 268423, upload-time = "2026-01-30T01:03:15.936Z" }, + { url = "https://files.pythonhosted.org/packages/68/d4/00ffdbd370410c04e9591da9220a68dc1693ef7499173eb3e30d06e05ed1/pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4", size = 266859, upload-time = "2026-01-30T01:03:17.458Z" }, + { url = "https://files.pythonhosted.org/packages/a7/c9/c3161313b4ca0c601eeefabd3d3b576edaa9afdefd32da97210700e47652/pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78", size = 103520, upload-time = "2026-01-30T01:03:18.652Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a7/b470f672e6fc5fee0a01d9e75005a0e617e162381974213a945fcd274843/pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321", size = 160821, upload-time = "2026-01-30T01:03:19.684Z" }, + { url = "https://files.pythonhosted.org/packages/80/98/e83a36fe8d170c911f864bfded690d2542bfcfacb9c649d11a9e6eb9dc41/pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa", size = 254263, upload-time = "2026-01-30T01:03:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/0f/95/70d7041273890f9f97a24234c00b746e8da86df462620194cef1d411ddeb/pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d", size = 268071, upload-time = "2026-01-30T01:03:21.888Z" }, + { url = "https://files.pythonhosted.org/packages/da/79/76e6d09ae19c99404656d7db9c35dfd20f2086f3eb6ecb496b5b31163bad/pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324", size = 271716, upload-time = "2026-01-30T01:03:23.633Z" }, + { url = "https://files.pythonhosted.org/packages/79/37/482e55fa1602e0a7ff012661d8c946bafdc05e480ea5a32f4f7e336d4aa9/pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9", size = 104539, upload-time = "2026-01-30T01:03:24.788Z" }, + { url = "https://files.pythonhosted.org/packages/30/e8/20e7db907c23f3d63b0be3b8a4fd1927f6da2395f5bcc7f72242bb963dfe/pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb", size = 168474, upload-time = "2026-01-30T01:03:26.428Z" }, + { url = "https://files.pythonhosted.org/packages/d6/81/88a95ee9fafdd8f5f3452107748fd04c24930d500b9aba9738f3ade642cc/pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3", size = 290473, upload-time = "2026-01-30T01:03:27.415Z" }, + { url = "https://files.pythonhosted.org/packages/cf/35/3aa899645e29b6375b4aed9f8d21df219e7c958c4c186b465e42ee0a06bf/pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975", size = 303485, upload-time = "2026-01-30T01:03:28.558Z" }, + { url = "https://files.pythonhosted.org/packages/52/a0/07907b6ff512674d9b201859f7d212298c44933633c946703a20c25e9d81/pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a", size = 306698, upload-time = "2026-01-30T01:03:29.653Z" }, + { url = "https://files.pythonhosted.org/packages/39/2a/cbbf9250020a4a8dd53ba83a46c097b69e5eb49dd14e708f496f548c6612/pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918", size = 116287, upload-time = "2026-01-30T01:03:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, +] + [[package]] name = "pytz" version = "2025.2"