diff --git a/riskqueue/data/validation.py b/riskqueue/data/validation.py index 5c26649..6ab047f 100644 --- a/riskqueue/data/validation.py +++ b/riskqueue/data/validation.py @@ -19,6 +19,21 @@ class DataValidationError(ValueError): """Raised when transaction data cannot be used safely.""" +def _numeric_column(data: pd.DataFrame, column: str) -> pd.Series: + """Return a finite numeric column or raise the public validation error.""" + try: + values = pd.to_numeric(data[column], errors="raise") + raw_values = values.to_numpy() + if np.iscomplexobj(raw_values): + raise ValueError + numeric_values = raw_values.astype(float) + except (TypeError, ValueError, OverflowError): + raise DataValidationError(f"{column} must contain numeric values") from None + if not np.isfinite(numeric_values).all(): + raise DataValidationError(f"{column} must contain finite numeric values") + return values + + def validate_transactions(frame: pd.DataFrame) -> pd.DataFrame: """Validate the minimum PaySim-compatible contract and return a copy.""" missing = REQUIRED_COLUMNS.difference(frame.columns) @@ -29,13 +44,25 @@ def validate_transactions(frame: pd.DataFrame) -> pd.DataFrame: raise DataValidationError("Dataset is empty") if data[list(REQUIRED_COLUMNS)].isnull().any().any(): raise DataValidationError("Required columns contain null values") - if not np.isfinite(data["amount"].astype(float)).all() or (data["amount"] < 0).any(): + + amount = _numeric_column(data, "amount") + if (amount < 0).any(): raise DataValidationError("amount must contain finite non-negative values") + data["amount"] = amount.astype(float) + + step = _numeric_column(data, "step") + if (step < 0).any() or (step % 1 != 0).any(): + raise DataValidationError("step must contain finite non-negative integers") + if (step > np.iinfo(np.int64).max).any(): + raise DataValidationError("step exceeds the supported integer range") + data["step"] = step.astype(np.int64) + + labels = _numeric_column(data, "isFraud") + if not labels.isin([0, 1]).all(): + raise DataValidationError("isFraud must be binary") + data["isFraud"] = labels.astype(np.int64) + invalid_types = set(data["type"]).difference(TRANSACTION_TYPES) if invalid_types: raise DataValidationError(f"Invalid transaction types: {sorted(invalid_types)}") - if not set(data["isFraud"].unique()).issubset({0, 1}): - raise DataValidationError("isFraud must be binary") - if (data["step"] < 0).any(): - raise DataValidationError("step must be non-negative") return data.sort_values(["step"], kind="stable").reset_index(drop=True) diff --git a/tests/test_validation.py b/tests/test_validation.py index d79e2ee..56f762e 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -22,6 +22,36 @@ def test_invalid_amounts(transactions, amount): validate_transactions(transactions) +@pytest.mark.parametrize( + ("column", "value"), + [ + ("amount", "not-a-number"), + ("step", "not-a-step"), + ("step", 1.5), + ("step", np.iinfo(np.int64).max + 1), + ("isFraud", "yes"), + ("isFraud", 2), + ], +) +def test_malformed_numeric_values_raise_validation_error(transactions, column, value): + transactions[column] = transactions[column].astype(object) + transactions.loc[0, column] = value + with pytest.raises(DataValidationError, match=column): + validate_transactions(transactions) + + +def test_numeric_strings_are_normalized(transactions): + transactions["amount"] = transactions.amount.astype(str) + transactions["step"] = transactions.step.astype(str) + transactions["isFraud"] = transactions.isFraud.astype(str) + + result = validate_transactions(transactions) + + assert result.amount.dtype.kind == "f" + assert result.step.dtype == np.dtype("int64") + assert result.isFraud.dtype == np.dtype("int64") + + def test_unknown_type(transactions): transactions.loc[0, "type"] = "WIRE" with pytest.raises(DataValidationError, match="Invalid transaction"):