Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 32 additions & 5 deletions riskqueue/data/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,21 @@ class DataValidationError(ValueError):
"""Raised when transaction data cannot be used safely."""


def _numeric_column(data: pd.DataFrame, column: str) -> pd.Series:
"""Return a finite numeric column or raise the public validation error."""
try:
values = pd.to_numeric(data[column], errors="raise")
raw_values = values.to_numpy()
if np.iscomplexobj(raw_values):
raise ValueError
numeric_values = raw_values.astype(float)
except (TypeError, ValueError, OverflowError):
raise DataValidationError(f"{column} must contain numeric values") from None
if not np.isfinite(numeric_values).all():
raise DataValidationError(f"{column} must contain finite numeric values")
return values


def validate_transactions(frame: pd.DataFrame) -> pd.DataFrame:
"""Validate the minimum PaySim-compatible contract and return a copy."""
missing = REQUIRED_COLUMNS.difference(frame.columns)
Expand All @@ -29,13 +44,25 @@ def validate_transactions(frame: pd.DataFrame) -> pd.DataFrame:
raise DataValidationError("Dataset is empty")
if data[list(REQUIRED_COLUMNS)].isnull().any().any():
raise DataValidationError("Required columns contain null values")
if not np.isfinite(data["amount"].astype(float)).all() or (data["amount"] < 0).any():

amount = _numeric_column(data, "amount")
if (amount < 0).any():
raise DataValidationError("amount must contain finite non-negative values")
data["amount"] = amount.astype(float)

step = _numeric_column(data, "step")
if (step < 0).any() or (step % 1 != 0).any():
raise DataValidationError("step must contain finite non-negative integers")
if (step > np.iinfo(np.int64).max).any():
raise DataValidationError("step exceeds the supported integer range")
data["step"] = step.astype(np.int64)

labels = _numeric_column(data, "isFraud")
if not labels.isin([0, 1]).all():
raise DataValidationError("isFraud must be binary")
data["isFraud"] = labels.astype(np.int64)

invalid_types = set(data["type"]).difference(TRANSACTION_TYPES)
if invalid_types:
raise DataValidationError(f"Invalid transaction types: {sorted(invalid_types)}")
if not set(data["isFraud"].unique()).issubset({0, 1}):
raise DataValidationError("isFraud must be binary")
if (data["step"] < 0).any():
raise DataValidationError("step must be non-negative")
return data.sort_values(["step"], kind="stable").reset_index(drop=True)
30 changes: 30 additions & 0 deletions tests/test_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,36 @@ def test_invalid_amounts(transactions, amount):
validate_transactions(transactions)


@pytest.mark.parametrize(
("column", "value"),
[
("amount", "not-a-number"),
("step", "not-a-step"),
("step", 1.5),
("step", np.iinfo(np.int64).max + 1),
("isFraud", "yes"),
("isFraud", 2),
],
)
def test_malformed_numeric_values_raise_validation_error(transactions, column, value):
transactions[column] = transactions[column].astype(object)
transactions.loc[0, column] = value
with pytest.raises(DataValidationError, match=column):
validate_transactions(transactions)


def test_numeric_strings_are_normalized(transactions):
transactions["amount"] = transactions.amount.astype(str)
transactions["step"] = transactions.step.astype(str)
transactions["isFraud"] = transactions.isFraud.astype(str)

result = validate_transactions(transactions)

assert result.amount.dtype.kind == "f"
assert result.step.dtype == np.dtype("int64")
assert result.isFraud.dtype == np.dtype("int64")


def test_unknown_type(transactions):
transactions.loc[0, "type"] = "WIRE"
with pytest.raises(DataValidationError, match="Invalid transaction"):
Expand Down