From 642bd56b8610c4a93186f0136f91d692f530e867 Mon Sep 17 00:00:00 2001 From: sarahmish Date: Thu, 17 Sep 2026 12:55:52 -0400 Subject: [PATCH 1/8] wip --- sdmetrics/reports/base_report.py | 18 +- sdmetrics/reports/diagnostic_report.py | 8 +- .../multi_table/base_multi_table_report.py | 10 +- .../reports/test_unified_reports.py | 187 +++++++++++++++++- .../test_base_multi_table_report.py | 19 +- .../multi_table/test_diagnostic_report.py | 8 +- tests/unit/reports/test_base_report.py | 60 ++++++ 7 files changed, 292 insertions(+), 18 deletions(-) diff --git a/sdmetrics/reports/base_report.py b/sdmetrics/reports/base_report.py index 5b02a70e..a404494e 100644 --- a/sdmetrics/reports/base_report.py +++ b/sdmetrics/reports/base_report.py @@ -143,7 +143,7 @@ def _get_skipped_properties(self, metadata): """ return set() - def generate(self, real_data, synthetic_data, metadata, verbose=True): + def generate(self, real_data, synthetic_data, metadata, constraints=None, verbose=True): """Generate report. This method generates the report by iterating through each property and calculating @@ -156,10 +156,16 @@ def generate(self, real_data, synthetic_data, metadata, verbose=True): The synthetic data. metadata (dict): The metadata, which contains each column's data type as well as relationships. + constraints (list[dict] or None): + A list of constraints to evaluate their adherence, each represented as a + dictionary with a ``class_name`` and a ``parameters`` key. Defaults to None. verbose (bool): Whether or not to print report summary and progress. """ self._validate(real_data, synthetic_data, metadata) + if 'Constraint Validity' in self._properties: + self._properties['Constraint Validity']._validate_constraints(constraints) + self._skipped_properties = self._get_skipped_properties(metadata) self._original_datetime_columns = self.convert_datetimes( real_data, synthetic_data, metadata @@ -197,8 +203,14 @@ def generate(self, real_data, synthetic_data, metadata, verbose=True): continue + property_arguments = {} + if property_name == 'Constraint Validity': + property_arguments['constraints'] = constraints + if verbose: - num_iterations = int(property_instance._get_num_iterations(metadata)) + num_iterations = int( + property_instance._get_num_iterations(metadata, **property_arguments) + ) progress_bar = tqdm.tqdm( total=num_iterations, file=sys.stdout, bar_format='{desc}|{bar}{r_bar}|' ) @@ -218,7 +230,7 @@ def generate(self, real_data, synthetic_data, metadata, verbose=True): self._properties[property_name].num_rows_subsample = self.num_rows_subsample score = self._properties[property_name].get_score( - real_data, synthetic_data, metadata, progress_bar=progress_bar + real_data, synthetic_data, metadata, progress_bar=progress_bar, **property_arguments ) scores.append(score) if verbose: diff --git a/sdmetrics/reports/diagnostic_report.py b/sdmetrics/reports/diagnostic_report.py index 5077f41e..75e3160d 100644 --- a/sdmetrics/reports/diagnostic_report.py +++ b/sdmetrics/reports/diagnostic_report.py @@ -1,7 +1,12 @@ """Unified diagnostic report.""" from sdmetrics.reports.base_unified_report import BaseUnifiedReport -from sdmetrics.reports.multi_table._properties import DataValidity, RelationshipValidity, Structure +from sdmetrics.reports.multi_table._properties import ( + ConstraintValidity, + DataValidity, + RelationshipValidity, + Structure, +) class DiagnosticReport(BaseUnifiedReport): @@ -17,6 +22,7 @@ def __init__(self): 'Data Validity': DataValidity(), 'Data Structure': Structure(), 'Relationship Validity': RelationshipValidity(), + 'Constraint Validity': ConstraintValidity(), } def _validate_metadata_matches_data(self, real_data, synthetic_data, metadata): diff --git a/sdmetrics/reports/multi_table/base_multi_table_report.py b/sdmetrics/reports/multi_table/base_multi_table_report.py index e6b78a4c..5bd7bfff 100644 --- a/sdmetrics/reports/multi_table/base_multi_table_report.py +++ b/sdmetrics/reports/multi_table/base_multi_table_report.py @@ -84,7 +84,7 @@ def _validate_metadata_matches_data(self, real_data, synthetic_data, metadata): self._validate_relationships(real_data, synthetic_data, metadata) - def generate(self, real_data, synthetic_data, metadata, verbose=True): + def generate(self, real_data, synthetic_data, metadata, constraints=None, verbose=True): """Generate report. This method generates the report by iterating through each property and calculating @@ -97,10 +97,12 @@ def generate(self, real_data, synthetic_data, metadata, verbose=True): The synthetic data. metadata (dict): The metadata, which contains each column's data type as well as relationships. + constraints (list[dict] or None): + A list of constraints to evaluate their adherence. Defaults to None. verbose (bool): Whether or not to print report summary and progress. """ - results = super().generate(real_data, synthetic_data, metadata, verbose) + results = super().generate(real_data, synthetic_data, metadata, constraints, verbose) self.table_names = list(metadata.get('tables', {}).keys()) return results @@ -173,6 +175,10 @@ def get_visualization(self, property_name, table_name=None): if property_name == 'Data Structure': return self._properties[property_name].get_visualization(table_name) + if property_name == 'Constraint Validity': + self._validate_property_generated(property_name) + return self._properties[property_name].get_visualization() + if table_name is None: raise ValueError('Please provide a table name to get a visualization for the property.') diff --git a/tests/integration/reports/test_unified_reports.py b/tests/integration/reports/test_unified_reports.py index 4927915b..5209182d 100644 --- a/tests/integration/reports/test_unified_reports.py +++ b/tests/integration/reports/test_unified_reports.py @@ -2,11 +2,37 @@ import numpy as np import pandas as pd +import pytest from sdmetrics.demos import load_multi_table_demo, load_single_table_demo +from sdmetrics.errors import VisualizationUnavailableError from sdmetrics.reports import DiagnosticReport, QualityReport from tests.utils import assert_report_scores_are_not_nan +SINGLE_TABLE_CONSTRAINTS = [ + { + 'class_name': 'FixedCombinations', + 'parameters': { + 'table_name': 'student_placements', + 'column_names': ['gender', 'degree_type'], + }, + }, +] +MULTI_TABLE_CONSTRAINTS = [ + { + 'class_name': 'FixedCombinations', + 'parameters': {'table_name': 'sessions', 'column_names': ['device', 'os']}, + }, + { + 'class_name': 'Inequality', + 'parameters': { + 'table_name': 'transactions', + 'low_column_name': 'transaction_id', + 'high_column_name': 'amount', + }, + }, +] + def _set_thresholds_zero(report): report.real_correlation_threshold = 0 @@ -128,12 +154,18 @@ def test_unified_diagnostic_report_single_table(): # Run report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, verbose=False) + report.generate(real_data, synthetic_data, metadata, SINGLE_TABLE_CONSTRAINTS, verbose=False) # Assert expected_properties = pd.DataFrame({ - 'Property': ['Data Validity', 'Data Structure'], - 'Score': [1.0, 1.0], + 'Property': ['Data Validity', 'Data Structure', 'Constraint Validity'], + 'Score': [1.0, 1.0, 1.0], + }) + expected_details_constraint_validity = pd.DataFrame({ + 'Constraint': ['FixedCombinations'], + 'Metric': ['ConstraintAdherence'], + 'Parameters': [SINGLE_TABLE_CONSTRAINTS[0]['parameters']], + 'Score': [1.0], }) expected_details_data_validity = pd.DataFrame({ 'Table': ['student_placements'] * 20, @@ -196,6 +228,9 @@ def test_unified_diagnostic_report_single_table(): pd.testing.assert_frame_equal( report.get_details('Data Structure'), expected_details_data_structure ) + pd.testing.assert_frame_equal( + report.get_details('Constraint Validity'), expected_details_constraint_validity + ) assert report.get_score() == 1.0 assert_report_scores_are_not_nan(report) _assert_report_info( @@ -302,18 +337,21 @@ def test_unified_diagnostic_report_single_table_verbose_skips_relationship_valid # Run report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, verbose=True) + report.generate(real_data, synthetic_data, metadata, SINGLE_TABLE_CONSTRAINTS, verbose=True) output = capsys.readouterr().out # Assert expected_lines = [ 'Generating report ...', - '(1/3) Evaluating Data Validity:', + '(1/4) Evaluating Data Validity:', 'Data Validity Score: 100.0%', - '(2/3) Evaluating Data Structure:', + '(2/4) Evaluating Data Structure:', 'Data Structure Score: 100.0%', - '(3/3) Evaluating Relationship Validity: N/A', + '(3/4) Evaluating Relationship Validity: N/A', 'This property does not apply to single-table data.', + '(4/4) Evaluating Constraint Validity:', + '1/1', + 'Constraint Validity Score: 100.0%', 'Overall Score (Average): 100.0%', ] for line in expected_lines: @@ -322,6 +360,7 @@ def test_unified_diagnostic_report_single_table_verbose_skips_relationship_valid assert list(report.get_properties()['Property']) == [ 'Data Validity', 'Data Structure', + 'Constraint Validity', ] assert report.get_score() == 1.0 @@ -398,12 +437,23 @@ def test_unified_diagnostic_report_multi_table(): # Run report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, verbose=False) + report.generate(real_data, synthetic_data, metadata, MULTI_TABLE_CONSTRAINTS, verbose=False) # Assert expected_properties = pd.DataFrame({ - 'Property': ['Data Validity', 'Data Structure', 'Relationship Validity'], - 'Score': [1.0, 1.0, 1.0], + 'Property': [ + 'Data Validity', + 'Data Structure', + 'Relationship Validity', + 'Constraint Validity', + ], + 'Score': [1.0, 1.0, 1.0, 1.0], + }) + expected_details_constraint_validity = pd.DataFrame({ + 'Constraint': ['FixedCombinations', 'Inequality'], + 'Metric': ['ConstraintAdherence', 'ConstraintAdherence'], + 'Parameters': [constraint['parameters'] for constraint in MULTI_TABLE_CONSTRAINTS], + 'Score': [1.0, 1.0], }) expected_details_data_validity = pd.DataFrame({ 'Table': [ @@ -493,6 +543,12 @@ def test_unified_diagnostic_report_multi_table(): pd.testing.assert_frame_equal( report.get_details('Data Validity', 'users'), expected_details_users ) + pd.testing.assert_frame_equal( + report.get_details('Constraint Validity'), expected_details_constraint_validity + ) + pd.testing.assert_frame_equal( + report.get_details('Constraint Validity', 'users'), expected_details_constraint_validity + ) assert report.get_score() == 1.0 assert_report_scores_are_not_nan(report) _assert_report_info( @@ -706,12 +762,123 @@ def test_unified_diagnostic_report_multi_table_with_no_relationships_does_not_sk 'Data Validity', 'Data Structure', 'Relationship Validity', + 'Constraint Validity', ] assert pd.isna( properties.loc[properties['Property'] == 'Relationship Validity', 'Score'].iloc[0] ) +def test_unified_diagnostic_report_multi_table_verbose_with_constraints(capsys): + """Test the Constraint Validity property runs last with one progress step per constraint.""" + # Setup + real_data, synthetic_data, metadata = load_multi_table_demo() + + # Run + report = DiagnosticReport() + report.generate(real_data, synthetic_data, metadata, MULTI_TABLE_CONSTRAINTS, verbose=True) + output = capsys.readouterr().out + + # Assert + expected_lines = [ + 'Generating report ...', + '(1/4) Evaluating Data Validity:', + '(2/4) Evaluating Data Structure:', + '(3/4) Evaluating Relationship Validity:', + '(4/4) Evaluating Constraint Validity:', + '2/2', + 'Constraint Validity Score: 100.0%', + 'Overall Score (Average): 100.0%', + ] + for line in expected_lines: + assert line in output + + assert output.index('Evaluating Relationship Validity') < output.index( + 'Evaluating Constraint Validity' + ) + assert report.get_score() == 1.0 + + +@pytest.mark.parametrize('constraints', [None, []]) +def test_unified_diagnostic_report_without_constraints(constraints): + """Test the Constraint Validity score is NaN and ignored when there are no constraints.""" + # Setup + real_data, synthetic_data, metadata = load_multi_table_demo() + + # Run + report = DiagnosticReport() + report.generate(real_data, synthetic_data, metadata, constraints, verbose=False) + properties = report.get_properties() + details = report.get_details('Constraint Validity') + + # Assert + assert list(properties['Property']) == [ + 'Data Validity', + 'Data Structure', + 'Relationship Validity', + 'Constraint Validity', + ] + assert pd.isna(properties.loc[properties['Property'] == 'Constraint Validity', 'Score'].iloc[0]) + assert details.empty + assert list(details.columns) == ['Constraint', 'Metric', 'Parameters', 'Score'] + assert report.get_score() == 1.0 + + +def test_unified_diagnostic_report_with_invalid_constraint_rows(): + """Test the Constraint Validity score reflects the rows that break the constraints.""" + # Setup + real_data, synthetic_data, metadata = load_multi_table_demo() + synthetic_data['sessions'] = synthetic_data['sessions'].copy() + synthetic_data['sessions']['os'] = 'unknown' + + # Run + report = DiagnosticReport() + report.generate(real_data, synthetic_data, metadata, MULTI_TABLE_CONSTRAINTS, verbose=False) + properties = report.get_properties() + details = report.get_details('Constraint Validity') + + # Assert + constraint_score = properties.loc[ + properties['Property'] == 'Constraint Validity', 'Score' + ].iloc[0] + assert constraint_score == 0.5 + assert details['Score'].tolist() == [0.0, 1.0] + assert report.get_score() == properties['Score'].mean() + + +def test_unified_diagnostic_report_invalid_constraints(): + """Test the report raises an error before generating anything for invalid constraints.""" + # Setup + real_data, synthetic_data, metadata = load_multi_table_demo() + report = DiagnosticReport() + expected_message = "The 'constraints' parameter must be a list of dictionaries" + + # Run and Assert + with pytest.raises(ValueError, match=expected_message): + report.generate(real_data, synthetic_data, metadata, ['invalid'], verbose=False) + + assert report.is_generated is False + + +def test_unified_diagnostic_report_constraint_validity_visualization(): + """Test asking for the Constraint Validity visualization raises a friendly error.""" + # Setup + real_data, synthetic_data, metadata = load_multi_table_demo() + report = DiagnosticReport() + report.generate(real_data, synthetic_data, metadata, MULTI_TABLE_CONSTRAINTS, verbose=False) + expected_message = ( + 'Error: No visualization is available for Constraint Validity. To see the ' + "detailed score breakdowns, use the 'get_details' function." + ) + + # Run and Assert + with pytest.raises(VisualizationUnavailableError, match=expected_message): + report.get_visualization('Constraint Validity') + + with pytest.raises(VisualizationUnavailableError, match=expected_message): + report.get_visualization('Constraint Validity', 'users') + + def test_unified_quality_report_multi_table_with_no_relationships_does_not_skip_properties(): """Test unified quality reports do not skip properties (2 tables, no relationships).""" # Setup diff --git a/tests/unit/reports/multi_table/test_base_multi_table_report.py b/tests/unit/reports/multi_table/test_base_multi_table_report.py index 63a7dbd1..17e924ea 100644 --- a/tests/unit/reports/multi_table/test_base_multi_table_report.py +++ b/tests/unit/reports/multi_table/test_base_multi_table_report.py @@ -226,7 +226,7 @@ def test_generate(self, mock_generate): # Assert assert report.table_names == ['Table_1', 'Table_2'] - mock_generate.assert_called_once_with(real_data, synthetic_data, metadata, True) + mock_generate.assert_called_once_with(real_data, synthetic_data, metadata, None, True) def test__check_table_names(self): """Test the ``_check_table_names`` method.""" @@ -404,6 +404,23 @@ def test_get_visualization_without_table_name(self): with pytest.raises(ValueError, match=expected_error_message): report.get_visualization('Property_1') + def test_get_visualization_for_constraint_validity_property(self): + """Test ``get_visualization`` for the constraint validity property ignores the table.""" + # Setup + report = BaseMultiTableReport() + report._validate_property_generated = Mock() + report._properties = {'Constraint Validity': Mock()} + + # Run + report.get_visualization('Constraint Validity', 'Table_1') + report.get_visualization('Constraint Validity') + + # Assert + report._validate_property_generated.assert_called_with('Constraint Validity') + assert report._validate_property_generated.call_count == 2 + report._properties['Constraint Validity'].get_visualization.assert_called_with() + assert report._properties['Constraint Validity'].get_visualization.call_count == 2 + def test_get_visualization_for_structure_property(self): """Test the ``get_visualization`` method for the structure property.""" # Setup diff --git a/tests/unit/reports/multi_table/test_diagnostic_report.py b/tests/unit/reports/multi_table/test_diagnostic_report.py index 4cc35c87..be483989 100644 --- a/tests/unit/reports/multi_table/test_diagnostic_report.py +++ b/tests/unit/reports/multi_table/test_diagnostic_report.py @@ -4,7 +4,12 @@ import pytest from sdmetrics.reports import DiagnosticReport -from sdmetrics.reports.multi_table._properties import DataValidity, RelationshipValidity, Structure +from sdmetrics.reports.multi_table._properties import ( + ConstraintValidity, + DataValidity, + RelationshipValidity, + Structure, +) class TestDiagnosticReport: @@ -52,3 +57,4 @@ def test___init__unified(self): assert isinstance(report._properties['Data Validity'], DataValidity) assert isinstance(report._properties['Data Structure'], Structure) assert isinstance(report._properties['Relationship Validity'], RelationshipValidity) + assert isinstance(report._properties['Constraint Validity'], ConstraintValidity) diff --git a/tests/unit/reports/test_base_report.py b/tests/unit/reports/test_base_report.py index 19f6b2f0..6dbeb207 100644 --- a/tests/unit/reports/test_base_report.py +++ b/tests/unit/reports/test_base_report.py @@ -509,6 +509,66 @@ def test_generate_verbose_with_skipped_property(self, mock_tqdm, mock_write): ) base_report._print_results.assert_called_once_with(True) + @patch('tqdm.tqdm') + def test_generate_with_constraints(self, mock_tqdm): + """Test ``generate`` only passes the constraints to the Constraint Validity property.""" + # Setup + base_report = BaseReport() + base_report._validate = Mock() + base_report._print_results = Mock() + base_report._properties['Property 1'] = Mock() + base_report._properties['Property 1'].get_score.return_value = 1.0 + base_report._properties['Property 1']._get_num_iterations.return_value = 4 + base_report._properties['Constraint Validity'] = Mock() + base_report._properties['Constraint Validity'].get_score.return_value = 0.5 + base_report._properties['Constraint Validity']._get_num_iterations.return_value = 1 + real_data = pd.DataFrame({'column1': [1, 2, 3]}) + synthetic_data = pd.DataFrame({'column1': [1, 2, 3]}) + metadata = {'columns': {'column1': {'sdtype': 'numerical'}}} + constraints = [{'class_name': 'Range', 'parameters': {}}] + + # Run + base_report.generate(real_data, synthetic_data, metadata, constraints, verbose=True) + + # Assert + base_report._properties['Property 1']._get_num_iterations.assert_called_once_with(metadata) + base_report._properties['Property 1'].get_score.assert_called_once_with( + real_data, synthetic_data, metadata, progress_bar=mock_tqdm.return_value + ) + base_report._properties['Constraint Validity']._get_num_iterations.assert_called_once_with( + metadata, constraints=constraints + ) + base_report._properties['Constraint Validity'].get_score.assert_called_once_with( + real_data, + synthetic_data, + metadata, + progress_bar=mock_tqdm.return_value, + constraints=constraints, + ) + assert base_report._overall_score == 0.75 + + def test_generate_invalid_constraints(self): + """Test ``generate`` validates the constraints before computing any property.""" + # Setup + base_report = BaseReport() + base_report._validate = Mock() + base_report._properties['Property 1'] = Mock() + base_report._properties['Constraint Validity'] = Mock() + base_report._properties[ + 'Constraint Validity' + ]._validate_constraints.side_effect = ValueError('invalid constraints') + real_data = pd.DataFrame({'column1': [1, 2, 3]}) + synthetic_data = pd.DataFrame({'column1': [1, 2, 3]}) + metadata = {'columns': {'column1': {'sdtype': 'numerical'}}} + + # Run and Assert + with pytest.raises(ValueError, match='invalid constraints'): + base_report.generate(real_data, synthetic_data, metadata, 'invalid', verbose=False) + + base_report._properties['Property 1'].get_score.assert_not_called() + base_report._properties['Constraint Validity'].get_score.assert_not_called() + assert base_report.is_generated is False + def test__check_report_generated(self): """Test the ``check_report_generated`` method.""" # Setup From c1fc9afa8a3dd269490adba85ae2eaebb2ba0808 Mon Sep 17 00:00:00 2001 From: sarahmish Date: Thu, 17 Sep 2026 17:28:55 -0400 Subject: [PATCH 2/8] update --- sdmetrics/reports/base_report.py | 14 +- sdmetrics/reports/base_unified_report.py | 11 +- .../_properties/constraint_validity.py | 1 + .../reports/test_unified_reports.py | 149 ++++++++++++------ .../_properties/test_constraint_validity.py | 1 + .../test_base_multi_table_report.py | 25 +-- tests/unit/reports/test_base_report.py | 28 ++-- .../unit/reports/test_base_unified_report.py | 44 +++++- 8 files changed, 186 insertions(+), 87 deletions(-) diff --git a/sdmetrics/reports/base_report.py b/sdmetrics/reports/base_report.py index a404494e..91a342b7 100644 --- a/sdmetrics/reports/base_report.py +++ b/sdmetrics/reports/base_report.py @@ -130,12 +130,14 @@ def _print_results(self, verbose): if verbose: sys.stdout.write(f'Overall Score (Average): {round(self._overall_score * 100, 2)}%\n\n') - def _get_skipped_properties(self, metadata): + def _get_skipped_properties(self, metadata, constraints=None): """Return properties that should not be computed for the metadata. Args: metadata (dict): The metadata dict. + constraints (list[dict] or None): + The constraints given to the report. Defaults to None. Returns: set[str]: @@ -163,10 +165,7 @@ def generate(self, real_data, synthetic_data, metadata, constraints=None, verbos Whether or not to print report summary and progress. """ self._validate(real_data, synthetic_data, metadata) - if 'Constraint Validity' in self._properties: - self._properties['Constraint Validity']._validate_constraints(constraints) - - self._skipped_properties = self._get_skipped_properties(metadata) + self._skipped_properties = self._get_skipped_properties(metadata, constraints) self._original_datetime_columns = self.convert_datetimes( real_data, synthetic_data, metadata ) @@ -196,9 +195,10 @@ def generate(self, real_data, synthetic_data, metadata, constraints=None, verbos property_instance.is_computed = False property_instance.details = pd.DataFrame() if verbose: - sys.stdout.write( - f'{property_description}: N/A\n{self._skipped_property_message}\n\n' + skipped_message = getattr( + property_instance, '_skipped_message', self._skipped_property_message ) + sys.stdout.write(f'{property_description}: N/A\n{skipped_message}\n\n') sys.stdout.flush() continue diff --git a/sdmetrics/reports/base_unified_report.py b/sdmetrics/reports/base_unified_report.py index 50c39f57..fb4a717e 100644 --- a/sdmetrics/reports/base_unified_report.py +++ b/sdmetrics/reports/base_unified_report.py @@ -16,21 +16,26 @@ class BaseUnifiedReport(BaseMultiTableReport): }) _skipped_property_message = 'This property does not apply to single-table data.' - def _get_skipped_properties(self, metadata): + def _get_skipped_properties(self, metadata, constraints=None): """Return properties unavailable to single-table data. Args: metadata (dict): The metadata dict. + constraints (list[dict] or None): + The constraints given to the report. Defaults to None. Returns: set[str]: Names of properties to skip. """ + skipped_properties = set() if len(metadata.get('tables', {})) == 1: - return self._SINGLE_TABLE_SKIPPED_PROPERTIES + skipped_properties.update(self._SINGLE_TABLE_SKIPPED_PROPERTIES) + if not constraints: + skipped_properties.add('Constraint Validity') - return super()._get_skipped_properties(metadata) + return skipped_properties or super()._get_skipped_properties(metadata, constraints) def _validate_data_format(self, real_data, synthetic_data): """Validate that the real and synthetic data have compatible formats. diff --git a/sdmetrics/reports/multi_table/_properties/constraint_validity.py b/sdmetrics/reports/multi_table/_properties/constraint_validity.py index 8bb5ed02..6ec343cf 100644 --- a/sdmetrics/reports/multi_table/_properties/constraint_validity.py +++ b/sdmetrics/reports/multi_table/_properties/constraint_validity.py @@ -17,6 +17,7 @@ class ConstraintValidity(BaseMultiTableProperty): """ _num_iteration_case = 'constraint' + _skipped_message = 'No constraints were provided.' def _get_num_iterations(self, metadata, constraints=None): """Get the number of iterations for the property, which is one per constraint.""" diff --git a/tests/integration/reports/test_unified_reports.py b/tests/integration/reports/test_unified_reports.py index 5209182d..a8454180 100644 --- a/tests/integration/reports/test_unified_reports.py +++ b/tests/integration/reports/test_unified_reports.py @@ -9,30 +9,6 @@ from sdmetrics.reports import DiagnosticReport, QualityReport from tests.utils import assert_report_scores_are_not_nan -SINGLE_TABLE_CONSTRAINTS = [ - { - 'class_name': 'FixedCombinations', - 'parameters': { - 'table_name': 'student_placements', - 'column_names': ['gender', 'degree_type'], - }, - }, -] -MULTI_TABLE_CONSTRAINTS = [ - { - 'class_name': 'FixedCombinations', - 'parameters': {'table_name': 'sessions', 'column_names': ['device', 'os']}, - }, - { - 'class_name': 'Inequality', - 'parameters': { - 'table_name': 'transactions', - 'low_column_name': 'transaction_id', - 'high_column_name': 'amount', - }, - }, -] - def _set_thresholds_zero(report): report.real_correlation_threshold = 0 @@ -150,11 +126,18 @@ def _load_single_table_quality_report_data(): def test_unified_diagnostic_report_single_table(): # Setup + fixed_combination = { + 'class_name': 'FixedCombinations', + 'parameters': { + 'table_name': 'student_placements', + 'column_names': ['gender', 'degree_type'], + }, + } real_data, synthetic_data, metadata = load_single_table_demo() # Run report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, SINGLE_TABLE_CONSTRAINTS, verbose=False) + report.generate(real_data, synthetic_data, metadata, [fixed_combination], verbose=False) # Assert expected_properties = pd.DataFrame({ @@ -164,7 +147,7 @@ def test_unified_diagnostic_report_single_table(): expected_details_constraint_validity = pd.DataFrame({ 'Constraint': ['FixedCombinations'], 'Metric': ['ConstraintAdherence'], - 'Parameters': [SINGLE_TABLE_CONSTRAINTS[0]['parameters']], + 'Parameters': [fixed_combination['parameters']], 'Score': [1.0], }) expected_details_data_validity = pd.DataFrame({ @@ -337,7 +320,7 @@ def test_unified_diagnostic_report_single_table_verbose_skips_relationship_valid # Run report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, SINGLE_TABLE_CONSTRAINTS, verbose=True) + report.generate(real_data, synthetic_data, metadata, verbose=True) output = capsys.readouterr().out # Assert @@ -349,9 +332,8 @@ def test_unified_diagnostic_report_single_table_verbose_skips_relationship_valid 'Data Structure Score: 100.0%', '(3/4) Evaluating Relationship Validity: N/A', 'This property does not apply to single-table data.', - '(4/4) Evaluating Constraint Validity:', - '1/1', - 'Constraint Validity Score: 100.0%', + '(4/4) Evaluating Constraint Validity: N/A', + 'No constraints were provided.', 'Overall Score (Average): 100.0%', ] for line in expected_lines: @@ -365,6 +347,42 @@ def test_unified_diagnostic_report_single_table_verbose_skips_relationship_valid assert report.get_score() == 1.0 +def test_unified_diagnostic_report_single_table_verbose_with_constraints(capsys): + """Test unified diagnostic report prints the Constraint Validity progress for single-table.""" + # Setup + fixed_combination = { + 'class_name': 'FixedCombinations', + 'parameters': { + 'table_name': 'student_placements', + 'column_names': ['gender', 'degree_type'], + }, + } + real_data, synthetic_data, metadata = load_single_table_demo() + + # Run + report = DiagnosticReport() + report.generate(real_data, synthetic_data, metadata, [fixed_combination], verbose=True) + output = capsys.readouterr().out + + # Assert + expected_lines = [ + 'Generating report ...', + '(1/4) Evaluating Data Validity:', + 'Data Validity Score: 100.0%', + '(2/4) Evaluating Data Structure:', + 'Data Structure Score: 100.0%', + '(3/4) Evaluating Relationship Validity: N/A', + 'This property does not apply to single-table data.', + '(4/4) Evaluating Constraint Validity:', + 'Constraint Validity Score: 100.0%', + 'Overall Score (Average): 100.0%', + ] + for line in expected_lines: + assert line in output + + assert report.get_score() == 1.0 + + def test_diagnostic_report_with_ordinal_sdtype(): """Test diagnostic report handles ordinal sdtype correctly. @@ -433,11 +451,25 @@ def test_unified_quality_report_single_table_verbose_skips_relationship_properti def test_unified_diagnostic_report_multi_table(): # Setup + multiple_constraints = [ + { + 'class_name': 'FixedCombinations', + 'parameters': {'table_name': 'sessions', 'column_names': ['device', 'os']}, + }, + { + 'class_name': 'Inequality', + 'parameters': { + 'table_name': 'transactions', + 'low_column_name': 'transaction_id', + 'high_column_name': 'amount', + }, + }, + ] real_data, synthetic_data, metadata = load_multi_table_demo() # Run report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, MULTI_TABLE_CONSTRAINTS, verbose=False) + report.generate(real_data, synthetic_data, metadata, multiple_constraints, verbose=False) # Assert expected_properties = pd.DataFrame({ @@ -452,7 +484,7 @@ def test_unified_diagnostic_report_multi_table(): expected_details_constraint_validity = pd.DataFrame({ 'Constraint': ['FixedCombinations', 'Inequality'], 'Metric': ['ConstraintAdherence', 'ConstraintAdherence'], - 'Parameters': [constraint['parameters'] for constraint in MULTI_TABLE_CONSTRAINTS], + 'Parameters': [constraint['parameters'] for constraint in multiple_constraints], 'Score': [1.0, 1.0], }) expected_details_data_validity = pd.DataFrame({ @@ -772,21 +804,37 @@ def test_unified_diagnostic_report_multi_table_with_no_relationships_does_not_sk def test_unified_diagnostic_report_multi_table_verbose_with_constraints(capsys): """Test the Constraint Validity property runs last with one progress step per constraint.""" # Setup + multiple_constraints = [ + { + 'class_name': 'FixedCombinations', + 'parameters': {'table_name': 'sessions', 'column_names': ['device', 'os']}, + }, + { + 'class_name': 'Inequality', + 'parameters': { + 'table_name': 'transactions', + 'low_column_name': 'transaction_id', + 'high_column_name': 'amount', + }, + }, + ] real_data, synthetic_data, metadata = load_multi_table_demo() # Run report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, MULTI_TABLE_CONSTRAINTS, verbose=True) + report.generate(real_data, synthetic_data, metadata, multiple_constraints, verbose=True) output = capsys.readouterr().out # Assert expected_lines = [ 'Generating report ...', '(1/4) Evaluating Data Validity:', + 'Data Validity Score: 100.0%', '(2/4) Evaluating Data Structure:', + 'Data Structure Score: 100.0%', '(3/4) Evaluating Relationship Validity:', + 'Relationship Validity Score: 100.0%', '(4/4) Evaluating Constraint Validity:', - '2/2', 'Constraint Validity Score: 100.0%', 'Overall Score (Average): 100.0%', ] @@ -820,20 +868,33 @@ def test_unified_diagnostic_report_without_constraints(constraints): ] assert pd.isna(properties.loc[properties['Property'] == 'Constraint Validity', 'Score'].iloc[0]) assert details.empty - assert list(details.columns) == ['Constraint', 'Metric', 'Parameters', 'Score'] assert report.get_score() == 1.0 def test_unified_diagnostic_report_with_invalid_constraint_rows(): """Test the Constraint Validity score reflects the rows that break the constraints.""" # Setup + multiple_constraints = [ + { + 'class_name': 'FixedCombinations', + 'parameters': {'table_name': 'sessions', 'column_names': ['device', 'os']}, + }, + { + 'class_name': 'Inequality', + 'parameters': { + 'table_name': 'transactions', + 'low_column_name': 'transaction_id', + 'high_column_name': 'amount', + }, + }, + ] real_data, synthetic_data, metadata = load_multi_table_demo() synthetic_data['sessions'] = synthetic_data['sessions'].copy() synthetic_data['sessions']['os'] = 'unknown' # Run report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, MULTI_TABLE_CONSTRAINTS, verbose=False) + report.generate(real_data, synthetic_data, metadata, multiple_constraints, verbose=False) properties = report.get_properties() details = report.get_details('Constraint Validity') @@ -846,26 +907,12 @@ def test_unified_diagnostic_report_with_invalid_constraint_rows(): assert report.get_score() == properties['Score'].mean() -def test_unified_diagnostic_report_invalid_constraints(): - """Test the report raises an error before generating anything for invalid constraints.""" - # Setup - real_data, synthetic_data, metadata = load_multi_table_demo() - report = DiagnosticReport() - expected_message = "The 'constraints' parameter must be a list of dictionaries" - - # Run and Assert - with pytest.raises(ValueError, match=expected_message): - report.generate(real_data, synthetic_data, metadata, ['invalid'], verbose=False) - - assert report.is_generated is False - - def test_unified_diagnostic_report_constraint_validity_visualization(): """Test asking for the Constraint Validity visualization raises a friendly error.""" # Setup real_data, synthetic_data, metadata = load_multi_table_demo() report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, MULTI_TABLE_CONSTRAINTS, verbose=False) + report.generate(real_data, synthetic_data, metadata, verbose=False) expected_message = ( 'Error: No visualization is available for Constraint Validity. To see the ' "detailed score breakdowns, use the 'get_details' function." diff --git a/tests/unit/reports/multi_table/_properties/test_constraint_validity.py b/tests/unit/reports/multi_table/_properties/test_constraint_validity.py index c4417724..dd636128 100644 --- a/tests/unit/reports/multi_table/_properties/test_constraint_validity.py +++ b/tests/unit/reports/multi_table/_properties/test_constraint_validity.py @@ -80,6 +80,7 @@ def test__init__(self): # Assert assert isinstance(constraint_validity, BaseMultiTableProperty) assert constraint_validity._num_iteration_case == 'constraint' + assert constraint_validity._skipped_message == 'No constraints were provided.' assert constraint_validity.is_computed is False assert constraint_validity.details.empty diff --git a/tests/unit/reports/multi_table/test_base_multi_table_report.py b/tests/unit/reports/multi_table/test_base_multi_table_report.py index 17e924ea..0be22aae 100644 --- a/tests/unit/reports/multi_table/test_base_multi_table_report.py +++ b/tests/unit/reports/multi_table/test_base_multi_table_report.py @@ -7,6 +7,8 @@ import pytest from sdmetrics.demos import load_demo +from sdmetrics.errors import VisualizationUnavailableError +from sdmetrics.reports.multi_table._properties import ConstraintValidity from sdmetrics.reports.multi_table.base_multi_table_report import BaseMultiTableReport from sdmetrics.reports.utils import DEFAULT_NUM_ROWS_SUBSAMPLE @@ -404,22 +406,23 @@ def test_get_visualization_without_table_name(self): with pytest.raises(ValueError, match=expected_error_message): report.get_visualization('Property_1') - def test_get_visualization_for_constraint_validity_property(self): - """Test ``get_visualization`` for the constraint validity property ignores the table.""" + @pytest.mark.parametrize('table_name', [None, 'Table_1']) + def test_get_visualization_for_constraint_validity_property(self, table_name): + """Test ``get_visualization`` raises the property error, with or without a table.""" # Setup report = BaseMultiTableReport() report._validate_property_generated = Mock() - report._properties = {'Constraint Validity': Mock()} + report._properties = {'Constraint Validity': ConstraintValidity()} + expected_message = ( + 'Error: No visualization is available for Constraint Validity. To see the ' + "detailed score breakdowns, use the 'get_details' function." + ) - # Run - report.get_visualization('Constraint Validity', 'Table_1') - report.get_visualization('Constraint Validity') + # Run and Assert + with pytest.raises(VisualizationUnavailableError, match=expected_message): + report.get_visualization('Constraint Validity', table_name) - # Assert - report._validate_property_generated.assert_called_with('Constraint Validity') - assert report._validate_property_generated.call_count == 2 - report._properties['Constraint Validity'].get_visualization.assert_called_with() - assert report._properties['Constraint Validity'].get_visualization.call_count == 2 + report._validate_property_generated.assert_called_once_with('Constraint Validity') def test_get_visualization_for_structure_property(self): """Test the ``get_visualization`` method for the structure property.""" diff --git a/tests/unit/reports/test_base_report.py b/tests/unit/reports/test_base_report.py index 6dbeb207..4002b7ef 100644 --- a/tests/unit/reports/test_base_report.py +++ b/tests/unit/reports/test_base_report.py @@ -467,6 +467,7 @@ def test_generate_verbose_with_skipped_property(self, mock_tqdm, mock_write): base_report._skipped_properties = {'Old Property'} base_report._get_skipped_properties = Mock(return_value={'Property 1'}) base_report._properties['Property 1'] = Mock() + del base_report._properties['Property 1']._skipped_message base_report._properties['Property 1'].details = pd.DataFrame({'old': [1]}) base_report._properties['Property 1'].is_computed = True base_report._properties['Property 1']._compute_average.return_value = float('nan') @@ -488,7 +489,7 @@ def test_generate_verbose_with_skipped_property(self, mock_tqdm, mock_write): assert base_report._properties['Property 1'].details.empty assert not base_report._properties['Property 1'].is_computed assert base_report._skipped_properties == {'Property 1'} - base_report._get_skipped_properties.assert_called_once_with(metadata) + base_report._get_skipped_properties.assert_called_once_with(metadata, None) base_report._properties['Property 2'].get_score.assert_called_once_with( real_data, synthetic_data, metadata, progress_bar=mock_tqdm.return_value ) @@ -547,27 +548,28 @@ def test_generate_with_constraints(self, mock_tqdm): ) assert base_report._overall_score == 0.75 - def test_generate_invalid_constraints(self): - """Test ``generate`` validates the constraints before computing any property.""" + @patch('sys.stdout.write') + @patch('tqdm.tqdm') + def test_generate_verbose_with_skipped_property_message(self, mock_tqdm, mock_write): + """Test a skipped property prints its own message when it defines one.""" # Setup base_report = BaseReport() base_report._validate = Mock() - base_report._properties['Property 1'] = Mock() - base_report._properties['Constraint Validity'] = Mock() - base_report._properties[ - 'Constraint Validity' - ]._validate_constraints.side_effect = ValueError('invalid constraints') + base_report.convert_datetimes = Mock() + base_report._print_results = Mock() + base_report._get_skipped_properties = Mock(return_value={'Property 1'}) + base_report._properties['Property 1'] = Mock(_skipped_message='Property message.') + base_report._properties['Property 1']._compute_average.return_value = float('nan') real_data = pd.DataFrame({'column1': [1, 2, 3]}) synthetic_data = pd.DataFrame({'column1': [1, 2, 3]}) metadata = {'columns': {'column1': {'sdtype': 'numerical'}}} - # Run and Assert - with pytest.raises(ValueError, match='invalid constraints'): - base_report.generate(real_data, synthetic_data, metadata, 'invalid', verbose=False) + # Run + base_report.generate(real_data, synthetic_data, metadata, verbose=True) + # Assert + mock_write.assert_any_call('(1/1) Evaluating Property 1: N/A\nProperty message.\n\n') base_report._properties['Property 1'].get_score.assert_not_called() - base_report._properties['Constraint Validity'].get_score.assert_not_called() - assert base_report.is_generated is False def test__check_report_generated(self): """Test the ``check_report_generated`` method.""" diff --git a/tests/unit/reports/test_base_unified_report.py b/tests/unit/reports/test_base_unified_report.py index a6077037..3926a646 100644 --- a/tests/unit/reports/test_base_unified_report.py +++ b/tests/unit/reports/test_base_unified_report.py @@ -181,8 +181,10 @@ def test__get_skipped_properties_single_table(self): 'relationships': [], } + constraints = [{'class_name': 'Range', 'parameters': {}}] + # Run - skipped_properties = base_report._get_skipped_properties(metadata) + skipped_properties = base_report._get_skipped_properties(metadata, constraints) # Assert assert skipped_properties == {'Relationship Validity', 'Cardinality', 'Intertable Trends'} @@ -207,12 +209,50 @@ def test__get_skipped_properties_multi_table(self): 'relationships': [], } + constraints = [{'class_name': 'Range', 'parameters': {}}] + # Run - skipped_properties = base_report._get_skipped_properties(metadata) + skipped_properties = base_report._get_skipped_properties(metadata, constraints) # Assert assert skipped_properties == set() + @pytest.mark.parametrize('constraints', [None, []]) + def test__get_skipped_properties_without_constraints(self, constraints): + """Test Constraint Validity is skipped when no constraints are given.""" + # Setup + base_report = BaseUnifiedReport() + metadata = { + 'tables': { + 'table1': {'columns': {'column1': {'sdtype': 'numerical'}}}, + 'table2': {'columns': {'column2': {'sdtype': 'numerical'}}}, + }, + 'relationships': [], + } + + # Run + skipped_properties = base_report._get_skipped_properties(metadata, constraints) + + # Assert + assert skipped_properties == {'Constraint Validity'} + + def test__get_skipped_properties_single_table_without_constraints(self): + """Test the single-table skips are combined with the Constraint Validity skip.""" + # Setup + base_report = BaseUnifiedReport() + metadata = {'tables': {'table1': {'columns': {'column1': {'sdtype': 'numerical'}}}}} + + # Run + skipped_properties = base_report._get_skipped_properties(metadata) + + # Assert + assert skipped_properties == { + 'Relationship Validity', + 'Cardinality', + 'Intertable Trends', + 'Constraint Validity', + } + @patch('sdmetrics.reports.base_unified_report._validate_unified_metadata') def test__validate_metadata_error(self, mock__validate_metadata): """Test the ``_validate`` method when metadata validation fails.""" From c9f3090a0c80cd07365ca71d336e9721e11a3e1f Mon Sep 17 00:00:00 2001 From: sarahmish Date: Thu, 17 Sep 2026 17:54:35 -0400 Subject: [PATCH 3/8] clean up --- sdmetrics/reports/base_unified_report.py | 4 ++-- sdmetrics/reports/multi_table/base_multi_table_report.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sdmetrics/reports/base_unified_report.py b/sdmetrics/reports/base_unified_report.py index fb4a717e..02b0acd1 100644 --- a/sdmetrics/reports/base_unified_report.py +++ b/sdmetrics/reports/base_unified_report.py @@ -29,13 +29,13 @@ def _get_skipped_properties(self, metadata, constraints=None): set[str]: Names of properties to skip. """ - skipped_properties = set() + skipped_properties = super()._get_skipped_properties(metadata, constraints) if len(metadata.get('tables', {})) == 1: skipped_properties.update(self._SINGLE_TABLE_SKIPPED_PROPERTIES) if not constraints: skipped_properties.add('Constraint Validity') - return skipped_properties or super()._get_skipped_properties(metadata, constraints) + return skipped_properties def _validate_data_format(self, real_data, synthetic_data): """Validate that the real and synthetic data have compatible formats. diff --git a/sdmetrics/reports/multi_table/base_multi_table_report.py b/sdmetrics/reports/multi_table/base_multi_table_report.py index 5bd7bfff..a071012a 100644 --- a/sdmetrics/reports/multi_table/base_multi_table_report.py +++ b/sdmetrics/reports/multi_table/base_multi_table_report.py @@ -176,7 +176,6 @@ def get_visualization(self, property_name, table_name=None): return self._properties[property_name].get_visualization(table_name) if property_name == 'Constraint Validity': - self._validate_property_generated(property_name) return self._properties[property_name].get_visualization() if table_name is None: From 544c2a19a9b9d616b3df8dccc0f0ee6f7e52af62 Mon Sep 17 00:00:00 2001 From: sarahmish Date: Thu, 17 Sep 2026 19:16:00 -0400 Subject: [PATCH 4/8] update test --- .../reports/test_unified_reports.py | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/tests/integration/reports/test_unified_reports.py b/tests/integration/reports/test_unified_reports.py index a8454180..5e9edcbe 100644 --- a/tests/integration/reports/test_unified_reports.py +++ b/tests/integration/reports/test_unified_reports.py @@ -841,9 +841,6 @@ def test_unified_diagnostic_report_multi_table_verbose_with_constraints(capsys): for line in expected_lines: assert line in output - assert output.index('Evaluating Relationship Validity') < output.index( - 'Evaluating Constraint Validity' - ) assert report.get_score() == 1.0 @@ -871,42 +868,6 @@ def test_unified_diagnostic_report_without_constraints(constraints): assert report.get_score() == 1.0 -def test_unified_diagnostic_report_with_invalid_constraint_rows(): - """Test the Constraint Validity score reflects the rows that break the constraints.""" - # Setup - multiple_constraints = [ - { - 'class_name': 'FixedCombinations', - 'parameters': {'table_name': 'sessions', 'column_names': ['device', 'os']}, - }, - { - 'class_name': 'Inequality', - 'parameters': { - 'table_name': 'transactions', - 'low_column_name': 'transaction_id', - 'high_column_name': 'amount', - }, - }, - ] - real_data, synthetic_data, metadata = load_multi_table_demo() - synthetic_data['sessions'] = synthetic_data['sessions'].copy() - synthetic_data['sessions']['os'] = 'unknown' - - # Run - report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, multiple_constraints, verbose=False) - properties = report.get_properties() - details = report.get_details('Constraint Validity') - - # Assert - constraint_score = properties.loc[ - properties['Property'] == 'Constraint Validity', 'Score' - ].iloc[0] - assert constraint_score == 0.5 - assert details['Score'].tolist() == [0.0, 1.0] - assert report.get_score() == properties['Score'].mean() - - def test_unified_diagnostic_report_constraint_validity_visualization(): """Test asking for the Constraint Validity visualization raises a friendly error.""" # Setup From 74dde93fe6f110bba5fd7c4f726d1f5ae04a3bc0 Mon Sep 17 00:00:00 2001 From: sarahmish Date: Thu, 17 Sep 2026 19:34:29 -0400 Subject: [PATCH 5/8] fix test --- tests/unit/reports/multi_table/test_base_multi_table_report.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/unit/reports/multi_table/test_base_multi_table_report.py b/tests/unit/reports/multi_table/test_base_multi_table_report.py index 0be22aae..db52179a 100644 --- a/tests/unit/reports/multi_table/test_base_multi_table_report.py +++ b/tests/unit/reports/multi_table/test_base_multi_table_report.py @@ -411,7 +411,6 @@ def test_get_visualization_for_constraint_validity_property(self, table_name): """Test ``get_visualization`` raises the property error, with or without a table.""" # Setup report = BaseMultiTableReport() - report._validate_property_generated = Mock() report._properties = {'Constraint Validity': ConstraintValidity()} expected_message = ( 'Error: No visualization is available for Constraint Validity. To see the ' @@ -422,8 +421,6 @@ def test_get_visualization_for_constraint_validity_property(self, table_name): with pytest.raises(VisualizationUnavailableError, match=expected_message): report.get_visualization('Constraint Validity', table_name) - report._validate_property_generated.assert_called_once_with('Constraint Validity') - def test_get_visualization_for_structure_property(self): """Test the ``get_visualization`` method for the structure property.""" # Setup From 68b56722df614441aa64cb13ca3ec10d50f4a061 Mon Sep 17 00:00:00 2001 From: sarahmish Date: Fri, 18 Sep 2026 11:34:22 -0400 Subject: [PATCH 6/8] fix --- .../_properties/test_constraint_validity.py | 60 ++++++++++++------- .../reports/test_unified_reports.py | 32 +++------- .../_properties/test_constraint_validity.py | 10 +++- tests/unit/reports/test_base_report.py | 7 ++- 4 files changed, 59 insertions(+), 50 deletions(-) diff --git a/tests/integration/reports/multi_table/_properties/test_constraint_validity.py b/tests/integration/reports/multi_table/_properties/test_constraint_validity.py index a1e61bf5..fbe91501 100644 --- a/tests/integration/reports/multi_table/_properties/test_constraint_validity.py +++ b/tests/integration/reports/multi_table/_properties/test_constraint_validity.py @@ -9,20 +9,32 @@ from sdmetrics.errors import VisualizationUnavailableError from sdmetrics.reports.multi_table._properties import ConstraintValidity +NUM_ROWS = 215 +FIXED_COMBINATIONS_SCORE = 213 / NUM_ROWS +INEQUALITY_SCORE = 212 / NUM_ROWS + @pytest.fixture def constraints(): + """Constraints for the single table demo. + + The degree type depends on the high school specialization, and a placement + can not end before it starts. + """ return [ { 'class_name': 'FixedCombinations', - 'parameters': {'table_name': 'sessions', 'column_names': ['device', 'os']}, + 'parameters': { + 'table_name': 'student_placements', + 'column_names': ['high_spec', 'degree_type'], + }, }, { 'class_name': 'Inequality', 'parameters': { - 'table_name': 'transactions', - 'low_column_name': 'transaction_id', - 'high_column_name': 'amount', + 'table_name': 'student_placements', + 'low_column_name': 'start_date', + 'high_column_name': 'end_date', }, }, ] @@ -32,13 +44,13 @@ class TestConstraintValidity: def test_end_to_end(self, constraints): """Test the constraint validity property end to end.""" # Setup - real_data, synthetic_data, metadata = load_demo(modality='multi_table') + real_data, synthetic_data, metadata = load_demo(modality='single_table') constraint_validity = ConstraintValidity() expected_details = pd.DataFrame({ 'Constraint': ['FixedCombinations', 'Inequality'], 'Metric': ['ConstraintAdherence', 'ConstraintAdherence'], 'Parameters': [constraints[0]['parameters'], constraints[1]['parameters']], - 'Score': [1.0, 1.0], + 'Score': [FIXED_COMBINATIONS_SCORE, INEQUALITY_SCORE], }) # Run @@ -46,15 +58,20 @@ def test_end_to_end(self, constraints): details = constraint_validity.get_details() # Assert - assert result == 1.0 + assert result == pytest.approx(np.mean([FIXED_COMBINATIONS_SCORE, INEQUALITY_SCORE])) pd.testing.assert_frame_equal(details, expected_details) def test_end_to_end_with_invalid_rows(self, constraints): - """Test the score reflects the proportion of rows that break the constraints.""" + """Test the score drops when every placement ends before it starts. + + Rows without a start date can not break the inequality, so they stay valid. + """ # Setup - real_data, synthetic_data, metadata = load_demo(modality='multi_table') - synthetic_data['sessions'] = synthetic_data['sessions'].copy() - synthetic_data['sessions']['os'] = 'unknown' + real_data, synthetic_data, metadata = load_demo(modality='single_table') + synthetic_table = synthetic_data['student_placements'].copy() + has_start_date = synthetic_table['start_date'].notna() + synthetic_table.loc[has_start_date, 'end_date'] = '2000-01-01' + synthetic_data['student_placements'] = synthetic_table constraint_validity = ConstraintValidity() # Run @@ -62,20 +79,23 @@ def test_end_to_end_with_invalid_rows(self, constraints): details = constraint_validity.get_details() # Assert - assert result == 0.5 - assert details['Score'].tolist() == [0.0, 1.0] + missing_start_date_score = (~has_start_date).sum() / NUM_ROWS + assert details['Score'].tolist() == [FIXED_COMBINATIONS_SCORE, missing_start_date_score] + assert result == pytest.approx( + np.mean([FIXED_COMBINATIONS_SCORE, missing_start_date_score]) + ) def test_end_to_end_with_unsupported_constraint(self, constraints): """Test an unsupported constraint gets a NaN score and does not affect the average.""" # Setup - real_data, synthetic_data, metadata = load_demo(modality='multi_table') + real_data, synthetic_data, metadata = load_demo(modality='single_table') constraints.append({'class_name': 'Unsupported', 'parameters': {}}) constraint_validity = ConstraintValidity() expected_details = pd.DataFrame({ 'Constraint': ['FixedCombinations', 'Inequality', 'Unsupported'], 'Metric': ['ConstraintAdherence', 'ConstraintAdherence', 'ConstraintAdherence'], 'Parameters': [constraints[0]['parameters'], constraints[1]['parameters'], {}], - 'Score': [1.0, 1.0, np.nan], + 'Score': [FIXED_COMBINATIONS_SCORE, INEQUALITY_SCORE, np.nan], 'Error': [None, None, "ValueError: Unsupported constraint class 'Unsupported'."], }) @@ -84,13 +104,13 @@ def test_end_to_end_with_unsupported_constraint(self, constraints): details = constraint_validity.get_details() # Assert - assert result == 1.0 + assert result == pytest.approx(np.mean([FIXED_COMBINATIONS_SCORE, INEQUALITY_SCORE])) pd.testing.assert_frame_equal(details, expected_details) def test_end_to_end_without_constraints(self): """Test the score is NaN when there are no constraints.""" # Setup - real_data, synthetic_data, metadata = load_demo(modality='multi_table') + real_data, synthetic_data, metadata = load_demo(modality='single_table') constraint_validity = ConstraintValidity() # Run @@ -105,7 +125,7 @@ def test_end_to_end_without_constraints(self): def test_with_progress_bar(self, constraints, capsys): """Test that the progress bar is updated once per constraint.""" # Setup - real_data, synthetic_data, metadata = load_demo(modality='multi_table') + real_data, synthetic_data, metadata = load_demo(modality='single_table') constraint_validity = ConstraintValidity() num_constraints = constraint_validity._get_num_iterations(metadata, constraints) progress_bar = tqdm(total=num_constraints, file=sys.stdout) @@ -118,7 +138,7 @@ def test_with_progress_bar(self, constraints, capsys): output = capsys.readouterr().out # Assert - assert result == 1.0 + assert result == pytest.approx(np.mean([FIXED_COMBINATIONS_SCORE, INEQUALITY_SCORE])) assert num_constraints == 2 assert '100%' in output assert f'{num_constraints}/{num_constraints}' in output @@ -126,7 +146,7 @@ def test_with_progress_bar(self, constraints, capsys): def test_get_visualization(self, constraints): """Test ``get_visualization`` raises an error.""" # Setup - real_data, synthetic_data, metadata = load_demo(modality='multi_table') + real_data, synthetic_data, metadata = load_demo(modality='single_table') constraint_validity = ConstraintValidity() constraint_validity.get_score(real_data, synthetic_data, metadata, constraints) expected_message = ( diff --git a/tests/integration/reports/test_unified_reports.py b/tests/integration/reports/test_unified_reports.py index 5e9edcbe..d19cc17b 100644 --- a/tests/integration/reports/test_unified_reports.py +++ b/tests/integration/reports/test_unified_reports.py @@ -451,25 +451,17 @@ def test_unified_quality_report_single_table_verbose_skips_relationship_properti def test_unified_diagnostic_report_multi_table(): # Setup - multiple_constraints = [ + constraints = [ { 'class_name': 'FixedCombinations', 'parameters': {'table_name': 'sessions', 'column_names': ['device', 'os']}, }, - { - 'class_name': 'Inequality', - 'parameters': { - 'table_name': 'transactions', - 'low_column_name': 'transaction_id', - 'high_column_name': 'amount', - }, - }, ] real_data, synthetic_data, metadata = load_multi_table_demo() # Run report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, multiple_constraints, verbose=False) + report.generate(real_data, synthetic_data, metadata, constraints, verbose=False) # Assert expected_properties = pd.DataFrame({ @@ -482,10 +474,10 @@ def test_unified_diagnostic_report_multi_table(): 'Score': [1.0, 1.0, 1.0, 1.0], }) expected_details_constraint_validity = pd.DataFrame({ - 'Constraint': ['FixedCombinations', 'Inequality'], - 'Metric': ['ConstraintAdherence', 'ConstraintAdherence'], - 'Parameters': [constraint['parameters'] for constraint in multiple_constraints], - 'Score': [1.0, 1.0], + 'Constraint': ['FixedCombinations'], + 'Metric': ['ConstraintAdherence'], + 'Parameters': [constraints[0]['parameters']], + 'Score': [1.0], }) expected_details_data_validity = pd.DataFrame({ 'Table': [ @@ -804,25 +796,17 @@ def test_unified_diagnostic_report_multi_table_with_no_relationships_does_not_sk def test_unified_diagnostic_report_multi_table_verbose_with_constraints(capsys): """Test the Constraint Validity property runs last with one progress step per constraint.""" # Setup - multiple_constraints = [ + constraints = [ { 'class_name': 'FixedCombinations', 'parameters': {'table_name': 'sessions', 'column_names': ['device', 'os']}, }, - { - 'class_name': 'Inequality', - 'parameters': { - 'table_name': 'transactions', - 'low_column_name': 'transaction_id', - 'high_column_name': 'amount', - }, - }, ] real_data, synthetic_data, metadata = load_multi_table_demo() # Run report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, multiple_constraints, verbose=True) + report.generate(real_data, synthetic_data, metadata, constraints, verbose=True) output = capsys.readouterr().out # Assert diff --git a/tests/unit/reports/multi_table/_properties/test_constraint_validity.py b/tests/unit/reports/multi_table/_properties/test_constraint_validity.py index dd636128..9e6feafb 100644 --- a/tests/unit/reports/multi_table/_properties/test_constraint_validity.py +++ b/tests/unit/reports/multi_table/_properties/test_constraint_validity.py @@ -20,6 +20,8 @@ def data(): 'user_id': [0, 0, 1], 'device': ['mobile', 'tablet', 'mobile'], 'os': ['android', 'ios', 'android'], + 'session_start': ['2024-01-01', '2024-01-02', '2024-01-03'], + 'session_end': ['2024-01-02', '2024-01-02', '2024-01-05'], }), } @@ -39,6 +41,8 @@ def metadata(): 'user_id': {'sdtype': 'id'}, 'device': {'sdtype': 'categorical'}, 'os': {'sdtype': 'categorical'}, + 'session_start': {'sdtype': 'datetime', 'datetime_format': '%Y-%m-%d'}, + 'session_end': {'sdtype': 'datetime', 'datetime_format': '%Y-%m-%d'}, }, }, }, @@ -63,9 +67,9 @@ def constraints(): { 'class_name': 'Inequality', 'parameters': { - 'table_name': 'users', - 'low_column_name': 'user_id', - 'high_column_name': 'age', + 'table_name': 'sessions', + 'low_column_name': 'session_start', + 'high_column_name': 'session_end', }, }, ] diff --git a/tests/unit/reports/test_base_report.py b/tests/unit/reports/test_base_report.py index ee7916fd..fdf8daa8 100644 --- a/tests/unit/reports/test_base_report.py +++ b/tests/unit/reports/test_base_report.py @@ -551,16 +551,17 @@ def test_generate_with_constraints(self, mock_tqdm): base_report.generate(real_data, synthetic_data, metadata, constraints, verbose=True) # Assert + copied_real_data, copied_synthetic_data, _ = base_report._validate.call_args.args base_report._properties['Property 1']._get_num_iterations.assert_called_once_with(metadata) base_report._properties['Property 1'].get_score.assert_called_once_with( - real_data, synthetic_data, metadata, progress_bar=mock_tqdm.return_value + copied_real_data, copied_synthetic_data, metadata, progress_bar=mock_tqdm.return_value ) base_report._properties['Constraint Validity']._get_num_iterations.assert_called_once_with( metadata, constraints=constraints ) base_report._properties['Constraint Validity'].get_score.assert_called_once_with( - real_data, - synthetic_data, + copied_real_data, + copied_synthetic_data, metadata, progress_bar=mock_tqdm.return_value, constraints=constraints, From d3cfec67193683cfdf4f77f3435e09ad251c4029 Mon Sep 17 00:00:00 2001 From: sarahmish Date: Fri, 18 Sep 2026 20:10:30 -0400 Subject: [PATCH 7/8] update --- .../_properties/test_constraint_validity.py | 60 +++++++------------ .../reports/test_unified_reports.py | 29 +++++---- .../_properties/test_constraint_validity.py | 10 +--- 3 files changed, 39 insertions(+), 60 deletions(-) diff --git a/tests/integration/reports/multi_table/_properties/test_constraint_validity.py b/tests/integration/reports/multi_table/_properties/test_constraint_validity.py index fbe91501..a1e61bf5 100644 --- a/tests/integration/reports/multi_table/_properties/test_constraint_validity.py +++ b/tests/integration/reports/multi_table/_properties/test_constraint_validity.py @@ -9,32 +9,20 @@ from sdmetrics.errors import VisualizationUnavailableError from sdmetrics.reports.multi_table._properties import ConstraintValidity -NUM_ROWS = 215 -FIXED_COMBINATIONS_SCORE = 213 / NUM_ROWS -INEQUALITY_SCORE = 212 / NUM_ROWS - @pytest.fixture def constraints(): - """Constraints for the single table demo. - - The degree type depends on the high school specialization, and a placement - can not end before it starts. - """ return [ { 'class_name': 'FixedCombinations', - 'parameters': { - 'table_name': 'student_placements', - 'column_names': ['high_spec', 'degree_type'], - }, + 'parameters': {'table_name': 'sessions', 'column_names': ['device', 'os']}, }, { 'class_name': 'Inequality', 'parameters': { - 'table_name': 'student_placements', - 'low_column_name': 'start_date', - 'high_column_name': 'end_date', + 'table_name': 'transactions', + 'low_column_name': 'transaction_id', + 'high_column_name': 'amount', }, }, ] @@ -44,13 +32,13 @@ class TestConstraintValidity: def test_end_to_end(self, constraints): """Test the constraint validity property end to end.""" # Setup - real_data, synthetic_data, metadata = load_demo(modality='single_table') + real_data, synthetic_data, metadata = load_demo(modality='multi_table') constraint_validity = ConstraintValidity() expected_details = pd.DataFrame({ 'Constraint': ['FixedCombinations', 'Inequality'], 'Metric': ['ConstraintAdherence', 'ConstraintAdherence'], 'Parameters': [constraints[0]['parameters'], constraints[1]['parameters']], - 'Score': [FIXED_COMBINATIONS_SCORE, INEQUALITY_SCORE], + 'Score': [1.0, 1.0], }) # Run @@ -58,20 +46,15 @@ def test_end_to_end(self, constraints): details = constraint_validity.get_details() # Assert - assert result == pytest.approx(np.mean([FIXED_COMBINATIONS_SCORE, INEQUALITY_SCORE])) + assert result == 1.0 pd.testing.assert_frame_equal(details, expected_details) def test_end_to_end_with_invalid_rows(self, constraints): - """Test the score drops when every placement ends before it starts. - - Rows without a start date can not break the inequality, so they stay valid. - """ + """Test the score reflects the proportion of rows that break the constraints.""" # Setup - real_data, synthetic_data, metadata = load_demo(modality='single_table') - synthetic_table = synthetic_data['student_placements'].copy() - has_start_date = synthetic_table['start_date'].notna() - synthetic_table.loc[has_start_date, 'end_date'] = '2000-01-01' - synthetic_data['student_placements'] = synthetic_table + real_data, synthetic_data, metadata = load_demo(modality='multi_table') + synthetic_data['sessions'] = synthetic_data['sessions'].copy() + synthetic_data['sessions']['os'] = 'unknown' constraint_validity = ConstraintValidity() # Run @@ -79,23 +62,20 @@ def test_end_to_end_with_invalid_rows(self, constraints): details = constraint_validity.get_details() # Assert - missing_start_date_score = (~has_start_date).sum() / NUM_ROWS - assert details['Score'].tolist() == [FIXED_COMBINATIONS_SCORE, missing_start_date_score] - assert result == pytest.approx( - np.mean([FIXED_COMBINATIONS_SCORE, missing_start_date_score]) - ) + assert result == 0.5 + assert details['Score'].tolist() == [0.0, 1.0] def test_end_to_end_with_unsupported_constraint(self, constraints): """Test an unsupported constraint gets a NaN score and does not affect the average.""" # Setup - real_data, synthetic_data, metadata = load_demo(modality='single_table') + real_data, synthetic_data, metadata = load_demo(modality='multi_table') constraints.append({'class_name': 'Unsupported', 'parameters': {}}) constraint_validity = ConstraintValidity() expected_details = pd.DataFrame({ 'Constraint': ['FixedCombinations', 'Inequality', 'Unsupported'], 'Metric': ['ConstraintAdherence', 'ConstraintAdherence', 'ConstraintAdherence'], 'Parameters': [constraints[0]['parameters'], constraints[1]['parameters'], {}], - 'Score': [FIXED_COMBINATIONS_SCORE, INEQUALITY_SCORE, np.nan], + 'Score': [1.0, 1.0, np.nan], 'Error': [None, None, "ValueError: Unsupported constraint class 'Unsupported'."], }) @@ -104,13 +84,13 @@ def test_end_to_end_with_unsupported_constraint(self, constraints): details = constraint_validity.get_details() # Assert - assert result == pytest.approx(np.mean([FIXED_COMBINATIONS_SCORE, INEQUALITY_SCORE])) + assert result == 1.0 pd.testing.assert_frame_equal(details, expected_details) def test_end_to_end_without_constraints(self): """Test the score is NaN when there are no constraints.""" # Setup - real_data, synthetic_data, metadata = load_demo(modality='single_table') + real_data, synthetic_data, metadata = load_demo(modality='multi_table') constraint_validity = ConstraintValidity() # Run @@ -125,7 +105,7 @@ def test_end_to_end_without_constraints(self): def test_with_progress_bar(self, constraints, capsys): """Test that the progress bar is updated once per constraint.""" # Setup - real_data, synthetic_data, metadata = load_demo(modality='single_table') + real_data, synthetic_data, metadata = load_demo(modality='multi_table') constraint_validity = ConstraintValidity() num_constraints = constraint_validity._get_num_iterations(metadata, constraints) progress_bar = tqdm(total=num_constraints, file=sys.stdout) @@ -138,7 +118,7 @@ def test_with_progress_bar(self, constraints, capsys): output = capsys.readouterr().out # Assert - assert result == pytest.approx(np.mean([FIXED_COMBINATIONS_SCORE, INEQUALITY_SCORE])) + assert result == 1.0 assert num_constraints == 2 assert '100%' in output assert f'{num_constraints}/{num_constraints}' in output @@ -146,7 +126,7 @@ def test_with_progress_bar(self, constraints, capsys): def test_get_visualization(self, constraints): """Test ``get_visualization`` raises an error.""" # Setup - real_data, synthetic_data, metadata = load_demo(modality='single_table') + real_data, synthetic_data, metadata = load_demo(modality='multi_table') constraint_validity = ConstraintValidity() constraint_validity.get_score(real_data, synthetic_data, metadata, constraints) expected_message = ( diff --git a/tests/integration/reports/test_unified_reports.py b/tests/integration/reports/test_unified_reports.py index d19cc17b..7420d37b 100644 --- a/tests/integration/reports/test_unified_reports.py +++ b/tests/integration/reports/test_unified_reports.py @@ -126,18 +126,20 @@ def _load_single_table_quality_report_data(): def test_unified_diagnostic_report_single_table(): # Setup - fixed_combination = { - 'class_name': 'FixedCombinations', + inequality = { + 'class_name': 'Inequality', 'parameters': { 'table_name': 'student_placements', - 'column_names': ['gender', 'degree_type'], + 'low_column_name': 'start_date', + 'high_column_name': 'end_date', }, } real_data, synthetic_data, metadata = load_single_table_demo() + synthetic_data['student_placements'].loc[[43, 93, 179], 'end_date'] = None # Run report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, [fixed_combination], verbose=False) + report.generate(real_data, synthetic_data, metadata, [inequality], verbose=False) # Assert expected_properties = pd.DataFrame({ @@ -145,9 +147,9 @@ def test_unified_diagnostic_report_single_table(): 'Score': [1.0, 1.0, 1.0], }) expected_details_constraint_validity = pd.DataFrame({ - 'Constraint': ['FixedCombinations'], + 'Constraint': ['Inequality'], 'Metric': ['ConstraintAdherence'], - 'Parameters': [fixed_combination['parameters']], + 'Parameters': [inequality['parameters']], 'Score': [1.0], }) expected_details_data_validity = pd.DataFrame({ @@ -350,18 +352,19 @@ def test_unified_diagnostic_report_single_table_verbose_skips_relationship_valid def test_unified_diagnostic_report_single_table_verbose_with_constraints(capsys): """Test unified diagnostic report prints the Constraint Validity progress for single-table.""" # Setup - fixed_combination = { - 'class_name': 'FixedCombinations', + inequality = { + 'class_name': 'Inequality', 'parameters': { 'table_name': 'student_placements', - 'column_names': ['gender', 'degree_type'], + 'low_column_name': 'start_date', + 'high_column_name': 'end_date', }, } real_data, synthetic_data, metadata = load_single_table_demo() # Run report = DiagnosticReport() - report.generate(real_data, synthetic_data, metadata, [fixed_combination], verbose=True) + report.generate(real_data, synthetic_data, metadata, [inequality], verbose=True) output = capsys.readouterr().out # Assert @@ -374,13 +377,13 @@ def test_unified_diagnostic_report_single_table_verbose_with_constraints(capsys) '(3/4) Evaluating Relationship Validity: N/A', 'This property does not apply to single-table data.', '(4/4) Evaluating Constraint Validity:', - 'Constraint Validity Score: 100.0%', - 'Overall Score (Average): 100.0%', + 'Constraint Validity Score: 98.6%', + 'Overall Score (Average): 99.53%', ] for line in expected_lines: assert line in output - assert report.get_score() == 1.0 + assert report.get_score() >= 0.99 def test_diagnostic_report_with_ordinal_sdtype(): diff --git a/tests/unit/reports/multi_table/_properties/test_constraint_validity.py b/tests/unit/reports/multi_table/_properties/test_constraint_validity.py index 9e6feafb..dd636128 100644 --- a/tests/unit/reports/multi_table/_properties/test_constraint_validity.py +++ b/tests/unit/reports/multi_table/_properties/test_constraint_validity.py @@ -20,8 +20,6 @@ def data(): 'user_id': [0, 0, 1], 'device': ['mobile', 'tablet', 'mobile'], 'os': ['android', 'ios', 'android'], - 'session_start': ['2024-01-01', '2024-01-02', '2024-01-03'], - 'session_end': ['2024-01-02', '2024-01-02', '2024-01-05'], }), } @@ -41,8 +39,6 @@ def metadata(): 'user_id': {'sdtype': 'id'}, 'device': {'sdtype': 'categorical'}, 'os': {'sdtype': 'categorical'}, - 'session_start': {'sdtype': 'datetime', 'datetime_format': '%Y-%m-%d'}, - 'session_end': {'sdtype': 'datetime', 'datetime_format': '%Y-%m-%d'}, }, }, }, @@ -67,9 +63,9 @@ def constraints(): { 'class_name': 'Inequality', 'parameters': { - 'table_name': 'sessions', - 'low_column_name': 'session_start', - 'high_column_name': 'session_end', + 'table_name': 'users', + 'low_column_name': 'user_id', + 'high_column_name': 'age', }, }, ] From 789a2e0ce4fb9fa9daedf9c288e39e736a08c58f Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish <40212131+sarahmish@users.noreply.github.com> Date: Wed, 23 Sep 2026 20:13:16 -0400 Subject: [PATCH 8/8] update test description --- tests/integration/reports/test_unified_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/reports/test_unified_reports.py b/tests/integration/reports/test_unified_reports.py index 7420d37b..e16efeab 100644 --- a/tests/integration/reports/test_unified_reports.py +++ b/tests/integration/reports/test_unified_reports.py @@ -797,7 +797,7 @@ def test_unified_diagnostic_report_multi_table_with_no_relationships_does_not_sk def test_unified_diagnostic_report_multi_table_verbose_with_constraints(capsys): - """Test the Constraint Validity property runs last with one progress step per constraint.""" + """Test unified diagnostic report prints the Constraint Validity progress for multi-table.""" # Setup constraints = [ {