From f89e1d6eebc520a7a7a0f7b9462d2e4dcea59855 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 11 May 2026 11:04:40 -0600 Subject: [PATCH 1/5] allow LinearStandardFormCompiler to return unhandled nonlinear constraints --- pyomo/repn/plugins/standard_form.py | 125 ++++++++++++++++++++++--- pyomo/repn/tests/test_standard_form.py | 84 +++++++++++++++++ 2 files changed, 196 insertions(+), 13 deletions(-) diff --git a/pyomo/repn/plugins/standard_form.py b/pyomo/repn/plugins/standard_form.py index 70b1b74b8cc..ee4ba1422b1 100644 --- a/pyomo/repn/plugins/standard_form.py +++ b/pyomo/repn/plugins/standard_form.py @@ -99,7 +99,9 @@ class LinearStandardFormInfo: objectives : List[ObjectiveData] - The list of Pyomo objective objects corresponding to the active objectives + The list of Pyomo objective objects corresponding to the active + objectives whose expressions are purely linear (and thus appear + in `c`). eliminated_vars: List[Tuple[VarData, NumericExpression]] @@ -111,9 +113,35 @@ class LinearStandardFormInfo: all variables appearing in the expression must either have appeared in the standard form, or appear *earlier* in this list. + nonlinear_constraints : List[ConstraintData] or None + + Constraints skipped because they contain nonlinear terms. ``None`` + when ``allow_nonlinear=False`` (the default). When + ``allow_nonlinear=True``, holds the list of constraints with nonlinear + terms that were omitted from the compiled matrices (may be empty). + + nonlinear_objectives : List[ObjectiveData] or None + + Objectives skipped because they contain nonlinear terms. ``None`` + when ``allow_nonlinear=False`` (the default). When + ``allow_nonlinear=True``, holds the list of objectives with nonlinear + terms that were omitted from the compiled matrices (may be empty). + """ - def __init__(self, c, c_offset, A, rhs, rows, columns, objectives, eliminated_vars): + def __init__( + self, + c, + c_offset, + A, + rhs, + rows, + columns, + objectives, + eliminated_vars, + nonlinear_constraints=None, + nonlinear_objectives=None, + ): self.c = c self.c_offset = c_offset self.A = A @@ -122,6 +150,8 @@ def __init__(self, c, c_offset, A, rhs, rows, columns, objectives, eliminated_va self.columns = columns self.objectives = objectives self.eliminated_vars = eliminated_vars + self.nonlinear_constraints = nonlinear_constraints + self.nonlinear_objectives = nonlinear_objectives @property def x(self): @@ -188,6 +218,18 @@ class LinearStandardFormCompiler: description='If not None, map all objectives to the specified sense.', ), ) + CONFIG.declare( + 'allow_nonlinear', + ConfigValue( + default=False, + domain=bool, + description='If True, constraints and objectives containing nonlinear ' + 'terms are collected into ``LinearStandardFormInfo.nonlinear_constraints`` ' + 'and ``LinearStandardFormInfo.nonlinear_objectives`` rather than raising ' + 'an exception. The nonlinear components are omitted from the compiled ' + 'matrices.', + ), + ) CONFIG.declare( 'show_section_timing', ConfigValue( @@ -364,6 +406,7 @@ def write(self, model): # Process objective # set_sense = self.config.set_sense + allow_nonlinear = self.config.allow_nonlinear objectives = [] for blk in component_map[Objective]: objectives.extend( @@ -376,29 +419,54 @@ def write(self, model): obj_data = [] obj_index = [] obj_index_ptr = [0] + linear_objectives = [] + nonlinear_objectives = [] for obj in objectives: if hasattr(obj, 'template_expr'): - offset, linear_index, linear_data, lb, ub = ( - template_visitor.expand_expression(obj, obj.template_expr()) - ) + if allow_nonlinear: + try: + offset, linear_index, linear_data, lb, ub = ( + template_visitor.expand_expression(obj, obj.template_expr()) + ) + except InvalidExpressionError: + nonlinear_objectives.append(obj) + if with_debug_timing: + timer.toc( + 'Objective %s (nonlinear)', obj, level=logging.DEBUG + ) + continue + else: + offset, linear_index, linear_data, lb, ub = ( + template_visitor.expand_expression(obj, obj.template_expr()) + ) assert lb is None and ub is None N = len(linear_index) obj_index.append(linear_index) obj_data.append(linear_data) obj_offset.append(offset) + linear_objectives.append(obj) else: repn = visitor.walk_expression(obj.expr) - N = len(repn.linear) - obj_index.append(map(var_recorder.var_order.__getitem__, repn.linear)) - obj_data.append(repn.linear.values()) - obj_offset.append(repn.constant) if repn.nonlinear is not None: + if allow_nonlinear: + nonlinear_objectives.append(obj) + if with_debug_timing: + timer.toc( + 'Objective %s (nonlinear)', obj, level=logging.DEBUG + ) + continue raise InvalidExpressionError( f"Model objective ({obj.name}) contains nonlinear terms that " "cannot be compiled to standard (linear) form." ) + N = len(repn.linear) + obj_index.append(map(var_recorder.var_order.__getitem__, repn.linear)) + obj_data.append(repn.linear.values()) + obj_offset.append(repn.constant) + linear_objectives.append(obj) + obj_nnz += N if set_sense is not None and set_sense != obj.sense: obj_data[-1] = -self._to_vector(obj_data[-1], float, N) @@ -406,6 +474,7 @@ def write(self, model): obj_index_ptr.append(obj_index_ptr[-1] + N) if with_debug_timing: timer.toc('Objective %s', obj, level=logging.DEBUG) + objectives = linear_objectives # # Tabulate constraints @@ -420,6 +489,7 @@ def write(self, model): con_data = [] con_index = [] con_index_ptr = [0] + nonlinear_constraints = [] last_parent = None for con in ordered_active_constraints(model, self.config): if with_debug_timing and con._component is not last_parent: @@ -428,9 +498,22 @@ def write(self, model): last_parent = con._component if hasattr(con, 'template_expr'): - offset, linear_index, linear_data, lb, ub = ( - template_visitor.expand_expression(con, con.template_expr()) - ) + if allow_nonlinear: + try: + offset, linear_index, linear_data, lb, ub = ( + template_visitor.expand_expression(con, con.template_expr()) + ) + except InvalidExpressionError: + nonlinear_constraints.append(con) + if with_debug_timing: + timer.toc( + 'Constraint %s (nonlinear)', con, level=logging.DEBUG + ) + continue + else: + offset, linear_index, linear_data, lb, ub = ( + template_visitor.expand_expression(con, con.template_expr()) + ) N = len(linear_data) else: # Note: lb and ub could be a number, expression, or None. @@ -442,6 +525,13 @@ def write(self, model): ub = value(ub) repn = visitor.walk_expression(body) if repn.nonlinear is not None: + if allow_nonlinear: + nonlinear_constraints.append(con) + if with_debug_timing: + timer.toc( + 'Constraint %s (nonlinear)', con, level=logging.DEBUG + ) + continue raise InvalidConstraintError( f"Model constraint ({con.name}) contains nonlinear terms that " "cannot be compiled to standard (linear) form." @@ -613,7 +703,16 @@ def write(self, model): eliminated_vars = [] info = LinearStandardFormInfo( - c, obj_offset, A, rhs, rows, columns, objectives, eliminated_vars + c, + obj_offset, + A, + rhs, + rows, + columns, + objectives, + eliminated_vars, + nonlinear_constraints=nonlinear_constraints if allow_nonlinear else None, + nonlinear_objectives=nonlinear_objectives if allow_nonlinear else None, ) timer.toc("Generated linear standard form representation", delta=False) return info diff --git a/pyomo/repn/tests/test_standard_form.py b/pyomo/repn/tests/test_standard_form.py index eaa515c21bc..73ac6dd388c 100644 --- a/pyomo/repn/tests/test_standard_form.py +++ b/pyomo/repn/tests/test_standard_form.py @@ -351,6 +351,90 @@ def test_alternative_forms(self): self.assertTrue(np.all(repn.c == ref)) self._verify_solution(soln, repn, True) + def test_nonlinear_fields_none_when_not_allowed(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.c = pyo.Constraint(expr=m.x <= 1) + m.o = pyo.Objective(expr=m.x) + + repn = LinearStandardFormCompiler().write(m, mixed_form=True) + self.assertIsNone(repn.nonlinear_constraints) + self.assertIsNone(repn.nonlinear_objectives) + + def test_allow_nonlinear_constraints(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.c_lin = pyo.Constraint(expr=m.x + m.y <= 5) + m.c_nl = pyo.Constraint(expr=m.x**2 + m.y <= 3) + m.o = pyo.Objective(expr=m.x + m.y) + + # Default (allow_nonlinear=False) must raise on the nonlinear constraint. + with self.assertRaises(Exception): + LinearStandardFormCompiler().write(m, mixed_form=True) + + # allow_nonlinear=True: nonlinear constraint is collected separately; + # the linear constraint still appears in A. + repn = LinearStandardFormCompiler().write( + m, mixed_form=True, allow_nonlinear=True + ) + self.assertEqual(repn.nonlinear_constraints, [m.c_nl]) + self.assertEqual(repn.nonlinear_objectives, []) + # Only the linear constraint appears in A. + self.assertEqual(len(repn.rows), 1) + self.assertEqual(repn.rows[0].constraint, m.c_lin) + # Linear objective is still compiled into c. + self.assertEqual(repn.objectives, [m.o]) + self.assertTrue(np.all(repn.c.toarray() != 0)) + + def test_allow_nonlinear_objective(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.c_lin = pyo.Constraint(expr=m.x + m.y <= 5) + m.o_nl = pyo.Objective(expr=m.x**2 + m.y) + + # Default must raise on the nonlinear objective. + with self.assertRaises(Exception): + LinearStandardFormCompiler().write(m, mixed_form=True) + + repn = LinearStandardFormCompiler().write( + m, mixed_form=True, allow_nonlinear=True + ) + # Nonlinear objective is NOT compiled into c; it appears in nonlinear_objectives. + self.assertEqual(repn.nonlinear_objectives, [m.o_nl]) + self.assertEqual(repn.objectives, []) + # c is empty (no linear objectives). + self.assertEqual(repn.c.shape[0], 0) + # The linear constraint is still compiled normally. + self.assertEqual(len(repn.rows), 1) + self.assertEqual(repn.rows[0].constraint, m.c_lin) + + def test_allow_nonlinear_mixed(self): + """Linear constraints/objectives compiled; nonlinear ones passed through.""" + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.c_lin = pyo.Constraint(expr=m.x + 2 * m.y >= 1) + m.c_nl = pyo.Constraint(expr=m.x * m.y <= 4) + m.o_lin = pyo.Objective(expr=m.x + m.y) + + repn = LinearStandardFormCompiler().write( + m, mixed_form=True, allow_nonlinear=True + ) + + # Exactly one linear row, one nonlinear constraint. + self.assertEqual(len(repn.rows), 1) + self.assertEqual(repn.rows[0].constraint, m.c_lin) + self.assertEqual(repn.nonlinear_constraints, [m.c_nl]) + # Linear objective compiles normally. + self.assertEqual(repn.objectives, [m.o_lin]) + self.assertEqual(repn.nonlinear_objectives, []) + # Both variables appear as columns (referenced by the linear constraint). + col_ids = {id(v) for v in repn.columns} + self.assertIn(id(m.x), col_ids) + self.assertIn(id(m.y), col_ids) + class TestTemplatedLinearStandardFormCompiler(TestLinearStandardFormCompiler): def setUp(self): From b0e18da22e1bdf08ffd4871aa5da1b72526bdfef Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Mon, 11 May 2026 12:10:47 -0600 Subject: [PATCH 2/5] allow SOS constraints (and other ctypes) to pass for standard form --- pyomo/repn/plugins/standard_form.py | 15 ++++++++++++++- pyomo/repn/tests/test_standard_form.py | 23 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/pyomo/repn/plugins/standard_form.py b/pyomo/repn/plugins/standard_form.py index ee4ba1422b1..4953b596e09 100644 --- a/pyomo/repn/plugins/standard_form.py +++ b/pyomo/repn/plugins/standard_form.py @@ -230,6 +230,18 @@ class LinearStandardFormCompiler: 'matrices.', ), ) + CONFIG.declare( + 'extra_valid_ctypes', + ConfigValue( + default=[], + description='Additional component types that are permitted to appear ' + 'in the model without causing an error, but that are not processed by ' + 'the compiler. Use this when the model contains component types ' + '(e.g., :class:`~pyomo.core.base.sos.SOSConstraint`) that are valid ' + 'for the calling solver but that the standard-form compiler does not ' + 'know how to handle.', + ), + ) CONFIG.declare( 'show_section_timing', ConfigValue( @@ -351,7 +363,8 @@ def write(self, model): RangeSet, Port, # TODO: Piecewise, Complementarity - }, + } + | set(self.config.extra_valid_ctypes), targets={Suffix, Objective}, ) if unknown: diff --git a/pyomo/repn/tests/test_standard_form.py b/pyomo/repn/tests/test_standard_form.py index 73ac6dd388c..3c5184227f7 100644 --- a/pyomo/repn/tests/test_standard_form.py +++ b/pyomo/repn/tests/test_standard_form.py @@ -435,6 +435,29 @@ def test_allow_nonlinear_mixed(self): self.assertIn(id(m.x), col_ids) self.assertIn(id(m.y), col_ids) + def test_extra_valid_ctypes(self): + """Component types in extra_valid_ctypes are permitted but not compiled.""" + m = pyo.ConcreteModel() + m.x = pyo.Var([1, 2, 3]) + m.y = pyo.Var() + m.obj = pyo.Objective(expr=m.y) + m.sos = pyo.SOSConstraint(var=m.x, sos=1) + + # Without extra_valid_ctypes, LSFC raises on the SOSConstraint. + with self.assertRaises(ValueError): + LinearStandardFormCompiler().write(m, mixed_form=True) + + # With extra_valid_ctypes, the SOSConstraint is silently skipped. + repn = LinearStandardFormCompiler().write( + m, mixed_form=True, extra_valid_ctypes=[pyo.SOSConstraint] + ) + # Only m.y appears in the objective; m.x[i] are unreferenced by + # linear constraints/objectives so not included in repn.columns. + self.assertEqual(len(repn.rows), 0) + col_ids = {id(v) for v in repn.columns} + self.assertIn(id(m.y), col_ids) + self.assertNotIn(id(m.x[1]), col_ids) + class TestTemplatedLinearStandardFormCompiler(TestLinearStandardFormCompiler): def setUp(self): From c993f8110a0575b6acad61273c350f338e910c60 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Thu, 21 May 2026 10:14:58 -0600 Subject: [PATCH 3/5] extra_valid_ctypes -> ignore_ctypes --- pyomo/repn/plugins/standard_form.py | 4 ++-- pyomo/repn/tests/test_standard_form.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyomo/repn/plugins/standard_form.py b/pyomo/repn/plugins/standard_form.py index 4953b596e09..ee536f26e52 100644 --- a/pyomo/repn/plugins/standard_form.py +++ b/pyomo/repn/plugins/standard_form.py @@ -231,7 +231,7 @@ class LinearStandardFormCompiler: ), ) CONFIG.declare( - 'extra_valid_ctypes', + 'ignore_ctypes', ConfigValue( default=[], description='Additional component types that are permitted to appear ' @@ -364,7 +364,7 @@ def write(self, model): Port, # TODO: Piecewise, Complementarity } - | set(self.config.extra_valid_ctypes), + | set(self.config.ignore_ctypes), targets={Suffix, Objective}, ) if unknown: diff --git a/pyomo/repn/tests/test_standard_form.py b/pyomo/repn/tests/test_standard_form.py index 3c5184227f7..0325af58373 100644 --- a/pyomo/repn/tests/test_standard_form.py +++ b/pyomo/repn/tests/test_standard_form.py @@ -435,21 +435,21 @@ def test_allow_nonlinear_mixed(self): self.assertIn(id(m.x), col_ids) self.assertIn(id(m.y), col_ids) - def test_extra_valid_ctypes(self): - """Component types in extra_valid_ctypes are permitted but not compiled.""" + def test_ignore_ctypes(self): + """Component types in ignore_ctypes are permitted but not compiled.""" m = pyo.ConcreteModel() m.x = pyo.Var([1, 2, 3]) m.y = pyo.Var() m.obj = pyo.Objective(expr=m.y) m.sos = pyo.SOSConstraint(var=m.x, sos=1) - # Without extra_valid_ctypes, LSFC raises on the SOSConstraint. + # Without ignore_ctypes, LSFC raises on the SOSConstraint. with self.assertRaises(ValueError): LinearStandardFormCompiler().write(m, mixed_form=True) - # With extra_valid_ctypes, the SOSConstraint is silently skipped. + # With ignore_ctypes, the SOSConstraint is silently skipped. repn = LinearStandardFormCompiler().write( - m, mixed_form=True, extra_valid_ctypes=[pyo.SOSConstraint] + m, mixed_form=True, ignore_ctypes=[pyo.SOSConstraint] ) # Only m.y appears in the objective; m.x[i] are unreferenced by # linear constraints/objectives so not included in repn.columns. From c492bccae290f532f95960ee584e418bda524a09 Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Thu, 21 May 2026 10:23:04 -0600 Subject: [PATCH 4/5] refactor to reduce branches --- pyomo/repn/plugins/standard_form.py | 34 ++++++++++++----------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/pyomo/repn/plugins/standard_form.py b/pyomo/repn/plugins/standard_form.py index ee536f26e52..ec6c086759a 100644 --- a/pyomo/repn/plugins/standard_form.py +++ b/pyomo/repn/plugins/standard_form.py @@ -436,22 +436,19 @@ def write(self, model): nonlinear_objectives = [] for obj in objectives: if hasattr(obj, 'template_expr'): - if allow_nonlinear: - try: - offset, linear_index, linear_data, lb, ub = ( - template_visitor.expand_expression(obj, obj.template_expr()) - ) - except InvalidExpressionError: + try: + offset, linear_index, linear_data, lb, ub = ( + template_visitor.expand_expression(obj, obj.template_expr()) + ) + except InvalidExpressionError: + if allow_nonlinear: nonlinear_objectives.append(obj) if with_debug_timing: timer.toc( 'Objective %s (nonlinear)', obj, level=logging.DEBUG ) continue - else: - offset, linear_index, linear_data, lb, ub = ( - template_visitor.expand_expression(obj, obj.template_expr()) - ) + raise assert lb is None and ub is None N = len(linear_index) obj_index.append(linear_index) @@ -511,22 +508,19 @@ def write(self, model): last_parent = con._component if hasattr(con, 'template_expr'): - if allow_nonlinear: - try: - offset, linear_index, linear_data, lb, ub = ( - template_visitor.expand_expression(con, con.template_expr()) - ) - except InvalidExpressionError: + try: + offset, linear_index, linear_data, lb, ub = ( + template_visitor.expand_expression(con, con.template_expr()) + ) + except InvalidExpressionError: + if allow_nonlinear: nonlinear_constraints.append(con) if with_debug_timing: timer.toc( 'Constraint %s (nonlinear)', con, level=logging.DEBUG ) continue - else: - offset, linear_index, linear_data, lb, ub = ( - template_visitor.expand_expression(con, con.template_expr()) - ) + raise N = len(linear_data) else: # Note: lb and ub could be a number, expression, or None. From e60ce1247589bf4e4ac107d7e323857ea7b6f11a Mon Sep 17 00:00:00 2001 From: Bernard Knueven Date: Thu, 21 May 2026 10:30:39 -0600 Subject: [PATCH 5/5] just use linear_objectives for clarity --- pyomo/repn/plugins/standard_form.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyomo/repn/plugins/standard_form.py b/pyomo/repn/plugins/standard_form.py index ec6c086759a..492fd3a0e79 100644 --- a/pyomo/repn/plugins/standard_form.py +++ b/pyomo/repn/plugins/standard_form.py @@ -484,7 +484,6 @@ def write(self, model): obj_index_ptr.append(obj_index_ptr[-1] + N) if with_debug_timing: timer.toc('Objective %s', obj, level=logging.DEBUG) - objectives = linear_objectives # # Tabulate constraints @@ -716,7 +715,7 @@ def write(self, model): rhs, rows, columns, - objectives, + linear_objectives, eliminated_vars, nonlinear_constraints=nonlinear_constraints if allow_nonlinear else None, nonlinear_objectives=nonlinear_objectives if allow_nonlinear else None,