From e1df337bf1904d94489bcde963657a0d3c6ab748 Mon Sep 17 00:00:00 2001 From: Christopher Cave-Ayland Date: Tue, 25 Jun 2024 21:32:06 +0100 Subject: [PATCH 01/11] Add new model and test scaffolds --- .../default_new_input/agent_objectives.csv | 3 + .../example/default_new_input/agent_pairs.csv | 2 + .../default_new_input/agent_regions.csv | 2 + .../data/example/default_new_input/agents.csv | 3 + .../data/example/default_new_input/assets.csv | 8 +++ .../example/default_new_input/commodities.csv | 6 ++ .../default_new_input/commodity_costs.csv | 58 +++++++++++++++++++ .../default_new_input/commodity_trade.csv | 1 + .../data/example/default_new_input/demand.csv | 3 + .../default_new_input/demand_slicing.csv | 7 +++ .../process_availabilities.csv | 6 ++ .../default_new_input/process_flows.csv | 12 ++++ .../default_new_input/process_parameters.csv | 6 ++ .../default_new_input/process_regions.csv | 6 ++ .../example/default_new_input/processes.csv | 6 ++ .../example/default_new_input/regions.csv | 2 + .../example/default_new_input/sectors.csv | 4 ++ .../example/default_new_input/time_slices.csv | 7 +++ src/muse/examples.py | 8 +++ 19 files changed, 150 insertions(+) create mode 100644 src/muse/data/example/default_new_input/agent_objectives.csv create mode 100644 src/muse/data/example/default_new_input/agent_pairs.csv create mode 100644 src/muse/data/example/default_new_input/agent_regions.csv create mode 100644 src/muse/data/example/default_new_input/agents.csv create mode 100644 src/muse/data/example/default_new_input/assets.csv create mode 100644 src/muse/data/example/default_new_input/commodities.csv create mode 100644 src/muse/data/example/default_new_input/commodity_costs.csv create mode 100644 src/muse/data/example/default_new_input/commodity_trade.csv create mode 100644 src/muse/data/example/default_new_input/demand.csv create mode 100644 src/muse/data/example/default_new_input/demand_slicing.csv create mode 100644 src/muse/data/example/default_new_input/process_availabilities.csv create mode 100644 src/muse/data/example/default_new_input/process_flows.csv create mode 100644 src/muse/data/example/default_new_input/process_parameters.csv create mode 100644 src/muse/data/example/default_new_input/process_regions.csv create mode 100644 src/muse/data/example/default_new_input/processes.csv create mode 100644 src/muse/data/example/default_new_input/regions.csv create mode 100644 src/muse/data/example/default_new_input/sectors.csv create mode 100644 src/muse/data/example/default_new_input/time_slices.csv diff --git a/src/muse/data/example/default_new_input/agent_objectives.csv b/src/muse/data/example/default_new_input/agent_objectives.csv new file mode 100644 index 000000000..c14612aaa --- /dev/null +++ b/src/muse/data/example/default_new_input/agent_objectives.csv @@ -0,0 +1,3 @@ +agent,objective,objective_data,objective_sort +Agent1,LCOE,1,TRUE +Agent2,LCOE,1,TRUE diff --git a/src/muse/data/example/default_new_input/agent_pairs.csv b/src/muse/data/example/default_new_input/agent_pairs.csv new file mode 100644 index 000000000..72e306f93 --- /dev/null +++ b/src/muse/data/example/default_new_input/agent_pairs.csv @@ -0,0 +1,2 @@ +name,new_agent,retrofit_agent,quantity +A1,Agent1,Agent2,1 diff --git a/src/muse/data/example/default_new_input/agent_regions.csv b/src/muse/data/example/default_new_input/agent_regions.csv new file mode 100644 index 000000000..257b59615 --- /dev/null +++ b/src/muse/data/example/default_new_input/agent_regions.csv @@ -0,0 +1,2 @@ +agent_pair,region +A1,R1 diff --git a/src/muse/data/example/default_new_input/agents.csv b/src/muse/data/example/default_new_input/agents.csv new file mode 100644 index 000000000..2d1261bde --- /dev/null +++ b/src/muse/data/example/default_new_input/agents.csv @@ -0,0 +1,3 @@ +agent,description,type,maturity_threshold,annual_cost_limit,search_rule,decision_rule +Agent1,New agent for A1,new,-1,inf,all,single +Agent2,Retrofit agent for A1,retrofit,-1,inf,all,single diff --git a/src/muse/data/example/default_new_input/assets.csv b/src/muse/data/example/default_new_input/assets.csv new file mode 100644 index 000000000..c30df1a13 --- /dev/null +++ b/src/muse/data/example/default_new_input/assets.csv @@ -0,0 +1,8 @@ +process_name,region,agent,capacity,year +gassupply1,R1,Agent2,15,2020 +gassupply1,R1,Agent2,15,2025 +gassupply1,R1,Agent2,7.5,2030 +gasCCGT,R1,Agent2,1,2020 +gasCCGT,R1,Agent2,1,2025 +gasboiler,R1,Agent2,10,2020 +gasboiler,R1,Agent2,5,2025 diff --git a/src/muse/data/example/default_new_input/commodities.csv b/src/muse/data/example/default_new_input/commodities.csv new file mode 100644 index 000000000..cec5cbf65 --- /dev/null +++ b/src/muse/data/example/default_new_input/commodities.csv @@ -0,0 +1,6 @@ +commodity_name,description,type,unit +electricity,Electricity,energy,PJ +gas,Gas,energy,PJ +heat,Heat,energy,PJ +wind,Wind,energy,PJ +C02f,Carbon dioxide,energy,kt diff --git a/src/muse/data/example/default_new_input/commodity_costs.csv b/src/muse/data/example/default_new_input/commodity_costs.csv new file mode 100644 index 000000000..85309f435 --- /dev/null +++ b/src/muse/data/example/default_new_input/commodity_costs.csv @@ -0,0 +1,58 @@ +year,region,commodity_name,value +2010,R1,electricity,14.81481472 +2015,R1,electricity,17.89814806 +2020,R1,electricity,19.5 +2025,R1,electricity,21.93518528 +2030,R1,electricity,26.50925917 +2035,R1,electricity,26.51851861 +2040,R1,electricity,23.85185194 +2045,R1,electricity,23.97222222 +2050,R1,electricity,24.06481472 +2055,R1,electricity,25.3425925 +2060,R1,electricity,25.53703694 +2065,R1,electricity,25.32407417 +2070,R1,electricity,23.36111111 +2075,R1,electricity,22.27777778 +2080,R1,electricity,22.25925917 +2085,R1,electricity,22.17592583 +2090,R1,electricity,22.03703694 +2095,R1,electricity,21.94444444 +2100,R1,electricity,21.39814806 +2010,R1,gas,6.6759 +2015,R1,gas,6.914325 +2020,R1,gas,7.15275 +2025,R1,gas,8.10645 +2030,R1,gas,9.06015 +2035,R1,gas,9.2191 +2040,R1,gas,9.37805 +2045,R1,gas,9.193829337 +2050,R1,gas,9.009608674 +2055,R1,gas,8.832625604 +2060,R1,gas,8.655642534 +2065,R1,gas,8.485612708 +2070,R1,gas,8.315582883 +2075,R1,gas,8.152233126 +2080,R1,gas,7.988883368 +2085,R1,gas,7.831951236 +2090,R1,gas,7.675019103 +2095,R1,gas,7.524252461 +2100,R1,gas,7.373485819 +2010,R1,CO2f,0 +2015,R1,CO2f,0.052913851 +2020,R1,CO2f,0.08314119 +2025,R1,CO2f,0.120069795 +2030,R1,CO2f,0.156998399 +2035,R1,CO2f,0.214877567 +2040,R1,CO2f,0.272756734 +2045,R1,CO2f,0.35394801 +2050,R1,CO2f,0.435139285 +2055,R1,CO2f,0.542365578 +2060,R1,CO2f,0.649591871 +2065,R1,CO2f,0.780892624 +2070,R1,CO2f,0.912193378 +2075,R1,CO2f,1.078321687 +2080,R1,CO2f,1.244449995 +2085,R1,CO2f,1.4253503 +2090,R1,CO2f,1.606250604 +2095,R1,CO2f,1.73877515 +2100,R1,CO2f,1.871299697 diff --git a/src/muse/data/example/default_new_input/commodity_trade.csv b/src/muse/data/example/default_new_input/commodity_trade.csv new file mode 100644 index 000000000..eb23c4b6c --- /dev/null +++ b/src/muse/data/example/default_new_input/commodity_trade.csv @@ -0,0 +1 @@ +commodity,region,net_import,year diff --git a/src/muse/data/example/default_new_input/demand.csv b/src/muse/data/example/default_new_input/demand.csv new file mode 100644 index 000000000..13c64fe8c --- /dev/null +++ b/src/muse/data/example/default_new_input/demand.csv @@ -0,0 +1,3 @@ +year,commodity_name,region,demand +2020,heat,R1,10 +2050,heat,R1,30 diff --git a/src/muse/data/example/default_new_input/demand_slicing.csv b/src/muse/data/example/default_new_input/demand_slicing.csv new file mode 100644 index 000000000..10d4693c2 --- /dev/null +++ b/src/muse/data/example/default_new_input/demand_slicing.csv @@ -0,0 +1,7 @@ +commodity,region,timeslice,fraction,year +heat,R1,night,0.1, +heat,R1,morning,0.15, +heat,R1,afternoon,0.1, +heat,R1,early-peak,0.15, +heat,R1,late-peak,0.3, +heat,R1,evening,0.2, diff --git a/src/muse/data/example/default_new_input/process_availabilities.csv b/src/muse/data/example/default_new_input/process_availabilities.csv new file mode 100644 index 000000000..300a11407 --- /dev/null +++ b/src/muse/data/example/default_new_input/process_availabilities.csv @@ -0,0 +1,6 @@ +process_name,timeslice,lim_type,value,year,region +gassupply1,ALL,UP,0.9,, +gasCCGT,ALL,UP,0.9,, +windturbine,ALL,UP,0.4,, +gasboiler,ALL,UP,1,, +heatpump,ALL,UP,1,, diff --git a/src/muse/data/example/default_new_input/process_flows.csv b/src/muse/data/example/default_new_input/process_flows.csv new file mode 100644 index 000000000..e6e72c67d --- /dev/null +++ b/src/muse/data/example/default_new_input/process_flows.csv @@ -0,0 +1,12 @@ +process_name,commodity_name,flow,year,region +gassupply1,gas,1,, +gasCCGT,gas,-1.67,, +gasCCGT,electricity,1,, +gasCCGT,CO2f,91.67,, +windturbine,wind,-1,, +windturbine,electricity,1,, +gasboiler,gas,-1.16,, +gasboiler,heat,1,, +gasboiler,CO2f,64.71,, +heatpump,electricity,-0.4,, +heatpump,heat,1,, diff --git a/src/muse/data/example/default_new_input/process_parameters.csv b/src/muse/data/example/default_new_input/process_parameters.csv new file mode 100644 index 000000000..00bb38bb7 --- /dev/null +++ b/src/muse/data/example/default_new_input/process_parameters.csv @@ -0,0 +1,6 @@ +process,cap_par,cap_exp,fix_par,fix_exp,var_par,var_exp,max_capacity_addition,max_capacity_growth,total_capacity_limit,life,scaling_size,efficiency,discount_rate,year,region +gassupply1,0,1,0,1,2.55,1,5,1,60,35,0.00000189,86,0.1,, +gasCCGT,23.78234399,1,0,1,0,1,2,1,60,35,0.00000189,86,0.1,, +windturbine,36.30771182,1,0,1,0,1,2,1,60,25,0.00000189,86,0.1,, +gasboiler,3.8,1,0,1,0,1,10,0.02,60,10,0.00000189,86,0.1,, +heatpump,8.866667,1,0,1,0,1,10,0.02,60,10,0.00000189,86,0.1,, diff --git a/src/muse/data/example/default_new_input/process_regions.csv b/src/muse/data/example/default_new_input/process_regions.csv new file mode 100644 index 000000000..e7ca8286c --- /dev/null +++ b/src/muse/data/example/default_new_input/process_regions.csv @@ -0,0 +1,6 @@ +process,region +gassupply1,R1 +gasCCGT,R1 +windturbine,R1 +gasboiler,R1 +heatpump,R1 diff --git a/src/muse/data/example/default_new_input/processes.csv b/src/muse/data/example/default_new_input/processes.csv new file mode 100644 index 000000000..7c4b9f818 --- /dev/null +++ b/src/muse/data/example/default_new_input/processes.csv @@ -0,0 +1,6 @@ +name,type,fuel,end_use,level,sector +gassupply1,energy,gas,gas,fixed,gas +gasCCGT,energy,gas,electricity,fixed,power +windturbine,energy,wind,electricity,fixed,power +gasboiler,energy,gas,heat,fixed,residential +heatpump,energy,electricity,heat,fixed,residential diff --git a/src/muse/data/example/default_new_input/regions.csv b/src/muse/data/example/default_new_input/regions.csv new file mode 100644 index 000000000..1583e5334 --- /dev/null +++ b/src/muse/data/example/default_new_input/regions.csv @@ -0,0 +1,2 @@ +name,description +R1,Region 1 diff --git a/src/muse/data/example/default_new_input/sectors.csv b/src/muse/data/example/default_new_input/sectors.csv new file mode 100644 index 000000000..a841328b6 --- /dev/null +++ b/src/muse/data/example/default_new_input/sectors.csv @@ -0,0 +1,4 @@ +name,description +gas,Gas sector +power,Power sector +residential,Residential sector diff --git a/src/muse/data/example/default_new_input/time_slices.csv b/src/muse/data/example/default_new_input/time_slices.csv new file mode 100644 index 000000000..376022d96 --- /dev/null +++ b/src/muse/data/example/default_new_input/time_slices.csv @@ -0,0 +1,7 @@ +season,day,time_of_day,fraction +all,all,night,0.1667 +all,all,morning,0.1667 +all,all,afternoon,0.1667 +all,all,early-peak,0.1667 +all,all,late-peak,0.1667 +all,all,evening,0.1667 diff --git a/src/muse/examples.py b/src/muse/examples.py index 189a82c85..40b06ac94 100644 --- a/src/muse/examples.py +++ b/src/muse/examples.py @@ -137,6 +137,8 @@ def copy_model( _copy_minimum_service(path) elif name.lower() == "trade": _copy_trade(path) + elif name.lower() == "default_new_input": + _copy_default_new_input(path) return path @@ -316,6 +318,12 @@ def update_lpsolver(data): modify_toml(path / "settings.toml", update_lpsolver) +def _copy_default_new_input(path: Path): + from shutil import copytree + + copytree(example_data_dir() / "default_new_input", path) + + def _copy_default_timeslice(path: Path): copytree(example_data_dir() / "default_timeslice", path) From 79016d948e9c4405534bea704c1c89f590b1e7e4 Mon Sep 17 00:00:00 2001 From: Christopher Cave-Ayland Date: Thu, 27 Jun 2024 16:06:00 +0100 Subject: [PATCH 02/11] Get tests running --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9e778e694..43220c550 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,9 @@ dependencies = [ "xlrd", "mypy-extensions", "pypubsub", - "tomlkit" + "tomlkit", + "duckdb", + "fsspec" ] dynamic = ["version"] From 549cdd17b4a8a9f7c7d86ee160f7687e2705fcdc Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 3 Jul 2024 16:44:02 +0100 Subject: [PATCH 03/11] Change column titles and order --- .../default_new_input/agent_objectives.csv | 2 +- .../example/default_new_input/agent_pairs.csv | 2 +- .../default_new_input/agent_regions.csv | 2 +- .../data/example/default_new_input/agents.csv | 2 +- .../data/example/default_new_input/assets.csv | 16 +-- .../example/default_new_input/commodities.csv | 2 +- .../default_new_input/commodity_costs.csv | 116 +++++++++--------- .../default_new_input/commodity_trade.csv | 2 +- .../data/example/default_new_input/demand.csv | 6 +- .../default_new_input/demand_slicing.csv | 14 +-- .../process_availabilities.csv | 12 +- .../default_new_input/process_flows.csv | 24 ++-- .../default_new_input/process_parameters.csv | 12 +- .../default_new_input/process_regions.csv | 2 +- .../example/default_new_input/processes.csv | 12 +- .../example/default_new_input/regions.csv | 2 +- .../example/default_new_input/sectors.csv | 2 +- 17 files changed, 115 insertions(+), 115 deletions(-) diff --git a/src/muse/data/example/default_new_input/agent_objectives.csv b/src/muse/data/example/default_new_input/agent_objectives.csv index c14612aaa..331c649c9 100644 --- a/src/muse/data/example/default_new_input/agent_objectives.csv +++ b/src/muse/data/example/default_new_input/agent_objectives.csv @@ -1,3 +1,3 @@ -agent,objective,objective_data,objective_sort +agent_id,objective,objective_data,objective_sort Agent1,LCOE,1,TRUE Agent2,LCOE,1,TRUE diff --git a/src/muse/data/example/default_new_input/agent_pairs.csv b/src/muse/data/example/default_new_input/agent_pairs.csv index 72e306f93..172632275 100644 --- a/src/muse/data/example/default_new_input/agent_pairs.csv +++ b/src/muse/data/example/default_new_input/agent_pairs.csv @@ -1,2 +1,2 @@ -name,new_agent,retrofit_agent,quantity +id,new_agent_id,retrofit_agent_id,quantity A1,Agent1,Agent2,1 diff --git a/src/muse/data/example/default_new_input/agent_regions.csv b/src/muse/data/example/default_new_input/agent_regions.csv index 257b59615..6a39852ea 100644 --- a/src/muse/data/example/default_new_input/agent_regions.csv +++ b/src/muse/data/example/default_new_input/agent_regions.csv @@ -1,2 +1,2 @@ -agent_pair,region +agent_pair_id,region_id A1,R1 diff --git a/src/muse/data/example/default_new_input/agents.csv b/src/muse/data/example/default_new_input/agents.csv index 2d1261bde..652e2d978 100644 --- a/src/muse/data/example/default_new_input/agents.csv +++ b/src/muse/data/example/default_new_input/agents.csv @@ -1,3 +1,3 @@ -agent,description,type,maturity_threshold,annual_cost_limit,search_rule,decision_rule +agent_id,description,type,maturity_threshold,annual_cost_limit,search_rule,decision_rule Agent1,New agent for A1,new,-1,inf,all,single Agent2,Retrofit agent for A1,retrofit,-1,inf,all,single diff --git a/src/muse/data/example/default_new_input/assets.csv b/src/muse/data/example/default_new_input/assets.csv index c30df1a13..8648bb891 100644 --- a/src/muse/data/example/default_new_input/assets.csv +++ b/src/muse/data/example/default_new_input/assets.csv @@ -1,8 +1,8 @@ -process_name,region,agent,capacity,year -gassupply1,R1,Agent2,15,2020 -gassupply1,R1,Agent2,15,2025 -gassupply1,R1,Agent2,7.5,2030 -gasCCGT,R1,Agent2,1,2020 -gasCCGT,R1,Agent2,1,2025 -gasboiler,R1,Agent2,10,2020 -gasboiler,R1,Agent2,5,2025 +agent_id,process_id,region_id,year,capacity +Agent2,gassupply1,R1,2020,15 +Agent2,gassupply1,R1,2025,15 +Agent2,gassupply1,R1,2030,7.5 +Agent2,gasCCGT,R1,2020,1 +Agent2,gasCCGT,R1,2025,1 +Agent2,gasboiler,R1,2020,10 +Agent2,gasboiler,R1,2025,5 diff --git a/src/muse/data/example/default_new_input/commodities.csv b/src/muse/data/example/default_new_input/commodities.csv index cec5cbf65..ac830346a 100644 --- a/src/muse/data/example/default_new_input/commodities.csv +++ b/src/muse/data/example/default_new_input/commodities.csv @@ -1,4 +1,4 @@ -commodity_name,description,type,unit +commodity_id,description,type,unit electricity,Electricity,energy,PJ gas,Gas,energy,PJ heat,Heat,energy,PJ diff --git a/src/muse/data/example/default_new_input/commodity_costs.csv b/src/muse/data/example/default_new_input/commodity_costs.csv index 85309f435..0a64542b3 100644 --- a/src/muse/data/example/default_new_input/commodity_costs.csv +++ b/src/muse/data/example/default_new_input/commodity_costs.csv @@ -1,58 +1,58 @@ -year,region,commodity_name,value -2010,R1,electricity,14.81481472 -2015,R1,electricity,17.89814806 -2020,R1,electricity,19.5 -2025,R1,electricity,21.93518528 -2030,R1,electricity,26.50925917 -2035,R1,electricity,26.51851861 -2040,R1,electricity,23.85185194 -2045,R1,electricity,23.97222222 -2050,R1,electricity,24.06481472 -2055,R1,electricity,25.3425925 -2060,R1,electricity,25.53703694 -2065,R1,electricity,25.32407417 -2070,R1,electricity,23.36111111 -2075,R1,electricity,22.27777778 -2080,R1,electricity,22.25925917 -2085,R1,electricity,22.17592583 -2090,R1,electricity,22.03703694 -2095,R1,electricity,21.94444444 -2100,R1,electricity,21.39814806 -2010,R1,gas,6.6759 -2015,R1,gas,6.914325 -2020,R1,gas,7.15275 -2025,R1,gas,8.10645 -2030,R1,gas,9.06015 -2035,R1,gas,9.2191 -2040,R1,gas,9.37805 -2045,R1,gas,9.193829337 -2050,R1,gas,9.009608674 -2055,R1,gas,8.832625604 -2060,R1,gas,8.655642534 -2065,R1,gas,8.485612708 -2070,R1,gas,8.315582883 -2075,R1,gas,8.152233126 -2080,R1,gas,7.988883368 -2085,R1,gas,7.831951236 -2090,R1,gas,7.675019103 -2095,R1,gas,7.524252461 -2100,R1,gas,7.373485819 -2010,R1,CO2f,0 -2015,R1,CO2f,0.052913851 -2020,R1,CO2f,0.08314119 -2025,R1,CO2f,0.120069795 -2030,R1,CO2f,0.156998399 -2035,R1,CO2f,0.214877567 -2040,R1,CO2f,0.272756734 -2045,R1,CO2f,0.35394801 -2050,R1,CO2f,0.435139285 -2055,R1,CO2f,0.542365578 -2060,R1,CO2f,0.649591871 -2065,R1,CO2f,0.780892624 -2070,R1,CO2f,0.912193378 -2075,R1,CO2f,1.078321687 -2080,R1,CO2f,1.244449995 -2085,R1,CO2f,1.4253503 -2090,R1,CO2f,1.606250604 -2095,R1,CO2f,1.73877515 -2100,R1,CO2f,1.871299697 +commodity_id,region_id,year,value +electricity,R1,2010,14.81481472 +electricity,R1,2015,17.89814806 +electricity,R1,2020,19.5 +electricity,R1,2025,21.93518528 +electricity,R1,2030,26.50925917 +electricity,R1,2035,26.51851861 +electricity,R1,2040,23.85185194 +electricity,R1,2045,23.97222222 +electricity,R1,2050,24.06481472 +electricity,R1,2055,25.3425925 +electricity,R1,2060,25.53703694 +electricity,R1,2065,25.32407417 +electricity,R1,2070,23.36111111 +electricity,R1,2075,22.27777778 +electricity,R1,2080,22.25925917 +electricity,R1,2085,22.17592583 +electricity,R1,2090,22.03703694 +electricity,R1,2095,21.94444444 +electricity,R1,2100,21.39814806 +gas,R1,2010,6.6759 +gas,R1,2015,6.914325 +gas,R1,2020,7.15275 +gas,R1,2025,8.10645 +gas,R1,2030,9.06015 +gas,R1,2035,9.2191 +gas,R1,2040,9.37805 +gas,R1,2045,9.193829337 +gas,R1,2050,9.009608674 +gas,R1,2055,8.832625604 +gas,R1,2060,8.655642534 +gas,R1,2065,8.485612708 +gas,R1,2070,8.315582883 +gas,R1,2075,8.152233126 +gas,R1,2080,7.988883368 +gas,R1,2085,7.831951236 +gas,R1,2090,7.675019103 +gas,R1,2095,7.524252461 +gas,R1,2100,7.373485819 +CO2f,R1,2010,0 +CO2f,R1,2015,0.052913851 +CO2f,R1,2020,0.08314119 +CO2f,R1,2025,0.120069795 +CO2f,R1,2030,0.156998399 +CO2f,R1,2035,0.214877567 +CO2f,R1,2040,0.272756734 +CO2f,R1,2045,0.35394801 +CO2f,R1,2050,0.435139285 +CO2f,R1,2055,0.542365578 +CO2f,R1,2060,0.649591871 +CO2f,R1,2065,0.780892624 +CO2f,R1,2070,0.912193378 +CO2f,R1,2075,1.078321687 +CO2f,R1,2080,1.244449995 +CO2f,R1,2085,1.4253503 +CO2f,R1,2090,1.606250604 +CO2f,R1,2095,1.73877515 +CO2f,R1,2100,1.871299697 diff --git a/src/muse/data/example/default_new_input/commodity_trade.csv b/src/muse/data/example/default_new_input/commodity_trade.csv index eb23c4b6c..092dd1559 100644 --- a/src/muse/data/example/default_new_input/commodity_trade.csv +++ b/src/muse/data/example/default_new_input/commodity_trade.csv @@ -1 +1 @@ -commodity,region,net_import,year +commodity_id,region_id,year,import,export diff --git a/src/muse/data/example/default_new_input/demand.csv b/src/muse/data/example/default_new_input/demand.csv index 13c64fe8c..b26c1b54d 100644 --- a/src/muse/data/example/default_new_input/demand.csv +++ b/src/muse/data/example/default_new_input/demand.csv @@ -1,3 +1,3 @@ -year,commodity_name,region,demand -2020,heat,R1,10 -2050,heat,R1,30 +commodity_id,region_id,year,demand +heat,R1,2020,10 +heat,R1,2050,30 diff --git a/src/muse/data/example/default_new_input/demand_slicing.csv b/src/muse/data/example/default_new_input/demand_slicing.csv index 10d4693c2..6877d5663 100644 --- a/src/muse/data/example/default_new_input/demand_slicing.csv +++ b/src/muse/data/example/default_new_input/demand_slicing.csv @@ -1,7 +1,7 @@ -commodity,region,timeslice,fraction,year -heat,R1,night,0.1, -heat,R1,morning,0.15, -heat,R1,afternoon,0.1, -heat,R1,early-peak,0.15, -heat,R1,late-peak,0.3, -heat,R1,evening,0.2, +commodity_id,region_id,year,timeslice,fraction +heat,R1,,night,0.1 +heat,R1,,morning,0.15 +heat,R1,,afternoon,0.1 +heat,R1,,early-peak,0.15 +heat,R1,,late-peak,0.3 +heat,R1,,evening,0.2 diff --git a/src/muse/data/example/default_new_input/process_availabilities.csv b/src/muse/data/example/default_new_input/process_availabilities.csv index 300a11407..1386d9db6 100644 --- a/src/muse/data/example/default_new_input/process_availabilities.csv +++ b/src/muse/data/example/default_new_input/process_availabilities.csv @@ -1,6 +1,6 @@ -process_name,timeslice,lim_type,value,year,region -gassupply1,ALL,UP,0.9,, -gasCCGT,ALL,UP,0.9,, -windturbine,ALL,UP,0.4,, -gasboiler,ALL,UP,1,, -heatpump,ALL,UP,1,, +process_id,region_id,year,timeslice,lim_type,value +gassupply1,,,ALL,UP,0.9 +gasCCGT,,,ALL,UP,0.9 +windturbine,,,ALL,UP,0.4 +gasboiler,,,ALL,UP,1 +heatpump,,,ALL,UP,1 diff --git a/src/muse/data/example/default_new_input/process_flows.csv b/src/muse/data/example/default_new_input/process_flows.csv index e6e72c67d..1c602f2d1 100644 --- a/src/muse/data/example/default_new_input/process_flows.csv +++ b/src/muse/data/example/default_new_input/process_flows.csv @@ -1,12 +1,12 @@ -process_name,commodity_name,flow,year,region -gassupply1,gas,1,, -gasCCGT,gas,-1.67,, -gasCCGT,electricity,1,, -gasCCGT,CO2f,91.67,, -windturbine,wind,-1,, -windturbine,electricity,1,, -gasboiler,gas,-1.16,, -gasboiler,heat,1,, -gasboiler,CO2f,64.71,, -heatpump,electricity,-0.4,, -heatpump,heat,1,, +process_id,commodity_id,region_id,year,flow +gassupply1,gas,,,1 +gasCCGT,gas,,,-1.67 +gasCCGT,electricity,,,1 +gasCCGT,CO2f,,,91.67 +windturbine,wind,,,-1 +windturbine,electricity,,,1 +gasboiler,gas,,,-1.16 +gasboiler,heat,,,1 +gasboiler,CO2f,,,64.71 +heatpump,electricity,,,-0.4 +heatpump,heat,,,1 diff --git a/src/muse/data/example/default_new_input/process_parameters.csv b/src/muse/data/example/default_new_input/process_parameters.csv index 00bb38bb7..5f162962a 100644 --- a/src/muse/data/example/default_new_input/process_parameters.csv +++ b/src/muse/data/example/default_new_input/process_parameters.csv @@ -1,6 +1,6 @@ -process,cap_par,cap_exp,fix_par,fix_exp,var_par,var_exp,max_capacity_addition,max_capacity_growth,total_capacity_limit,life,scaling_size,efficiency,discount_rate,year,region -gassupply1,0,1,0,1,2.55,1,5,1,60,35,0.00000189,86,0.1,, -gasCCGT,23.78234399,1,0,1,0,1,2,1,60,35,0.00000189,86,0.1,, -windturbine,36.30771182,1,0,1,0,1,2,1,60,25,0.00000189,86,0.1,, -gasboiler,3.8,1,0,1,0,1,10,0.02,60,10,0.00000189,86,0.1,, -heatpump,8.866667,1,0,1,0,1,10,0.02,60,10,0.00000189,86,0.1,, +process_id,region_id,year,cap_par,cap_exp,fix_par,fix_exp,var_par,var_exp,max_capacity_addition,max_capacity_growth,total_capacity_limit,life,scaling_size,efficiency,discount_rate +gassupply1,,,0,1,0,1,2.55,1,5,1,60,35,0.00000189,86,0.1 +gasCCGT,,,23.78234399,1,0,1,0,1,2,1,60,35,0.00000189,86,0.1 +windturbine,,,36.30771182,1,0,1,0,1,2,1,60,25,0.00000189,86,0.1 +gasboiler,,,3.8,1,0,1,0,1,10,0.02,60,10,0.00000189,86,0.1 +heatpump,,,8.866667,1,0,1,0,1,10,0.02,60,10,0.00000189,86,0.1 diff --git a/src/muse/data/example/default_new_input/process_regions.csv b/src/muse/data/example/default_new_input/process_regions.csv index e7ca8286c..8700ece5a 100644 --- a/src/muse/data/example/default_new_input/process_regions.csv +++ b/src/muse/data/example/default_new_input/process_regions.csv @@ -1,4 +1,4 @@ -process,region +process_id,region_id gassupply1,R1 gasCCGT,R1 windturbine,R1 diff --git a/src/muse/data/example/default_new_input/processes.csv b/src/muse/data/example/default_new_input/processes.csv index 7c4b9f818..e653e3ed6 100644 --- a/src/muse/data/example/default_new_input/processes.csv +++ b/src/muse/data/example/default_new_input/processes.csv @@ -1,6 +1,6 @@ -name,type,fuel,end_use,level,sector -gassupply1,energy,gas,gas,fixed,gas -gasCCGT,energy,gas,electricity,fixed,power -windturbine,energy,wind,electricity,fixed,power -gasboiler,energy,gas,heat,fixed,residential -heatpump,energy,electricity,heat,fixed,residential +id,sector_id,type,fuel,end_use,level +gassupply1,gas,energy,gas,gas,fixed +gasCCGT,power,energy,gas,electricity,fixed +windturbine,power,energy,wind,electricity,fixed +gasboiler,residential,energy,gas,heat,fixed +heatpump,residential,energy,electricity,heat,fixed diff --git a/src/muse/data/example/default_new_input/regions.csv b/src/muse/data/example/default_new_input/regions.csv index 1583e5334..1ce17d1ce 100644 --- a/src/muse/data/example/default_new_input/regions.csv +++ b/src/muse/data/example/default_new_input/regions.csv @@ -1,2 +1,2 @@ -name,description +id,description R1,Region 1 diff --git a/src/muse/data/example/default_new_input/sectors.csv b/src/muse/data/example/default_new_input/sectors.csv index a841328b6..7488adac9 100644 --- a/src/muse/data/example/default_new_input/sectors.csv +++ b/src/muse/data/example/default_new_input/sectors.csv @@ -1,4 +1,4 @@ -name,description +id,description gas,Gas sector power,Power sector residential,Residential sector From 01760a23fdfc3982c4d089aecc6323f82423fff5 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 3 Jul 2024 17:20:04 +0100 Subject: [PATCH 04/11] Correct id columns --- src/muse/data/example/default_new_input/agents.csv | 2 +- src/muse/data/example/default_new_input/commodities.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/muse/data/example/default_new_input/agents.csv b/src/muse/data/example/default_new_input/agents.csv index 652e2d978..a95e5b9dd 100644 --- a/src/muse/data/example/default_new_input/agents.csv +++ b/src/muse/data/example/default_new_input/agents.csv @@ -1,3 +1,3 @@ -agent_id,description,type,maturity_threshold,annual_cost_limit,search_rule,decision_rule +id,description,type,maturity_threshold,annual_cost_limit,search_rule,decision_rule Agent1,New agent for A1,new,-1,inf,all,single Agent2,Retrofit agent for A1,retrofit,-1,inf,all,single diff --git a/src/muse/data/example/default_new_input/commodities.csv b/src/muse/data/example/default_new_input/commodities.csv index ac830346a..5d87b119e 100644 --- a/src/muse/data/example/default_new_input/commodities.csv +++ b/src/muse/data/example/default_new_input/commodities.csv @@ -1,4 +1,4 @@ -commodity_id,description,type,unit +id,description,type,unit electricity,Electricity,energy,PJ gas,Gas,energy,PJ heat,Heat,energy,PJ From 08c9c70f24e094b4051a5e3972c4fd0325cbac3f Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Mon, 4 Aug 2025 15:38:13 +0100 Subject: [PATCH 05/11] Ignore default_new_input in regression tests --- src/muse/examples.py | 1 + tests/test_fullsim_regression.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/muse/examples.py b/src/muse/examples.py index 40b06ac94..6bfaa6e7e 100644 --- a/src/muse/examples.py +++ b/src/muse/examples.py @@ -52,6 +52,7 @@ "multiple_agents", "minimum_service", "trade", + "default_new_input", ] diff --git a/tests/test_fullsim_regression.py b/tests/test_fullsim_regression.py index 21897c660..878965c26 100644 --- a/tests/test_fullsim_regression.py +++ b/tests/test_fullsim_regression.py @@ -5,6 +5,9 @@ from muse.examples import AVAILABLE_EXAMPLES +# temporary skip for default_new_input as this is not yet working +AVAILABLE_EXAMPLES.pop("default_new_input") + @mark.regression @mark.example From 8059a2ea79a86ced1eb093ec2a5aed80bfe41f7d Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Mon, 4 Aug 2025 15:44:58 +0100 Subject: [PATCH 06/11] Fix typo in CO2 --- src/muse/data/example/default_new_input/commodities.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/muse/data/example/default_new_input/commodities.csv b/src/muse/data/example/default_new_input/commodities.csv index 5d87b119e..b4d546a74 100644 --- a/src/muse/data/example/default_new_input/commodities.csv +++ b/src/muse/data/example/default_new_input/commodities.csv @@ -3,4 +3,4 @@ electricity,Electricity,energy,PJ gas,Gas,energy,PJ heat,Heat,energy,PJ wind,Wind,energy,PJ -C02f,Carbon dioxide,energy,kt +CO2f,Carbon dioxide,energy,kt From 803a078f43a75a9f6470aa5f0b4fe05dbc75fcdc Mon Sep 17 00:00:00 2001 From: Christopher Cave-Ayland Date: Thu, 27 Jun 2024 17:20:23 +0100 Subject: [PATCH 07/11] First pass at duckdb data interface --- .../example/default_new_input/commodities.csv | 2 +- src/muse/new_input/readers.py | 76 +++++++++ tests/test_readers.py | 147 ++++++++++++++++++ 3 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 src/muse/new_input/readers.py diff --git a/src/muse/data/example/default_new_input/commodities.csv b/src/muse/data/example/default_new_input/commodities.csv index b4d546a74..09857851f 100644 --- a/src/muse/data/example/default_new_input/commodities.csv +++ b/src/muse/data/example/default_new_input/commodities.csv @@ -1,4 +1,4 @@ -id,description,type,unit +name,description,type,unit electricity,Electricity,energy,PJ gas,Gas,energy,PJ heat,Heat,energy,PJ diff --git a/src/muse/new_input/readers.py b/src/muse/new_input/readers.py new file mode 100644 index 000000000..eafa4fb07 --- /dev/null +++ b/src/muse/new_input/readers.py @@ -0,0 +1,76 @@ +import duckdb +import numpy as np +import xarray as xr + + +def read_inputs(data_dir): + data = {} + con = duckdb.connect(":memory:") + + with open(data_dir / "regions.csv") as f: + regions = read_regions_csv(f, con) # noqa: F841 + + with open(data_dir / "commodities.csv") as f: + commodities = read_commodities_csv(f, con) + + with open(data_dir / "demand.csv") as f: + demand = read_demand_csv(f, con) # noqa: F841 + + data["global_commodities"] = calculate_global_commodities(commodities) + return data + + +def read_regions_csv(buffer_, con): + sql = """CREATE TABLE regions ( + name VARCHAR PRIMARY KEY, + ); + """ + con.sql(sql) + rel = con.read_csv(buffer_, header=True, delimiter=",") # noqa: F841 + con.sql("INSERT INTO regions SELECT name FROM rel;") + return con.sql("SELECT name from regions").fetchnumpy() + + +def read_commodities_csv(buffer_, con): + sql = """CREATE TABLE commodities ( + name VARCHAR PRIMARY KEY, + type VARCHAR CHECK (type IN ('energy', 'service', 'material', 'environmental')), + unit VARCHAR, + ); + """ + con.sql(sql) + rel = con.read_csv(buffer_, header=True, delimiter=",") # noqa: F841 + con.sql("INSERT INTO commodities SELECT name, type, unit FROM rel;") + + return con.sql("select name, type, unit from commodities").fetchnumpy() + + +def calculate_global_commodities(commodities): + names = commodities["name"].astype(np.dtype("str")) + types = commodities["type"].astype(np.dtype("str")) + units = commodities["unit"].astype(np.dtype("str")) + + type_array = xr.DataArray( + data=types, dims=["commodity"], coords=dict(commodity=names) + ) + + unit_array = xr.DataArray( + data=units, dims=["commodity"], coords=dict(commodity=names) + ) + + data = xr.Dataset(data_vars=dict(type=type_array, unit=unit_array)) + return data + + +def read_demand_csv(buffer_, con): + sql = """CREATE TABLE demand ( + year BIGINT, + commodity VARCHAR REFERENCES commodities(name), + region VARCHAR REFERENCES regions(name), + demand DOUBLE, + ); + """ + con.sql(sql) + rel = con.read_csv(buffer_, header=True, delimiter=",") # noqa: F841 + con.sql("INSERT INTO demand SELECT year, commodity_name, region, demand FROM rel;") + return con.sql("SELECT * from demand").fetchnumpy() diff --git a/tests/test_readers.py b/tests/test_readers.py index 924dcacff..c36938b29 100644 --- a/tests/test_readers.py +++ b/tests/test_readers.py @@ -1,7 +1,10 @@ +from io import StringIO from itertools import chain, permutations from pathlib import Path import pandas as pd +import duckdb +import numpy as np import toml import xarray as xr from pytest import fixture, mark, raises @@ -314,3 +317,147 @@ def test_get_nan_coordinates(): dataset3 = xr.Dataset.from_dataframe(df3.set_index(["region", "year"])) nan_coords3 = get_nan_coordinates(dataset3) assert nan_coords3 == [] + + +@fixture +def default_new_input(tmp_path): + from muse.examples import copy_model + + copy_model("default_new_input", tmp_path) + return tmp_path / "model" + + +@fixture +def con(): + return duckdb.connect(":memory:") + + +@fixture +def populate_regions(default_new_input, con): + from muse.new_input.readers import read_regions_csv + + with open(default_new_input / "regions.csv") as f: + return read_regions_csv(f, con) + + +@fixture +def populate_commodities(default_new_input, con): + from muse.new_input.readers import read_commodities_csv + + with open(default_new_input / "commodities.csv") as f: + return read_commodities_csv(f, con) + + +@fixture +def populate_demand(default_new_input, con, populate_regions, populate_commodities): + from muse.new_input.readers import read_demand_csv + + with open(default_new_input / "demand.csv") as f: + return read_demand_csv(f, con) + + +def test_read_regions(populate_regions): + assert populate_regions["name"] == np.array(["R1"]) + + +def test_read_new_global_commodities(populate_commodities): + data = populate_commodities + assert list(data["name"]) == ["electricity", "gas", "heat", "wind", "CO2f"] + assert list(data["type"]) == ["energy"] * 5 + assert list(data["unit"]) == ["PJ"] * 4 + ["kt"] + + +def test_calculate_global_commodities(populate_commodities): + from muse.new_input.readers import calculate_global_commodities + + data = calculate_global_commodities(populate_commodities) + + assert isinstance(data, xr.Dataset) + assert set(data.dims) == {"commodity"} + for dt in data.dtypes.values(): + assert np.issubdtype(dt, np.dtype("str")) + + assert list(data.coords["commodity"].values) == list(populate_commodities["name"]) + assert list(data.data_vars["type"].values) == list(populate_commodities["type"]) + assert list(data.data_vars["unit"].values) == list(populate_commodities["unit"]) + + +def test_read_new_global_commodities_type_constraint(default_new_input, con): + from muse.new_input.readers import read_commodities_csv + + csv = StringIO("name,type,unit\nfoo,invalid,bar\n") + with raises(duckdb.ConstraintException): + read_commodities_csv(csv, con) + + +def test_new_read_demand_csv(populate_demand): + data = populate_demand + assert np.all(data["year"] == np.array([2020, 2050])) + assert np.all(data["commodity"] == np.array(["heat", "heat"])) + assert np.all(data["region"] == np.array(["R1", "R1"])) + assert np.all(data["demand"] == np.array([10, 30])) + + +def test_new_read_demand_csv_commodity_constraint( + default_new_input, con, populate_commodities, populate_regions +): + from muse.new_input.readers import read_demand_csv + + csv = StringIO("year,commodity_name,region,demand\n2020,invalid,R1,0\n") + with raises(duckdb.ConstraintException, match=".*foreign key.*"): + read_demand_csv(csv, con) + + +def test_new_read_demand_csv_region_constraint( + default_new_input, con, populate_commodities, populate_regions +): + from muse.new_input.readers import read_demand_csv + + csv = StringIO("year,commodity_name,region,demand\n2020,heat,invalid,0\n") + with raises(duckdb.ConstraintException, match=".*foreign key.*"): + read_demand_csv(csv, con) + + +@mark.xfail +def test_demand_dataset(default_new_input): + import duckdb + from muse.new_input.readers import read_commodities, read_demand, read_regions + + con = duckdb.connect(":memory:") + + read_regions(default_new_input, con) + read_commodities(default_new_input, con) + data = read_demand(default_new_input, con) + + assert isinstance(data, xr.DataArray) + assert data.dtype == np.float64 + + assert set(data.dims) == {"year", "commodity", "region", "timeslice"} + assert list(data.coords["region"].values) == ["R1"] + assert list(data.coords["timeslice"].values) == list(range(1, 7)) + assert list(data.coords["year"].values) == [2020, 2050] + assert set(data.coords["commodity"].values) == { + "electricity", + "gas", + "heat", + "wind", + "CO2f", + } + + assert data.sel(year=2020, commodity="electricity", region="R1", timeslice=0) == 1 + + +@mark.xfail +def test_new_read_initial_market(default_new_input): + from muse.new_input.readers import read_inputs + + all_data = read_inputs(default_new_input) + data = all_data["initial_market"] + + assert isinstance(data, xr.Dataset) + assert set(data.dims) == {"region", "year", "commodity", "timeslice"} + assert dict(data.dtypes) == dict( + prices=np.float64, + exports=np.float64, + imports=np.float64, + static_trade=np.float64, From 4e861be9f892794aaf1151d794bd318e28cbfb34 Mon Sep 17 00:00:00 2001 From: Christopher Cave-Ayland Date: Fri, 28 Jun 2024 11:38:45 +0100 Subject: [PATCH 08/11] Define db schema using SQLAlchemy --- pyproject.toml | 4 ++- src/muse/new_input/readers.py | 67 ++++++++++++++++++++++------------- tests/test_readers.py | 55 ++++++++++++++++++++++++++-- 3 files changed, 97 insertions(+), 29 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 43220c550..0a34847c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,9 @@ dependencies = [ "pypubsub", "tomlkit", "duckdb", - "fsspec" + "fsspec", + "sqlalchemy", + "duckdb-engine" ] dynamic = ["version"] diff --git a/src/muse/new_input/readers.py b/src/muse/new_input/readers.py index eafa4fb07..ee17d0228 100644 --- a/src/muse/new_input/readers.py +++ b/src/muse/new_input/readers.py @@ -1,11 +1,48 @@ -import duckdb import numpy as np import xarray as xr +from sqlalchemy import CheckConstraint, ForeignKey +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + +class TableBase(DeclarativeBase): + pass + + +class Regions(TableBase): + __tablename__ = "regions" + + name: Mapped[str] = mapped_column(primary_key=True) + + +class Commodities(TableBase): + __tablename__ = "commodities" + + name: Mapped[str] = mapped_column(primary_key=True) + type: Mapped[str] = mapped_column( + CheckConstraint("type IN ('energy', 'service', 'material', 'environmental')") + ) + unit: Mapped[str] + + +class Demand(TableBase): + __tablename__ = "demand" + + year: Mapped[int] = mapped_column(primary_key=True, autoincrement=False) + commodity: Mapped[Commodities] = mapped_column( + ForeignKey("commodities.name"), primary_key=True + ) + region: Mapped[Regions] = mapped_column( + ForeignKey("regions.name"), primary_key=True + ) + demand: Mapped[float] def read_inputs(data_dir): - data = {} - con = duckdb.connect(":memory:") + from sqlalchemy import create_engine + + engine = create_engine("duckdb:///:memory:") + TableBase.metadata.create_all(engine) + con = engine.raw_connection().driver_connection with open(data_dir / "regions.csv") as f: regions = read_regions_csv(f, con) # noqa: F841 @@ -16,32 +53,20 @@ def read_inputs(data_dir): with open(data_dir / "demand.csv") as f: demand = read_demand_csv(f, con) # noqa: F841 + data = {} data["global_commodities"] = calculate_global_commodities(commodities) return data def read_regions_csv(buffer_, con): - sql = """CREATE TABLE regions ( - name VARCHAR PRIMARY KEY, - ); - """ - con.sql(sql) rel = con.read_csv(buffer_, header=True, delimiter=",") # noqa: F841 - con.sql("INSERT INTO regions SELECT name FROM rel;") + con.execute("INSERT INTO regions SELECT name FROM rel;") return con.sql("SELECT name from regions").fetchnumpy() def read_commodities_csv(buffer_, con): - sql = """CREATE TABLE commodities ( - name VARCHAR PRIMARY KEY, - type VARCHAR CHECK (type IN ('energy', 'service', 'material', 'environmental')), - unit VARCHAR, - ); - """ - con.sql(sql) rel = con.read_csv(buffer_, header=True, delimiter=",") # noqa: F841 con.sql("INSERT INTO commodities SELECT name, type, unit FROM rel;") - return con.sql("select name, type, unit from commodities").fetchnumpy() @@ -63,14 +88,6 @@ def calculate_global_commodities(commodities): def read_demand_csv(buffer_, con): - sql = """CREATE TABLE demand ( - year BIGINT, - commodity VARCHAR REFERENCES commodities(name), - region VARCHAR REFERENCES regions(name), - demand DOUBLE, - ); - """ - con.sql(sql) rel = con.read_csv(buffer_, header=True, delimiter=",") # noqa: F841 con.sql("INSERT INTO demand SELECT year, commodity_name, region, demand FROM rel;") return con.sql("SELECT * from demand").fetchnumpy() diff --git a/tests/test_readers.py b/tests/test_readers.py index c36938b29..812031d00 100644 --- a/tests/test_readers.py +++ b/tests/test_readers.py @@ -329,7 +329,14 @@ def default_new_input(tmp_path): @fixture def con(): - return duckdb.connect(":memory:") + from muse.new_input.readers import TableBase + from sqlalchemy import create_engine + from sqlalchemy.orm import Session + + engine = create_engine("duckdb:///:memory:") + session = Session(engine) + TableBase.metadata.create_all(engine) + return session.connection().connection @fixture @@ -360,7 +367,15 @@ def test_read_regions(populate_regions): assert populate_regions["name"] == np.array(["R1"]) -def test_read_new_global_commodities(populate_commodities): +def test_read_regions_primary_key_constraint(default_new_input, con): + from muse.new_input.readers import read_regions_csv + + csv = StringIO("name\nR1\nR1\n") + with raises(duckdb.ConstraintException, match=".*duplicate key.*"): + read_regions_csv(csv, con) + + +def test_read_new_commodities(populate_commodities): data = populate_commodities assert list(data["name"]) == ["electricity", "gas", "heat", "wind", "CO2f"] assert list(data["type"]) == ["energy"] * 5 @@ -382,7 +397,15 @@ def test_calculate_global_commodities(populate_commodities): assert list(data.data_vars["unit"].values) == list(populate_commodities["unit"]) -def test_read_new_global_commodities_type_constraint(default_new_input, con): +def test_read_new_commodities_primary_key_constraint(default_new_input, con): + from muse.new_input.readers import read_commodities_csv + + csv = StringIO("name,type,unit\nfoo,energy,bar\nfoo,energy,bar\n") + with raises(duckdb.ConstraintException, match=".*duplicate key.*"): + read_commodities_csv(csv, con) + + +def test_read_new_commodities_type_constraint(default_new_input, con): from muse.new_input.readers import read_commodities_csv csv = StringIO("name,type,unit\nfoo,invalid,bar\n") @@ -418,6 +441,32 @@ def test_new_read_demand_csv_region_constraint( read_demand_csv(csv, con) +def test_new_read_demand_csv_primary_key_constraint( + default_new_input, con, populate_commodities, populate_regions +): + from muse.new_input.readers import read_demand_csv, read_regions_csv + + # Add another region so we can test varying it as a primary key + csv = StringIO("name\nR2\n") + read_regions_csv(csv, con) + + # all fine so long as one primary key column differs + csv = StringIO( + """year,commodity_name,region,demand +2020,gas,R1,0 +2021,gas,R1,0 +2020,heat,R1,0 +2020,gas,R2,0 +""" + ) + read_demand_csv(csv, con) + + # no good if all primary key columns match a previous entry + csv = StringIO("year,commodity_name,region,demand\n2020,gas,R1,0") + with raises(duckdb.ConstraintException, match=".*duplicate key.*"): + read_demand_csv(csv, con) + + @mark.xfail def test_demand_dataset(default_new_input): import duckdb From 1c0d7429c6d7d93f547c345064e0a32239748236 Mon Sep 17 00:00:00 2001 From: Christopher Cave-Ayland Date: Fri, 28 Jun 2024 13:14:02 +0100 Subject: [PATCH 09/11] Adopt generic read_csv function --- src/muse/new_input/readers.py | 32 ++++++++++++---------------- tests/test_readers.py | 40 +++++++++++++++++------------------ 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/src/muse/new_input/readers.py b/src/muse/new_input/readers.py index ee17d0228..5c2dddde2 100644 --- a/src/muse/new_input/readers.py +++ b/src/muse/new_input/readers.py @@ -29,7 +29,9 @@ class Demand(TableBase): year: Mapped[int] = mapped_column(primary_key=True, autoincrement=False) commodity: Mapped[Commodities] = mapped_column( - ForeignKey("commodities.name"), primary_key=True + ForeignKey("commodities.name"), + primary_key=True, + info=dict(header="commodity_name"), ) region: Mapped[Regions] = mapped_column( ForeignKey("regions.name"), primary_key=True @@ -45,29 +47,29 @@ def read_inputs(data_dir): con = engine.raw_connection().driver_connection with open(data_dir / "regions.csv") as f: - regions = read_regions_csv(f, con) # noqa: F841 + regions = read_csv(f, Regions, con) # noqa: F841 with open(data_dir / "commodities.csv") as f: - commodities = read_commodities_csv(f, con) + commodities = read_csv(f, Commodities, con) with open(data_dir / "demand.csv") as f: - demand = read_demand_csv(f, con) # noqa: F841 + demand = read_csv(f, Demand, con) # noqa: F841 data = {} data["global_commodities"] = calculate_global_commodities(commodities) return data -def read_regions_csv(buffer_, con): - rel = con.read_csv(buffer_, header=True, delimiter=",") # noqa: F841 - con.execute("INSERT INTO regions SELECT name FROM rel;") - return con.sql("SELECT name from regions").fetchnumpy() - +def read_csv(buffer_, table_class, con): + table_name = table_class.__tablename__ + columns = ", ".join( + column.info.get("header", column.name) + for column in table_class.__table__.columns + ) -def read_commodities_csv(buffer_, con): rel = con.read_csv(buffer_, header=True, delimiter=",") # noqa: F841 - con.sql("INSERT INTO commodities SELECT name, type, unit FROM rel;") - return con.sql("select name, type, unit from commodities").fetchnumpy() + con.execute(f"INSERT INTO {table_name} SELECT {columns} FROM rel") + return con.execute(f"SELECT * from {table_name}").fetchnumpy() def calculate_global_commodities(commodities): @@ -85,9 +87,3 @@ def calculate_global_commodities(commodities): data = xr.Dataset(data_vars=dict(type=type_array, unit=unit_array)) return data - - -def read_demand_csv(buffer_, con): - rel = con.read_csv(buffer_, header=True, delimiter=",") # noqa: F841 - con.sql("INSERT INTO demand SELECT year, commodity_name, region, demand FROM rel;") - return con.sql("SELECT * from demand").fetchnumpy() diff --git a/tests/test_readers.py b/tests/test_readers.py index 812031d00..5d57e7156 100644 --- a/tests/test_readers.py +++ b/tests/test_readers.py @@ -341,26 +341,26 @@ def con(): @fixture def populate_regions(default_new_input, con): - from muse.new_input.readers import read_regions_csv + from muse.new_input.readers import Regions, read_csv with open(default_new_input / "regions.csv") as f: - return read_regions_csv(f, con) + return read_csv(f, Regions, con) @fixture def populate_commodities(default_new_input, con): - from muse.new_input.readers import read_commodities_csv + from muse.new_input.readers import Commodities, read_csv with open(default_new_input / "commodities.csv") as f: - return read_commodities_csv(f, con) + return read_csv(f, Commodities, con) @fixture def populate_demand(default_new_input, con, populate_regions, populate_commodities): - from muse.new_input.readers import read_demand_csv + from muse.new_input.readers import Demand, read_csv with open(default_new_input / "demand.csv") as f: - return read_demand_csv(f, con) + return read_csv(f, Demand, con) def test_read_regions(populate_regions): @@ -368,11 +368,11 @@ def test_read_regions(populate_regions): def test_read_regions_primary_key_constraint(default_new_input, con): - from muse.new_input.readers import read_regions_csv + from muse.new_input.readers import Regions, read_csv csv = StringIO("name\nR1\nR1\n") with raises(duckdb.ConstraintException, match=".*duplicate key.*"): - read_regions_csv(csv, con) + read_csv(csv, Regions, con) def test_read_new_commodities(populate_commodities): @@ -398,19 +398,19 @@ def test_calculate_global_commodities(populate_commodities): def test_read_new_commodities_primary_key_constraint(default_new_input, con): - from muse.new_input.readers import read_commodities_csv + from muse.new_input.readers import Commodities, read_csv csv = StringIO("name,type,unit\nfoo,energy,bar\nfoo,energy,bar\n") with raises(duckdb.ConstraintException, match=".*duplicate key.*"): - read_commodities_csv(csv, con) + read_csv(csv, Commodities, con) def test_read_new_commodities_type_constraint(default_new_input, con): - from muse.new_input.readers import read_commodities_csv + from muse.new_input.readers import Commodities, read_csv csv = StringIO("name,type,unit\nfoo,invalid,bar\n") with raises(duckdb.ConstraintException): - read_commodities_csv(csv, con) + read_csv(csv, Commodities, con) def test_new_read_demand_csv(populate_demand): @@ -424,31 +424,31 @@ def test_new_read_demand_csv(populate_demand): def test_new_read_demand_csv_commodity_constraint( default_new_input, con, populate_commodities, populate_regions ): - from muse.new_input.readers import read_demand_csv + from muse.new_input.readers import Demand, read_csv csv = StringIO("year,commodity_name,region,demand\n2020,invalid,R1,0\n") with raises(duckdb.ConstraintException, match=".*foreign key.*"): - read_demand_csv(csv, con) + read_csv(csv, Demand, con) def test_new_read_demand_csv_region_constraint( default_new_input, con, populate_commodities, populate_regions ): - from muse.new_input.readers import read_demand_csv + from muse.new_input.readers import Demand, read_csv csv = StringIO("year,commodity_name,region,demand\n2020,heat,invalid,0\n") with raises(duckdb.ConstraintException, match=".*foreign key.*"): - read_demand_csv(csv, con) + read_csv(csv, Demand, con) def test_new_read_demand_csv_primary_key_constraint( default_new_input, con, populate_commodities, populate_regions ): - from muse.new_input.readers import read_demand_csv, read_regions_csv + from muse.new_input.readers import Demand, Regions, read_csv # Add another region so we can test varying it as a primary key csv = StringIO("name\nR2\n") - read_regions_csv(csv, con) + read_csv(csv, Regions, con) # all fine so long as one primary key column differs csv = StringIO( @@ -459,12 +459,12 @@ def test_new_read_demand_csv_primary_key_constraint( 2020,gas,R2,0 """ ) - read_demand_csv(csv, con) + read_csv(csv, Demand, con) # no good if all primary key columns match a previous entry csv = StringIO("year,commodity_name,region,demand\n2020,gas,R1,0") with raises(duckdb.ConstraintException, match=".*duplicate key.*"): - read_demand_csv(csv, con) + read_csv(csv, Demand, con) @mark.xfail From 3ba664b220f54ad779278b345610d48e183a671e Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Mon, 4 Aug 2025 15:19:33 +0100 Subject: [PATCH 10/11] Fix merge mistake --- tests/test_readers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_readers.py b/tests/test_readers.py index 5d57e7156..9148969b1 100644 --- a/tests/test_readers.py +++ b/tests/test_readers.py @@ -2,9 +2,9 @@ from itertools import chain, permutations from pathlib import Path -import pandas as pd import duckdb import numpy as np +import pandas as pd import toml import xarray as xr from pytest import fixture, mark, raises @@ -329,10 +329,11 @@ def default_new_input(tmp_path): @fixture def con(): - from muse.new_input.readers import TableBase from sqlalchemy import create_engine from sqlalchemy.orm import Session + from muse.new_input.readers import TableBase + engine = create_engine("duckdb:///:memory:") session = Session(engine) TableBase.metadata.create_all(engine) @@ -470,6 +471,7 @@ def test_new_read_demand_csv_primary_key_constraint( @mark.xfail def test_demand_dataset(default_new_input): import duckdb + from muse.new_input.readers import read_commodities, read_demand, read_regions con = duckdb.connect(":memory:") @@ -510,3 +512,4 @@ def test_new_read_initial_market(default_new_input): exports=np.float64, imports=np.float64, static_trade=np.float64, + ) From e9361daecdcdab6177341cd481e4313fc3647ff0 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Fri, 8 Aug 2025 12:37:48 +0100 Subject: [PATCH 11/11] Fix popping error --- tests/test_fullsim_regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_fullsim_regression.py b/tests/test_fullsim_regression.py index 878965c26..49197e704 100644 --- a/tests/test_fullsim_regression.py +++ b/tests/test_fullsim_regression.py @@ -6,7 +6,7 @@ from muse.examples import AVAILABLE_EXAMPLES # temporary skip for default_new_input as this is not yet working -AVAILABLE_EXAMPLES.pop("default_new_input") +AVAILABLE_EXAMPLES.remove("default_new_input") @mark.regression