From dfe67146a4d68628e37c5b3d21e6fc796d441e73 Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Fri, 10 Jul 2026 18:02:01 +0200 Subject: [PATCH 01/14] implement cosmocov --- scratch/lgoh/cov_config_test.yaml | 27 ++ .../EwMC4wX25iaW5zPTIwX3dsLmRhdA== | 1 + .../AuMF9uYmlucz0yMF93bC5kYXQ= | 1 + .../AuMF9uYmlucz0yMF93bC5pbmk= | 1 + .../EwMC4wX25iaW5zPTIwX3dsLmRhdA== | 1 + .../EwMC4wX25iaW5zPTIwX3dsLmluaQ== | 1 + ...NlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 | 1 + ...NlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p | 1 + workflow/Snakefile | 8 +- workflow/common.py | 97 +++- workflow/rules/covariance.smk | 421 +++++++++++------- 11 files changed, 376 insertions(+), 184 deletions(-) create mode 100644 scratch/lgoh/cov_config_test.yaml create mode 100644 workflow/.snakemake/incomplete/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== create mode 100644 workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5kYXQ= create mode 100644 workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5pbmk= create mode 100644 workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== create mode 100644 workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmluaQ== create mode 100644 workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 create mode 100644 workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p diff --git a/scratch/lgoh/cov_config_test.yaml b/scratch/lgoh/cov_config_test.yaml new file mode 100644 index 000000000..ebe8462cd --- /dev/null +++ b/scratch/lgoh/cov_config_test.yaml @@ -0,0 +1,27 @@ +fiducial: + version: SP_v1.4.6 + blind: A + min_sep: 1.0 + max_sep: 100.0 + nbins: 20 + npatch: 30 + mock_version: SP_v1.4.6 + gaussian: g + +probe_3x2pt: wl +glass_mocks: + version: "v0" + seed_range: [1, 350] + +covariance: + default_masked: True + +tools: + python_executable: /n23data1/n06data/lgoh/scratch/my_env_new/bin/python + onecov_executable: /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py + +SP_v1.4.6: + cov_th: + sigma_e: 0.28, 0.28 + n_e: 5.0, 5.0 + A: 1000 \ No newline at end of file diff --git a/workflow/.snakemake/incomplete/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== b/workflow/.snakemake/incomplete/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== new file mode 100644 index 000000000..5d3401dcf --- /dev/null +++ b/workflow/.snakemake/incomplete/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== @@ -0,0 +1 @@ +{"external_jobid": null} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5kYXQ= b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5kYXQ= new file mode 100644 index 000000000..cf9de83bd --- /dev/null +++ b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5kYXQ= @@ -0,0 +1 @@ +{"rule": "covariance_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.ini"], "log": [], "shellcmd": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n\n cd /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl\n /n23data1/n06data/lgoh/scratch/my_env_new/bin/python /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.ini\n ", "params": ["'/n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.ini'", "'/n23data1/n06data/lgoh/scratch/my_env_new/bin/python'"], "code": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n\n cd {params.outdir}\n {params.python_executable} {params.onecov} {params.ini_path}\n ", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8739027612265, "starttime": 1783698870.0182004, "endtime": 1783698870.015105, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.ini": "sha256:2eb8d026dbff910f9cf3c6d725dca5527d0995e8304710b4534085ea64dca295"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5pbmk= b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5pbmk= new file mode 100644 index 000000000..75b02c39c --- /dev/null +++ b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5pbmk= @@ -0,0 +1 @@ +{"rule": "covariance_ini_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json"], "log": [], "shellcmd": "\n mkdir -p /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl\n\n cat > \"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.ini\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = 0\nssc = 0\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = False\nggl = False\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl\nfile = covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.dat\nstyle = matrix\nsave_configs = False\ncorrmatrix_plot = covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_corr_plot.pdf\n\n[covTHETAspace settings]\ntheta_min = 1.0\ntheta_max = 100.0\ntheta_bins = 20\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = \nmask_file_lensing = \nmask_file_clust = \nmask_file_ggl = \nellipticity_dispersion = 0.28, 0.28\nsurvey_area_lensing_in_deg2 = 1000\n\n\nn_eff_lensing = 5.0, 5.0\n\n\n[redshift]\nzlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/\n# /n17data/sguerrini/UNIONS/WL/nz/v1.4.6\nzlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii\n# nz_SP_v1.4.6_A.txt\nzclust_file = \nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = 0.8111\nh = 0.6736\nomega_m = 0.3153\nomega_b = 0.0493\nomega_de = 0.6847\nw0 = -1.0\nwa = 0.0\nns = 0.9649\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = 7.75\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "params": ["''", "''", "''", "''", "''", "''", "''", "''", "'/n17data/sguerrini/UNIONS/WL/nz/v1.4.6'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl'", "'0'", "'0.28, 0.28'", "'False'", "'False'", "'covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.dat'", "'covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_corr_plot.pdf'", "'n_eff_lensing = 5.0, 5.0'", "'nz_SP_v1.4.6_A.txt'", "'survey_area_lensing_in_deg2 = 1000'", "0.0493", "0.3153", "0.6736", "0.6847", "0.8111", "0.9649", "7.75"], "code": "\n mkdir -p {params.outdir}\n\n cat > \"{output}\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = {params.ng_value}\nssc = {params.ng_value}\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = {params.do_clustering}\nggl = {params.do_ggl}\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = {params.outdir}\nfile = {params.out_filename}\nstyle = matrix\nsave_configs = False\ncorrmatrix_plot = {params.out_plot_filename}\n\n[covTHETAspace settings]\ntheta_min = {wildcards.min_sep}\ntheta_max = {wildcards.max_sep}\ntheta_bins = {wildcards.nbins}\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = {params.mask_dir}\nmask_file_lensing = {params.mask_lens}\nmask_file_clust = {params.mask_clust}\nmask_file_ggl = {params.mask_ggl}\nellipticity_dispersion = {params.sigma_e_param}\n{params.area_lens_line}\n{params.area_clust_line}\n{params.area_ggl_line}\n{params.n_e_lens_line}\n{params.n_e_clust_line}\n\n[redshift]\nzlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/\n# {params.nz_dir}\nzlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii\n# {params.nz_lens}\nzclust_file = {params.nz_clust}\nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = {params.sigma_8}\nh = {params.h}\nomega_m = {params.omega_m}\nomega_b = {params.omega_b}\nomega_de = {params.omega_v}\nw0 = -1.0\nwa = 0.0\nns = {params.n_s}\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = {params.hmcode_logT_AGN}\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "record_format_version": 6, "conda_env": null, "container_img_url": "/n17data/cdaley/containers/containers", "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8739027612384, "starttime": 1783698635.9245446, "endtime": 1783698633.455901, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json": "sha256:c44a19ac7749152357d3bb7bb6781783c0da2f76083c7783abfc5377ec296703"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== new file mode 100644 index 000000000..c32d15ddd --- /dev/null +++ b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== @@ -0,0 +1 @@ +{"rule": "covariance_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.ini"], "log": [], "shellcmd": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n source activate /n23data1/n06data/lgoh/scratch/my_env_new\n cd /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl\n /n23data1/n06data/lgoh/scratch/my_env_new/bin/python /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.ini\n ", "params": ["'/n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.ini'"], "code": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n source activate /n23data1/n06data/lgoh/scratch/my_env_new\n cd {params.outdir}\n /n23data1/n06data/lgoh/scratch/my_env_new/bin/python {params.onecov} {params.ini_path}\n ", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8749163604585, "starttime": 1783694135.7577267, "endtime": 1783696080.0747812, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.ini": "sha256:6def65cf3f6f7510a577bc07fb8ea89d57a1267a884513131b52ffa5175681db"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmluaQ== b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmluaQ== new file mode 100644 index 000000000..467c15af5 --- /dev/null +++ b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmluaQ== @@ -0,0 +1 @@ +{"rule": "covariance_ini_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json"], "log": [], "shellcmd": "\n mkdir -p /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl\n\n cat > \"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.ini\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = 1\nssc = 1\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = False\nggl = False\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl\nfile = covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.dat\nstyle = matrix\nsave_configs = False\ncorrmatrix_plot = covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_corr_plot.pdf\n\n[covTHETAspace settings]\ntheta_min = 1.0\ntheta_max = 100.0\ntheta_bins = 20\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = \nmask_file_lensing = \nmask_file_clust = \nmask_file_ggl = \nellipticity_dispersion = 0.28, 0.28\nsurvey_area_lensing_in_deg2 = 1000\n\n\nn_eff_lensing = 5.0, 5.0\n\n\n[redshift]\nzlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/\n# /n17data/sguerrini/UNIONS/WL/nz/v1.4.6\nzlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii\n# nz_SP_v1.4.6_A.txt\nzclust_file = \nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = 0.8111\nh = 0.6736\nomega_m = 0.3153\nomega_b = 0.0493\nomega_de = 0.6847\nw0 = -1.0\nwa = 0.0\nns = 0.9649\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = 7.75\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "params": ["''", "''", "''", "''", "''", "''", "''", "''", "'/n17data/sguerrini/UNIONS/WL/nz/v1.4.6'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl'", "'0.28, 0.28'", "'1'", "'False'", "'False'", "'covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.dat'", "'covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_corr_plot.pdf'", "'n_eff_lensing = 5.0, 5.0'", "'nz_SP_v1.4.6_A.txt'", "'survey_area_lensing_in_deg2 = 1000'", "0.0493", "0.3153", "0.6736", "0.6847", "0.8111", "0.9649", "7.75"], "code": "\n mkdir -p {params.outdir}\n\n cat > \"{output}\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = {params.ng_value}\nssc = {params.ng_value}\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = {params.do_clustering}\nggl = {params.do_ggl}\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = {params.outdir}\nfile = {params.out_filename}\nstyle = matrix\nsave_configs = False\ncorrmatrix_plot = {params.out_plot_filename}\n\n[covTHETAspace settings]\ntheta_min = {wildcards.min_sep}\ntheta_max = {wildcards.max_sep}\ntheta_bins = {wildcards.nbins}\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = {params.mask_dir}\nmask_file_lensing = {params.mask_lens}\nmask_file_clust = {params.mask_clust}\nmask_file_ggl = {params.mask_ggl}\nellipticity_dispersion = {params.sigma_e_param}\n{params.area_lens_line}\n{params.area_clust_line}\n{params.area_ggl_line}\n{params.n_e_lens_line}\n{params.n_e_clust_line}\n\n[redshift]\nzlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/\n# {params.nz_dir}\nzlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii\n# {params.nz_lens}\nzclust_file = {params.nz_clust}\nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = {params.sigma_8}\nh = {params.h}\nomega_m = {params.omega_m}\nomega_b = {params.omega_b}\nomega_de = {params.omega_v}\nw0 = -1.0\nwa = 0.0\nns = {params.n_s}\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = {params.hmcode_logT_AGN}\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "record_format_version": 6, "conda_env": null, "container_img_url": "/n17data/cdaley/containers/containers", "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8756651165425, "starttime": 1783694134.5319173, "endtime": 1783698274.2580187, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json": "sha256:c44a19ac7749152357d3bb7bb6781783c0da2f76083c7783abfc5377ec296703"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 new file mode 100644 index 000000000..a5152b913 --- /dev/null +++ b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 @@ -0,0 +1 @@ +{"rule": "covariance_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini"], "log": [], "shellcmd": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n\n cd /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked\n python /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini\n ", "params": ["'/n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini'"], "code": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n\n cd {params.outdir}\n python {params.onecov} {params.ini_path}\n ", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8726735029882, "starttime": 1783693358.7314878, "endtime": 1783693684.0245638, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini": "sha256:6a2f939819d7d6ffa95955c672ae1875f8b764bdd33613281009f50117768280"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p new file mode 100644 index 000000000..dcb4a6120 --- /dev/null +++ b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p @@ -0,0 +1 @@ +{"rule": "covariance_ini_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json"], "log": [], "shellcmd": "\n mkdir -p /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked\n\n cat > \"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = 1\nssc = 1\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = False\nggl = False\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked\nfile = covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.dat\nstyle = matrix\n\n[covTHETAspace settings]\ntheta_min = 1.0\ntheta_max = 100.0\ntheta_bins = 20\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/mask\nmask_file_lensing = mask_cls_footprint_nside_4096_norm.txt\nmask_file_clust = \nmask_file_ggl = \nellipticity_dispersion = 0.28\nsurvey_area_lensing_in_deg2 = 3400\n\n\nn_eff_lensing = 5.0\n\n\n[redshift]\nzlens_directory = /n17data/sguerrini/UNIONS/WL/nz\nzlens_file = nz_SP_v1.4.6_A.txt\nzclust_file = \nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = 0.8111\nh = 0.6736\nomega_m = 0.3153\nomega_b = 0.0493\nomega_de = 0.6847\nw0 = -1.0\nwa = 0.0\nns = 0.9649\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = 7.75\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "params": ["''", "''", "''", "''", "''", "''", "'/n17data/sguerrini/UNIONS/WL/nz'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/mask'", "'1'", "'False'", "'False'", "'covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.dat'", "'mask_cls_footprint_nside_4096_norm.txt'", "'n_eff_lensing = 5.0'", "'nz_SP_v1.4.6_A.txt'", "'survey_area_lensing_in_deg2 = 3400'", "0.0493", "0.28", "0.3153", "0.6736", "0.6847", "0.8111", "0.9649", "7.75"], "code": "\n mkdir -p {params.outdir}\n\n cat > \"{output}\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = {params.ng_value}\nssc = {params.ng_value}\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = {params.do_clustering}\nggl = {params.do_ggl}\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = {params.outdir}\nfile = {params.out_filename}\nstyle = matrix\n\n[covTHETAspace settings]\ntheta_min = {wildcards.min_sep}\ntheta_max = {wildcards.max_sep}\ntheta_bins = {wildcards.nbins}\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = {params.mask_dir}\nmask_file_lensing = {params.mask_lens}\nmask_file_clust = {params.mask_clust}\nmask_file_ggl = {params.mask_ggl}\nellipticity_dispersion = {params.sigma_e_param}\n{params.area_lens_line}\n{params.area_clust_line}\n{params.area_ggl_line}\n{params.n_e_lens_line}\n{params.n_e_clust_line}\n\n[redshift]\nzlens_directory = {params.nz_dir}\nzlens_file = {params.nz_lens}\nzclust_file = {params.nz_clust}\nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = {params.sigma_8}\nh = {params.h}\nomega_m = {params.omega_m}\nomega_b = {params.omega_b}\nomega_de = {params.omega_v}\nw0 = -1.0\nwa = 0.0\nns = {params.n_s}\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = {params.hmcode_logT_AGN}\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "record_format_version": 6, "conda_env": null, "container_img_url": "/n17data/cdaley/containers/containers", "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8766909525762, "starttime": 1783693357.3336766, "endtime": 1783693343.2093573, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json": "sha256:c44a19ac7749152357d3bb7bb6781783c0da2f76083c7783abfc5377ec296703"}} \ No newline at end of file diff --git a/workflow/Snakefile b/workflow/Snakefile index 6f59827b4..9624ed58d 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -36,11 +36,11 @@ wildcard_constraints: **WILDCARD_CONSTRAINTS # Compute rules (infrastructure — raw outputs, no evidence.json) -include: "rules/twopoint.smk" +# include: "rules/twopoint.smk" include: "rules/covariance.smk" -include: "rules/inference.smk" -include: "rules/masks.smk" -include: "rules/glass_mock.smk" +# include: "rules/inference.smk" +# include: "rules/masks.smk" +# include: "rules/glass_mock.smk" # CosmologyValidation diagnostic suite (cosmo_val.py), decomposed rule-by-rule. # Only active when the config carries a `cosmo_val` block (papers/cosmo_val); diff --git a/workflow/common.py b/workflow/common.py index 3df3413be..1b3605924 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -9,15 +9,15 @@ # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL = Path( os.environ.get( - "COSMO_VAL", "/n17data/cdaley/unions/code/sp_validation/cosmo_val/output" + "COSMO_VAL", "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmo_val" ) ) COSMO_INFERENCE = Path( os.environ.get( - "COSMO_INFERENCE", "/n17data/cdaley/unions/code/sp_validation/cosmo_inference" + "COSMO_INFERENCE", "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference" ) ) -CAT_CONFIG = "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" +CAT_CONFIG = "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_val/cat_config.yaml" BLINDS = ["A", "B", "C"] BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")] @@ -25,7 +25,7 @@ # Source of truth: cs_util.cosmo.PLANCK18 # Regenerate with: snakemake results/cosmology/planck18.json # Resolved relative to the run directory at configure() time. -COSMOLOGY_PARAMS = "results/cosmology/planck18.json" +COSMOLOGY_PARAMS = "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json" # Wildcard constraints shared by every Snakefile that composes these rules. # Patterns must match all expected values; overly restrictive patterns cause @@ -37,6 +37,7 @@ "min_sep": r"[0-9.]+", "max_sep": r"[0-9.]+", "gaussian": r"(g|ng)", + "probe": r"(wl|ggl|3x2pt)", "block_pm": r"(\+\+|--|\+-)", "block_i": r"[123]", "mask_suffix": r"(_masked)?", @@ -46,22 +47,23 @@ FIDUCIAL = None DEFAULT_MASK_SUFFIX = "" +DEFAULT_PROBE = None CATALOG_CONFIG = None PLANCK18 = None def configure(workflow_config): """Install config-derived values after Snakemake has loaded configfiles.""" - global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18 + global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, DEFAULT_PROBE, FIDUCIAL, PLANCK18 CATALOG_CONFIG = workflow_config FIDUCIAL = workflow_config["fiducial"] + DEFAULT_PROBE = "wl" DEFAULT_MASK_SUFFIX = ( "_masked" if workflow_config["covariance"].get("default_masked", False) else "" ) with open(COSMOLOGY_PARAMS) as f: PLANCK18 = json.load(f) - def fiducial_binning_suffix(fiducial=None): """Return binning suffix for fiducial parameters.""" fiducial = fiducial or FIDUCIAL @@ -76,44 +78,75 @@ def resolve_covariance_version(version): return version +# Single source of truth for the covariance naming scheme. Rule outputs use +# this with wildcards left in braces; covariance_base() fills concrete values. +# Keep the two in sync by construction: covariance_base() formats this string. +COV_BASE_TEMPLATE = ( + "covariance_{version}_{blind}_{gaussian}" + "_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_{probe}{mask_suffix}" +) + +def cov_output(suffix): + """Wildcard-bearing output path under the covariance tree. + + E.g. cov_output(".ini") -> + /data/covariance//.ini + with {version}, {blind}, ... left as Snakemake wildcards. + """ + return str( + COSMO_INFERENCE / f"data/covariance/{COV_BASE_TEMPLATE}/{COV_BASE_TEMPLATE}{suffix}" + ) + +def cov_output_dirfile(filename): + """Wildcard-bearing path to a fixed-name file inside the covariance dir.""" + return str(COSMO_INFERENCE / f"data/covariance/{COV_BASE_TEMPLATE}/{filename}") + + def covariance_base( version, blind, - gaussian="ng", + gaussian=None, min_sep=None, max_sep=None, nbins=None, + probe=None, mask_suffix=None, resolve_version=True, fiducial=None, default_mask_suffix=None, ): - """Construct covariance base name.""" + """Construct covariance base name (concrete values, no wildcards).""" fiducial = fiducial or FIDUCIAL + gaussian = gaussian if gaussian is not None else fiducial["gaussian"] min_sep = min_sep if min_sep is not None else fiducial["min_sep"] max_sep = max_sep if max_sep is not None else fiducial["max_sep"] nbins = nbins if nbins is not None else fiducial["nbins"] - mask_suffix = ( - mask_suffix - if mask_suffix is not None - else ( + probe = probe if probe is not None else DEFAULT_PROBE + if mask_suffix is None: + mask_suffix = ( DEFAULT_MASK_SUFFIX if default_mask_suffix is None else default_mask_suffix ) - ) cov_version = resolve_covariance_version(version) if resolve_version else version - return ( - f"covariance_{cov_version}_{blind}_{gaussian}" - f"_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}" + return COV_BASE_TEMPLATE.format( + version=cov_version, + blind=blind, + gaussian=gaussian, + min_sep=min_sep, + max_sep=max_sep, + nbins=nbins, + probe=probe, + mask_suffix=mask_suffix, ) def covariance_dir( version, blind, - gaussian="ng", + gaussian=None, min_sep=None, max_sep=None, nbins=None, + probe=None, mask_suffix=None, resolve_version=True, ): @@ -125,6 +158,7 @@ def covariance_dir( min_sep, max_sep, nbins, + probe, mask_suffix, resolve_version=resolve_version, ) @@ -134,12 +168,13 @@ def covariance_dir( def covariance_path( version, blind, - gaussian="ng", + gaussian=None, min_sep=None, max_sep=None, nbins=None, + probe=None, mask_suffix=None, - suffix="_processed.txt", + suffix=".dat", resolve_version=True, ): """Construct covariance file path.""" @@ -150,13 +185,14 @@ def covariance_path( min_sep, max_sep, nbins, + probe, mask_suffix, resolve_version=resolve_version, ) return str(COSMO_INFERENCE / f"data/covariance/{base}/{base}{suffix}") -def build_redshift_path(version, blind): +def build_redshift_dir(version): """Construct n(z) filepath for given catalog version and blind.""" base_version = re.sub(r"_leak_corr$", "", version) base_version = re.sub(r"_ecut\d+", "", base_version) @@ -164,9 +200,28 @@ def build_redshift_path(version, blind): base_version = "SP_v1.4.6" version_dir = base_version.replace("SP_", "") return ( - f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/nz_{base_version}_{blind}.txt" + f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/" + ) + +def build_redshift_path_lens(version, blind): + """Construct n(z) filepath for given catalog version and blind.""" + base_version = re.sub(r"_leak_corr$", "", version) + base_version = re.sub(r"_ecut\d+", "", base_version) + if "v1.4.11" in base_version: + base_version = "SP_v1.4.6" + return ( + f"nz_{base_version}_{blind}.txt" ) +def build_redshift_path_source(version, blind): + """Construct n(z) filepath for given catalog version and blind.""" + base_version = re.sub(r"_leak_corr$", "", version) + base_version = re.sub(r"_ecut\d+", "", base_version) + if "v1.4.11" in base_version: + base_version = "SP_v1.4.6" + return ( + f"nz_{base_version}_{blind}_source.txt" #TO DO: check lens and source file format + ) def get_shear_catalog(wildcards): """Resolve shear catalog path from config for a given version.""" diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index 49e402012..1a991e309 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -1,14 +1,23 @@ # BLOCK_PAIRS, PLANCK18, COSMOLOGY_PARAMS defined in Snakefile - +import os def get_cat_params(version): - """Extract covariance parameters (area, n_e, sigma_e) from catalog config.""" + """Extract covariance parameters (sigma_e, n_eff, area) from catalog config. + + Returns (sigma_e, (n_e_lens, n_e_clust), (A_lens, A_ggl, A_clust)). + For probe == "wl" the clustering/ggl slots are empty strings. + """ base_version = version.replace("_leak_corr", "") if base_version not in config: raise KeyError(f"Catalog configuration not found for {base_version}") cov_th = config[base_version]["cov_th"] - return cov_th["A"], cov_th["n_e"], cov_th["sigma_e"] - + if config["probe_3x2pt"] == "wl": + return cov_th["sigma_e"], (cov_th["n_e"], ""), (cov_th["A"], "", "") + return ( + cov_th["sigma_e"], + (cov_th["n_e_lens"], cov_th["n_e_clust"]), + (cov_th["A_lens"], cov_th["A_ggl"], cov_th["A_clust"]), + ) # covariance_dir(), covariance_base(), covariance_path() defined in Snakefile # Additional wildcard constraints defined locally for pseudo-Cl rules (line 327) @@ -17,21 +26,43 @@ def get_cat_params(version): # Footprint mask power spectra (nside=4096, from comprehensive catalog with spatial cuts only) MASK_CLS_BASE = str(COSMO_INFERENCE / "data/mask") MASK_CLS_FILES = { - "footprint": f"{MASK_CLS_BASE}/mask_cls_footprint_nside_4096_norm.txt", - "footprint_starhalo": f"{MASK_CLS_BASE}/mask_cls_footprint_starhalo_nside_4096_norm.txt", + "footprint_lens": "mask_cls_footprint_nside_4096_norm.txt", + "footprint_ggl": "mask_cls_footprint_nside_4096_norm.txt", + "footprint_clust": "mask_cls_footprint_nside_4096_norm.txt", + "footprint_lens_starhalo": "mask_cls_footprint_starhalo_nside_4096_norm.txt", } # v1.4.8 uses the star-halo footprint; all other versions use the standard footprint STARHALO_VERSIONS = {"v1.4.8"} +def get_mask_cls_file(version, kind="lens"): + """Return the mask Cl *filename* (OneCov wants dir and file separately).""" + version_dir = version.replace("_leak_corr", "").replace("SP_", "") + version_dir = re.sub(r"_ecut\d+", "", version_dir) + if kind == "lens" and version_dir in STARHALO_VERSIONS: + return MASK_CLS_FILES["footprint_lens_starhalo"] + return MASK_CLS_FILES[f"footprint_{kind}"] -def get_mask_cls_path(version): - """Return absolute mask Cl path for the requested catalog version.""" - version_dir = version.replace('_leak_corr', '').replace('SP_', '') - version_dir = re.sub(r'_ecut\d+', '', version_dir) - key = "footprint_starhalo" if version_dir in STARHALO_VERSIONS else "footprint" - return MASK_CLS_FILES[key] +def _onecov_mask_params(w): + """Resolve OneCov mask settings for one job. + + Returns dict with keys: dir, lens, ggl, clust. Empty strings when the + job is unmasked or the probe doesn't use a given field. + """ + if w.mask_suffix != "_masked": + return {"dir": "", "lens": "", "ggl": "", "clust": ""} + out = { + "dir": MASK_CLS_BASE, + "lens": get_mask_cls_file(w.version, "lens"), + "ggl": "", + "clust": "", + } + if w.probe in ("ggl", "3x2pt"): + out["ggl"] = get_mask_cls_file(w.version, "ggl") + if w.probe == "3x2pt": + out["clust"] = get_mask_cls_file(w.version, "clust") + return out rule cosmology_params: """Generate cosmology parameters JSON from sp_validation. @@ -57,112 +88,199 @@ with open('{output}', 'w') as f: " """ - -rule covariance_ini: +rule covariance_ini_onecov: input: - nz_file=lambda w: build_redshift_path(w.version, w.blind), - mask=lambda w: [] if w.mask_suffix != "_masked" else [get_mask_cls_path(w.version)], + # Ensures planck18.json exists before get_planck18() is called below. + cosmo=COSMOLOGY_PARAMS, output: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.ini") + cov_output(".ini"), params: outdir=lambda w: covariance_dir( - w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, - resolve_version=False + w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, + w.probe, w.mask_suffix, resolve_version=False, ), - ng_value=lambda wildcards: "1" if wildcards.gaussian == "ng" else "0", + out_filename=lambda w: covariance_base( + w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, + w.probe, w.mask_suffix, resolve_version=False, + ) + ".dat", + out_plot_filename=lambda w: covariance_base( + w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, + w.probe, w.mask_suffix, resolve_version=False, + ) + "_corr_plot.pdf", + ng_value=lambda w: "1" if w.gaussian == "ng" else "0", + do_ggl=lambda w: str(w.probe in ("ggl", "3x2pt")), + do_clustering=lambda w: str(w.probe == "3x2pt"), omega_m=PLANCK18["Omega_m"], omega_v=PLANCK18["Omega_v"], sigma_8=PLANCK18["sigma_8"], n_s=PLANCK18["n_s"], h=PLANCK18["h"], omega_b=PLANCK18["Omega_b"], - area=lambda w: get_cat_params(w.version)[0], - n_e=lambda w: get_cat_params(w.version)[1], - sigma_e_param=lambda w: get_cat_params(w.version)[2], - mask=lambda w: get_mask_cls_path(w.version) if w.mask_suffix == "_masked" else "", + hmcode_logT_AGN=7.75, + sigma_e_param=lambda w: get_cat_params(w.version)[0], + n_e_lens_line=lambda w: ( + f"n_eff_lensing = {get_cat_params(w.version)[1][0]}"), + n_e_clust_line=lambda w: ("" + if w.probe == "wl" + else f"n_eff_clust = {get_cat_params(w.version)[1][1]}"), + area_lens_line=lambda w: ( + f"survey_area_lensing_in_deg2 = {get_cat_params(w.version)[2][0]}"), + area_ggl_line=lambda w: ( + "" + if w.probe == "wl" + else f"survey_area_ggl_in_deg2 = {get_cat_params(w.version)[2][1]}" + ), + area_clust_line=lambda w: ( + "" + if w.probe == "wl" + else f"survey_area_clust_in_deg2 = {get_cat_params(w.version)[2][2]}" + ), + mask_dir=lambda w: _onecov_mask_params(w)["dir"], + mask_lens=lambda w: _onecov_mask_params(w)["lens"], + mask_ggl=lambda w: _onecov_mask_params(w)["ggl"], + mask_clust=lambda w: _onecov_mask_params(w)["clust"], + nz_dir=lambda w: os.path.dirname(build_redshift_dir(w.version)), + nz_lens=lambda w: os.path.basename(build_redshift_path_lens(w.version, w.blind)), + nz_clust=lambda w: ( + "" if w.probe == "wl" + else os.path.basename(build_redshift_path_source(w.version, w.blind)) + ), threads: 1 shell: """ mkdir -p {params.outdir} - cat > {output} << 'EOF' -# -# Cosmological parameters -# -Omega_m : {params.omega_m} -Omega_v : {params.omega_v} -sigma_8 : {params.sigma_8} -n_spec : {params.n_s} -w0 : -1 -wa : 0 -omb : {params.omega_b} -h0 : {params.h} - - -# Survey and galaxy parameters -# -# area in degrees -# n_gal,lens_n_gal in gals/arcmin^2 - -area : {params.area} -sourcephotoz : multihisto -lensphotoz : multihisto -source_tomobins : 1 -lens_tomobins : 1 -sigma_e : {params.sigma_e_param} -source_n_gal : {params.n_e} -lens_n_gal : {params.n_e} - - -shear_REDSHIFT_FILE : {input.nz_file} -clustering_REDSHIFT_FILE : {input.nz_file} -c_footprint_file : {params.mask} - - -# IA parameters -IA : 1 -A_ia : 0.0 -eta_ia : 0.0 - - -# Covariance parameters -# -# tmin,tmax in arcminutes -tmin : {wildcards.min_sep} -tmax : {wildcards.max_sep} -ntheta : {wildcards.nbins} -ng : {params.ng_value} -cng : {params.ng_value} - - -outdir : ./ -filename : cov_tmp -ss : true -ls : false -ll : false + cat > "{output}" << 'EOF' +[covariance terms] +gauss = True +split_gauss = False +nongauss = {params.ng_value} +ssc = {params.ng_value} +sn_only = False + +[observables] +cosmic_shear = True +est_shear = xi_pm +clustering = {params.do_clustering} +ggl = {params.do_ggl} +est_ggl = gamma_t +est_clust = w + +[output settings] +directory = {params.outdir} +file = {params.out_filename} +style = matrix +save_configs = False +corrmatrix_plot = {params.out_plot_filename} + +[covTHETAspace settings] +theta_min = {wildcards.min_sep} +theta_max = {wildcards.max_sep} +theta_bins = {wildcards.nbins} +theta_type = log +xi_pp = True +xi_mm = True +theta_accuracy = 1e-3 +integration_intervals = 400 + +[survey specs] +mask_directory = {params.mask_dir} +mask_file_lensing = {params.mask_lens} +mask_file_clust = {params.mask_clust} +mask_file_ggl = {params.mask_ggl} +ellipticity_dispersion = {params.sigma_e_param} +{params.area_lens_line} +{params.area_clust_line} +{params.area_ggl_line} +{params.n_e_lens_line} +{params.n_e_clust_line} + +[redshift] +zlens_directory = {params.nz_dir} +zlens_file = {params.nz_lens} +zclust_file = {params.nz_clust} +value_loc_in_lensbin = mid +value_loc_in_clustbin = mid + +[cosmo] +sigma8 = {params.sigma_8} +h = {params.h} +omega_m = {params.omega_m} +omega_b = {params.omega_b} +omega_de = {params.omega_v} +w0 = -1.0 +wa = 0.0 +ns = {params.n_s} +neff = 3.046 +m_nu = 0.06 +tcmb0 = 2.725 + +[IA] +A_IA = 0.0 +eta_IA = 0.0 +z_pivot_IA = 0.3 + +[powspec evaluation] +non_linear_model = mead2020_feedback +HMCode_logT_AGN = {params.hmcode_logT_AGN} +log10k_bins = 300 +log10k_min = -3.46 +log10k_max = 3.15 + +[hod] +model_mor_cen = double_powerlaw +model_mor_sat = double_powerlaw +dpow_logm0_cen = 10.51 +dpow_logm1_cen = 11.38 +dpow_a_cen = 7.096 +dpow_b_cen = 0.2 +dpow_norm_cen = 1.0 +dpow_norm_sat = 0.56 +model_scatter_cen = lognormal +model_scatter_sat = modschechter +logn_sigma_c_cen = 0.35 +modsch_logmref_sat = 13.0 +modsch_alpha_s_sat = -0.858 +modsch_b_sat = -0.024, 1.149 + +[halomodel evaluation] +m_bins = 900 +log10m_min = 6 +log10m_max = 18 +hmf_model = Tinker10 +mdef_model = SOMean +mdef_params = overdensity, 200 +disable_mass_conversion = True +delta_c = 1.686 +transfer_model = CAMB +small_k_damping_for1h = damped + +[misc] +num_cores = 8 + EOF """ - -rule covariance_cosmocov: +rule covariance_onecov: input: - rules.covariance_ini.output, + rules.covariance_ini_onecov.output, output: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/cov_tmp_ssss_{block_pm}_cov_Ntheta{nbins}_Ntomo1_{block_i}") + # Matches `[output settings] file = cov_tmp_onecov.dat` in the ini. + cov_output(".dat") params: - block_i="{block_i}", outdir=lambda w: covariance_dir( - w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, - resolve_version=False + w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, + w.probe, w.mask_suffix, resolve_version=False, ), ini_path=lambda w: covariance_path( - w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, - suffix=".ini", resolve_version=False + w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, + w.probe, w.mask_suffix, suffix=".ini", resolve_version=False, ), - cosmocov=config["tools"]["cosmocov_executable"], + onecov=config["tools"]["onecov_executable"], + python_executable=config["tools"]["python_executable"], container: None - threads: 1 + threads: 8 shell: """ module unload gcc || true @@ -172,68 +290,73 @@ rule covariance_cosmocov: module load openmpi cd {params.outdir} - {params.cosmocov} {params.block_i} {params.ini_path} + {params.python_executable} {params.onecov} {params.ini_path} """ -rule covariance_cat: - input: - cov_block=lambda w: [ - f"{covariance_dir(w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, w.mask_suffix, resolve_version=False)}" - f"/cov_tmp_ssss_{pm}_cov_Ntheta{w.nbins}_Ntomo1_{idx}" - for pm, idx in BLOCK_PAIRS - ], - threads: 1 - output: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") - shell: - """ - cat {input} > {output} - """ - +# Glass-mock roots. Env/config-overridable for the same reason as COSMO_*. +GLASS_MOCK_RESULTS = config.get("glass_mocks", {}).get( + "results_dir", "/n09data/guerrini/glass_mock_v1.4.6/results" +) +GLASS_MOCK_COV_DIR = config.get("glass_mocks", {}).get( + "covariance_dir", + "/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6", +) rule covariance_glass_mock: input: xi=expand( - "/n09data/guerrini/glass_mock_v1.4.6/results/xi_glass_mock_{seed:05d}_4096_nbins=20.fits", - seed=range(config["glass_mocks"]["seed_range"][0], config["glass_mocks"]["seed_range"][1] + 1), + GLASS_MOCK_RESULTS + "/xi_glass_mock_{seed:05d}_4096_nbins=20.fits", + seed=range( + config["glass_mocks"]["seed_range"][0], + config["glass_mocks"]["seed_range"][1] + 1, + ), ), cl=expand( - "/n09data/guerrini/glass_mock_v1.4.6/results/cl_glass_mock_{seed:05d}_4096.npy", - seed=range(config["glass_mocks"]["seed_range"][0], config["glass_mocks"]["seed_range"][1] + 1), + GLASS_MOCK_RESULTS + "/cl_glass_mock_{seed:05d}_4096.npy", + seed=range( + config["glass_mocks"]["seed_range"][0], + config["glass_mocks"]["seed_range"][1] + 1, + ), ), output: - xi_covariance="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/xi_covariance.npy", - cl_covariance="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/cl_covariance.npy", - combined_covariance="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/combined_covariance.npy", - correlation_plot="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/combined_correlation.png", - xi_mean="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/xi_mean.npy", - cl_mean="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/cl_mean.npy", - combined_mean="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/combined_mean.npy", + xi_covariance=f"{GLASS_MOCK_COV_DIR}/xi_covariance.npy", + cl_covariance=f"{GLASS_MOCK_COV_DIR}/cl_covariance.npy", + combined_covariance=f"{GLASS_MOCK_COV_DIR}/combined_covariance.npy", + correlation_plot=f"{GLASS_MOCK_COV_DIR}/combined_correlation.png", + xi_mean=f"{GLASS_MOCK_COV_DIR}/xi_mean.npy", + cl_mean=f"{GLASS_MOCK_COV_DIR}/cl_mean.npy", + combined_mean=f"{GLASS_MOCK_COV_DIR}/combined_mean.npy", script: "../scripts/compute_glass_mock_covariance.py" - -# fiducial_binning_suffix() defined in Snakefile - - rule generate_glass_mock_rhotau_samples: """Generate sampled tau statistics for glass mocks. Only tau is sampled; inference_prep_glass_mock uses real rho data. """ input: - cov_tau=str(COSMO_VAL / f"rho_tau_stats/cov_tau_{FIDUCIAL['mock_version']}{fiducial_binning_suffix()}_th.npy"), - ref_tau=str(COSMO_VAL / f"rho_tau_stats/tau_stats_{FIDUCIAL['mock_version']}{fiducial_binning_suffix()}.fits"), + cov_tau=str( + COSMO_VAL + / f"rho_tau_stats/cov_tau_{FIDUCIAL['mock_version']}{fiducial_binning_suffix()}_th.npy" + ), + ref_tau=str( + COSMO_VAL + / f"rho_tau_stats/tau_stats_{FIDUCIAL['mock_version']}{fiducial_binning_suffix()}.fits" + ), output: tau="results/glass_mock_rhotau_samples/{mock_id}/tau_stats_sampled.fits", params: mock_id="{mock_id}", output_dir="results/glass_mock_rhotau_samples", + # Resolved relative to this .smk file instead of a hard-coded checkout. + script=workflow.source_path( + "../scripts/generate_glass_mock_rhotau_samples.py" + ), threads: 1 shell: """ - python /n17data/cdaley/unions/pure_eb/code/sp_validation/workflow/scripts/generate_glass_mock_rhotau_samples.py \ + python {params.script} \ --cov-tau {input.cov_tau} \ --ref-tau {input.ref_tau} \ --output-dir {params.output_dir} \ @@ -241,55 +364,35 @@ rule generate_glass_mock_rhotau_samples: """ -rule covariance_process: - input: - str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") - output: - matrix=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed.txt"), - gaussian=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_g.txt"), - plot=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_plot.pdf") - params: - output_stub=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed") - threads: 1 - shell: - """ - python /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/scripts/cosmocov_process.py {input} {params.output_stub} - """ - - -def fiducial_covariance_outputs(mask_suffix=""): +def fiducial_covariance_outputs(mask_suffix="", probe=DEFAULT_PROBE): """Return processed covariance files for fiducial version/blind.""" - ng_path = covariance_path( - FIDUCIAL["version"], FIDUCIAL["blind"], "ng", - FIDUCIAL["min_sep"], FIDUCIAL["max_sep"], FIDUCIAL["nbins"], mask_suffix + path = covariance_path( + FIDUCIAL["version"], FIDUCIAL["blind"], FIDUCIAL["gaussian"], + FIDUCIAL["min_sep"], FIDUCIAL["max_sep"], FIDUCIAL["nbins"], + probe, mask_suffix, ) - g_path = covariance_path( - FIDUCIAL["version"], FIDUCIAL["blind"], "g", - FIDUCIAL["min_sep_int"], FIDUCIAL["max_sep_int"], FIDUCIAL["nbins_int"], mask_suffix - ) - return [ng_path, g_path] - + return path rule covariance: input: - fiducial_covariance_outputs(mask_suffix=DEFAULT_MASK_SUFFIX) - + fiducial_covariance_outputs(mask_suffix=DEFAULT_MASK_SUFFIX, probe=DEFAULT_PROBE), rule covariance_masked: input: - fiducial_covariance_outputs(mask_suffix="_masked") - + fiducial_covariance_outputs(mask_suffix="_masked", probe=DEFAULT_PROBE), rule covariance_unmasked: input: - fiducial_covariance_outputs(mask_suffix="") + fiducial_covariance_outputs(mask_suffix="", probe=DEFAULT_PROBE), +rule covariance_3x2pt: + input: + fiducial_covariance_outputs(mask_suffix="", + probe=config["probe_3x2pt"]), -ruleorder: covariance_ini > covariance_cosmocov > covariance_cat > covariance_process > covariance +# ruleorder: covariance_ini_onecov > covariance_onecov > covariance localrules: cosmology_params, - covariance_ini, - covariance_cat, - covariance_process, + covariance_ini_onecov, generate_glass_mock_rhotau_samples, From 0d8b478398474c6b586e9b5217d426e25f937fde Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Fri, 10 Jul 2026 18:02:51 +0200 Subject: [PATCH 02/14] ruff check --- .../EwMC4wX25iaW5zPTIwX3dsLmRhdA== | 1 - .../AuMF9uYmlucz0yMF93bC5kYXQ= | 1 - .../AuMF9uYmlucz0yMF93bC5pbmk= | 1 - .../EwMC4wX25iaW5zPTIwX3dsLmRhdA== | 1 - .../EwMC4wX25iaW5zPTIwX3dsLmluaQ== | 1 - .../1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 | 1 - .../1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p | 1 - workflow/common.py | 2 +- 8 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 workflow/.snakemake/incomplete/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== delete mode 100644 workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5kYXQ= delete mode 100644 workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5pbmk= delete mode 100644 workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== delete mode 100644 workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmluaQ== delete mode 100644 workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 delete mode 100644 workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p diff --git a/workflow/.snakemake/incomplete/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== b/workflow/.snakemake/incomplete/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== deleted file mode 100644 index 5d3401dcf..000000000 --- a/workflow/.snakemake/incomplete/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== +++ /dev/null @@ -1 +0,0 @@ -{"external_jobid": null} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5kYXQ= b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5kYXQ= deleted file mode 100644 index cf9de83bd..000000000 --- a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5kYXQ= +++ /dev/null @@ -1 +0,0 @@ -{"rule": "covariance_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.ini"], "log": [], "shellcmd": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n\n cd /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl\n /n23data1/n06data/lgoh/scratch/my_env_new/bin/python /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.ini\n ", "params": ["'/n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.ini'", "'/n23data1/n06data/lgoh/scratch/my_env_new/bin/python'"], "code": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n\n cd {params.outdir}\n {params.python_executable} {params.onecov} {params.ini_path}\n ", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8739027612265, "starttime": 1783698870.0182004, "endtime": 1783698870.015105, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.ini": "sha256:2eb8d026dbff910f9cf3c6d725dca5527d0995e8304710b4534085ea64dca295"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5pbmk= b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5pbmk= deleted file mode 100644 index 75b02c39c..000000000 --- a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9nX21pbnNlcD0xLjBfbWF4c2VwPTEwMC4wX25iaW5zPTIwX3dsL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZXA9MS4wX21heHNlcD0xMD/AuMF9uYmlucz0yMF93bC5pbmk= +++ /dev/null @@ -1 +0,0 @@ -{"rule": "covariance_ini_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json"], "log": [], "shellcmd": "\n mkdir -p /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl\n\n cat > \"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.ini\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = 0\nssc = 0\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = False\nggl = False\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl\nfile = covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.dat\nstyle = matrix\nsave_configs = False\ncorrmatrix_plot = covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_corr_plot.pdf\n\n[covTHETAspace settings]\ntheta_min = 1.0\ntheta_max = 100.0\ntheta_bins = 20\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = \nmask_file_lensing = \nmask_file_clust = \nmask_file_ggl = \nellipticity_dispersion = 0.28, 0.28\nsurvey_area_lensing_in_deg2 = 1000\n\n\nn_eff_lensing = 5.0, 5.0\n\n\n[redshift]\nzlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/\n# /n17data/sguerrini/UNIONS/WL/nz/v1.4.6\nzlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii\n# nz_SP_v1.4.6_A.txt\nzclust_file = \nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = 0.8111\nh = 0.6736\nomega_m = 0.3153\nomega_b = 0.0493\nomega_de = 0.6847\nw0 = -1.0\nwa = 0.0\nns = 0.9649\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = 7.75\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "params": ["''", "''", "''", "''", "''", "''", "''", "''", "'/n17data/sguerrini/UNIONS/WL/nz/v1.4.6'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl'", "'0'", "'0.28, 0.28'", "'False'", "'False'", "'covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl.dat'", "'covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_corr_plot.pdf'", "'n_eff_lensing = 5.0, 5.0'", "'nz_SP_v1.4.6_A.txt'", "'survey_area_lensing_in_deg2 = 1000'", "0.0493", "0.3153", "0.6736", "0.6847", "0.8111", "0.9649", "7.75"], "code": "\n mkdir -p {params.outdir}\n\n cat > \"{output}\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = {params.ng_value}\nssc = {params.ng_value}\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = {params.do_clustering}\nggl = {params.do_ggl}\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = {params.outdir}\nfile = {params.out_filename}\nstyle = matrix\nsave_configs = False\ncorrmatrix_plot = {params.out_plot_filename}\n\n[covTHETAspace settings]\ntheta_min = {wildcards.min_sep}\ntheta_max = {wildcards.max_sep}\ntheta_bins = {wildcards.nbins}\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = {params.mask_dir}\nmask_file_lensing = {params.mask_lens}\nmask_file_clust = {params.mask_clust}\nmask_file_ggl = {params.mask_ggl}\nellipticity_dispersion = {params.sigma_e_param}\n{params.area_lens_line}\n{params.area_clust_line}\n{params.area_ggl_line}\n{params.n_e_lens_line}\n{params.n_e_clust_line}\n\n[redshift]\nzlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/\n# {params.nz_dir}\nzlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii\n# {params.nz_lens}\nzclust_file = {params.nz_clust}\nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = {params.sigma_8}\nh = {params.h}\nomega_m = {params.omega_m}\nomega_b = {params.omega_b}\nomega_de = {params.omega_v}\nw0 = -1.0\nwa = 0.0\nns = {params.n_s}\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = {params.hmcode_logT_AGN}\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "record_format_version": 6, "conda_env": null, "container_img_url": "/n17data/cdaley/containers/containers", "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8739027612384, "starttime": 1783698635.9245446, "endtime": 1783698633.455901, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json": "sha256:c44a19ac7749152357d3bb7bb6781783c0da2f76083c7783abfc5377ec296703"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== deleted file mode 100644 index c32d15ddd..000000000 --- a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmRhdA== +++ /dev/null @@ -1 +0,0 @@ -{"rule": "covariance_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.ini"], "log": [], "shellcmd": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n source activate /n23data1/n06data/lgoh/scratch/my_env_new\n cd /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl\n /n23data1/n06data/lgoh/scratch/my_env_new/bin/python /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.ini\n ", "params": ["'/n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.ini'"], "code": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n source activate /n23data1/n06data/lgoh/scratch/my_env_new\n cd {params.outdir}\n /n23data1/n06data/lgoh/scratch/my_env_new/bin/python {params.onecov} {params.ini_path}\n ", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8749163604585, "starttime": 1783694135.7577267, "endtime": 1783696080.0747812, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.ini": "sha256:6def65cf3f6f7510a577bc07fb8ea89d57a1267a884513131b52ffa5175681db"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmluaQ== b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmluaQ== deleted file mode 100644 index 467c15af5..000000000 --- a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bC9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX25nX21pbnNlcD0xLjBfbWF4c2VwPT/EwMC4wX25iaW5zPTIwX3dsLmluaQ== +++ /dev/null @@ -1 +0,0 @@ -{"rule": "covariance_ini_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json"], "log": [], "shellcmd": "\n mkdir -p /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl\n\n cat > \"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.ini\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = 1\nssc = 1\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = False\nggl = False\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl\nfile = covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.dat\nstyle = matrix\nsave_configs = False\ncorrmatrix_plot = covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_corr_plot.pdf\n\n[covTHETAspace settings]\ntheta_min = 1.0\ntheta_max = 100.0\ntheta_bins = 20\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = \nmask_file_lensing = \nmask_file_clust = \nmask_file_ggl = \nellipticity_dispersion = 0.28, 0.28\nsurvey_area_lensing_in_deg2 = 1000\n\n\nn_eff_lensing = 5.0, 5.0\n\n\n[redshift]\nzlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/\n# /n17data/sguerrini/UNIONS/WL/nz/v1.4.6\nzlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii\n# nz_SP_v1.4.6_A.txt\nzclust_file = \nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = 0.8111\nh = 0.6736\nomega_m = 0.3153\nomega_b = 0.0493\nomega_de = 0.6847\nw0 = -1.0\nwa = 0.0\nns = 0.9649\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = 7.75\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "params": ["''", "''", "''", "''", "''", "''", "''", "''", "'/n17data/sguerrini/UNIONS/WL/nz/v1.4.6'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl'", "'0.28, 0.28'", "'1'", "'False'", "'False'", "'covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl.dat'", "'covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_corr_plot.pdf'", "'n_eff_lensing = 5.0, 5.0'", "'nz_SP_v1.4.6_A.txt'", "'survey_area_lensing_in_deg2 = 1000'", "0.0493", "0.3153", "0.6736", "0.6847", "0.8111", "0.9649", "7.75"], "code": "\n mkdir -p {params.outdir}\n\n cat > \"{output}\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = {params.ng_value}\nssc = {params.ng_value}\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = {params.do_clustering}\nggl = {params.do_ggl}\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = {params.outdir}\nfile = {params.out_filename}\nstyle = matrix\nsave_configs = False\ncorrmatrix_plot = {params.out_plot_filename}\n\n[covTHETAspace settings]\ntheta_min = {wildcards.min_sep}\ntheta_max = {wildcards.max_sep}\ntheta_bins = {wildcards.nbins}\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = {params.mask_dir}\nmask_file_lensing = {params.mask_lens}\nmask_file_clust = {params.mask_clust}\nmask_file_ggl = {params.mask_ggl}\nellipticity_dispersion = {params.sigma_e_param}\n{params.area_lens_line}\n{params.area_clust_line}\n{params.area_ggl_line}\n{params.n_e_lens_line}\n{params.n_e_clust_line}\n\n[redshift]\nzlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/\n# {params.nz_dir}\nzlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii\n# {params.nz_lens}\nzclust_file = {params.nz_clust}\nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = {params.sigma_8}\nh = {params.h}\nomega_m = {params.omega_m}\nomega_b = {params.omega_b}\nomega_de = {params.omega_v}\nw0 = -1.0\nwa = 0.0\nns = {params.n_s}\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = {params.hmcode_logT_AGN}\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "record_format_version": 6, "conda_env": null, "container_img_url": "/n17data/cdaley/containers/containers", "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8756651165425, "starttime": 1783694134.5319173, "endtime": 1783698274.2580187, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json": "sha256:c44a19ac7749152357d3bb7bb6781783c0da2f76083c7783abfc5377ec296703"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 deleted file mode 100644 index a5152b913..000000000 --- a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 +++ /dev/null @@ -1 +0,0 @@ -{"rule": "covariance_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini"], "log": [], "shellcmd": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n\n cd /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked\n python /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini\n ", "params": ["'/n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini'"], "code": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n\n cd {params.outdir}\n python {params.onecov} {params.ini_path}\n ", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8726735029882, "starttime": 1783693358.7314878, "endtime": 1783693684.0245638, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini": "sha256:6a2f939819d7d6ffa95955c672ae1875f8b764bdd33613281009f50117768280"}} \ No newline at end of file diff --git a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p b/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p deleted file mode 100644 index dcb4a6120..000000000 --- a/workflow/.snakemake/metadata/@L24yM2RhdGExL24wNmRhdGEvbGdvaC9zY3JhdGNoL1VOSU9OUy9zcF92YWxpZGF0aW9uL2Nvc21vX2luZmVyZW5jZS9kYXRhL2NvdmFyaWFuY2UvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQvY292YXJpYW5jZV9TUF92MS40LjZfQV9uZ19taW5zZXA9MS4wX2/1heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p +++ /dev/null @@ -1 +0,0 @@ -{"rule": "covariance_ini_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json"], "log": [], "shellcmd": "\n mkdir -p /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked\n\n cat > \"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = 1\nssc = 1\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = False\nggl = False\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked\nfile = covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.dat\nstyle = matrix\n\n[covTHETAspace settings]\ntheta_min = 1.0\ntheta_max = 100.0\ntheta_bins = 20\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = /n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/mask\nmask_file_lensing = mask_cls_footprint_nside_4096_norm.txt\nmask_file_clust = \nmask_file_ggl = \nellipticity_dispersion = 0.28\nsurvey_area_lensing_in_deg2 = 3400\n\n\nn_eff_lensing = 5.0\n\n\n[redshift]\nzlens_directory = /n17data/sguerrini/UNIONS/WL/nz\nzlens_file = nz_SP_v1.4.6_A.txt\nzclust_file = \nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = 0.8111\nh = 0.6736\nomega_m = 0.3153\nomega_b = 0.0493\nomega_de = 0.6847\nw0 = -1.0\nwa = 0.0\nns = 0.9649\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = 7.75\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "params": ["''", "''", "''", "''", "''", "''", "'/n17data/sguerrini/UNIONS/WL/nz'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked'", "'/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/mask'", "'1'", "'False'", "'False'", "'covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.dat'", "'mask_cls_footprint_nside_4096_norm.txt'", "'n_eff_lensing = 5.0'", "'nz_SP_v1.4.6_A.txt'", "'survey_area_lensing_in_deg2 = 3400'", "0.0493", "0.28", "0.3153", "0.6736", "0.6847", "0.8111", "0.9649", "7.75"], "code": "\n mkdir -p {params.outdir}\n\n cat > \"{output}\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = {params.ng_value}\nssc = {params.ng_value}\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = {params.do_clustering}\nggl = {params.do_ggl}\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = {params.outdir}\nfile = {params.out_filename}\nstyle = matrix\n\n[covTHETAspace settings]\ntheta_min = {wildcards.min_sep}\ntheta_max = {wildcards.max_sep}\ntheta_bins = {wildcards.nbins}\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = {params.mask_dir}\nmask_file_lensing = {params.mask_lens}\nmask_file_clust = {params.mask_clust}\nmask_file_ggl = {params.mask_ggl}\nellipticity_dispersion = {params.sigma_e_param}\n{params.area_lens_line}\n{params.area_clust_line}\n{params.area_ggl_line}\n{params.n_e_lens_line}\n{params.n_e_clust_line}\n\n[redshift]\nzlens_directory = {params.nz_dir}\nzlens_file = {params.nz_lens}\nzclust_file = {params.nz_clust}\nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = {params.sigma_8}\nh = {params.h}\nomega_m = {params.omega_m}\nomega_b = {params.omega_b}\nomega_de = {params.omega_v}\nw0 = -1.0\nwa = 0.0\nns = {params.n_s}\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = {params.hmcode_logT_AGN}\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "record_format_version": 6, "conda_env": null, "container_img_url": "/n17data/cdaley/containers/containers", "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8766909525762, "starttime": 1783693357.3336766, "endtime": 1783693343.2093573, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json": "sha256:c44a19ac7749152357d3bb7bb6781783c0da2f76083c7783abfc5377ec296703"}} \ No newline at end of file diff --git a/workflow/common.py b/workflow/common.py index 1b3605924..5432533b5 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -202,7 +202,7 @@ def build_redshift_dir(version): return ( f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/" ) - + def build_redshift_path_lens(version, blind): """Construct n(z) filepath for given catalog version and blind.""" base_version = re.sub(r"_leak_corr$", "", version) From 23ddd1b8945dd63b487be4133f8a3d91e47e1a8e Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Fri, 10 Jul 2026 18:12:51 +0200 Subject: [PATCH 03/14] ruff format --- workflow/common.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/workflow/common.py b/workflow/common.py index 5432533b5..c0f51e6b3 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -9,15 +9,19 @@ # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL = Path( os.environ.get( - "COSMO_VAL", "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmo_val" + "COSMO_VAL", + "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmo_val", ) ) COSMO_INFERENCE = Path( os.environ.get( - "COSMO_INFERENCE", "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference" + "COSMO_INFERENCE", + "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference", ) ) -CAT_CONFIG = "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_val/cat_config.yaml" +CAT_CONFIG = ( + "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_val/cat_config.yaml" +) BLINDS = ["A", "B", "C"] BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")] @@ -64,6 +68,7 @@ def configure(workflow_config): with open(COSMOLOGY_PARAMS) as f: PLANCK18 = json.load(f) + def fiducial_binning_suffix(fiducial=None): """Return binning suffix for fiducial parameters.""" fiducial = fiducial or FIDUCIAL @@ -86,6 +91,7 @@ def resolve_covariance_version(version): "_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_{probe}{mask_suffix}" ) + def cov_output(suffix): """Wildcard-bearing output path under the covariance tree. @@ -94,9 +100,11 @@ def cov_output(suffix): with {version}, {blind}, ... left as Snakemake wildcards. """ return str( - COSMO_INFERENCE / f"data/covariance/{COV_BASE_TEMPLATE}/{COV_BASE_TEMPLATE}{suffix}" + COSMO_INFERENCE + / f"data/covariance/{COV_BASE_TEMPLATE}/{COV_BASE_TEMPLATE}{suffix}" ) + def cov_output_dirfile(filename): """Wildcard-bearing path to a fixed-name file inside the covariance dir.""" return str(COSMO_INFERENCE / f"data/covariance/{COV_BASE_TEMPLATE}/{filename}") @@ -199,9 +207,8 @@ def build_redshift_dir(version): if "v1.4.11" in base_version: base_version = "SP_v1.4.6" version_dir = base_version.replace("SP_", "") - return ( - f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/" - ) + return f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/" + def build_redshift_path_lens(version, blind): """Construct n(z) filepath for given catalog version and blind.""" @@ -209,9 +216,8 @@ def build_redshift_path_lens(version, blind): base_version = re.sub(r"_ecut\d+", "", base_version) if "v1.4.11" in base_version: base_version = "SP_v1.4.6" - return ( - f"nz_{base_version}_{blind}.txt" - ) + return f"nz_{base_version}_{blind}.txt" + def build_redshift_path_source(version, blind): """Construct n(z) filepath for given catalog version and blind.""" @@ -219,9 +225,8 @@ def build_redshift_path_source(version, blind): base_version = re.sub(r"_ecut\d+", "", base_version) if "v1.4.11" in base_version: base_version = "SP_v1.4.6" - return ( - f"nz_{base_version}_{blind}_source.txt" #TO DO: check lens and source file format - ) + return f"nz_{base_version}_{blind}_source.txt" # TO DO: check lens and source file format + def get_shear_catalog(wildcards): """Resolve shear catalog path from config for a given version.""" From ac454fc5fb613ed746b91c70123a6474f09f39be Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Fri, 10 Jul 2026 18:16:23 +0200 Subject: [PATCH 04/14] ruff format --- workflow/Snakefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 9624ed58d..6f59827b4 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -36,11 +36,11 @@ wildcard_constraints: **WILDCARD_CONSTRAINTS # Compute rules (infrastructure — raw outputs, no evidence.json) -# include: "rules/twopoint.smk" +include: "rules/twopoint.smk" include: "rules/covariance.smk" -# include: "rules/inference.smk" -# include: "rules/masks.smk" -# include: "rules/glass_mock.smk" +include: "rules/inference.smk" +include: "rules/masks.smk" +include: "rules/glass_mock.smk" # CosmologyValidation diagnostic suite (cosmo_val.py), decomposed rule-by-rule. # Only active when the config carries a `cosmo_val` block (papers/cosmo_val); From 46953c6c2a6dde2057ef0fe68e9481f813da5719 Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Tue, 14 Jul 2026 14:26:34 +0200 Subject: [PATCH 05/14] edits after review --- ...NlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 | 1 + ...NlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p | 1 + scratch/lgoh/cov_config_test.yaml | 2 +- workflow/Snakefile | 8 ++-- workflow/common.py | 26 ++++------- workflow/rules/covariance.smk | 45 ++++++++++--------- 6 files changed, 41 insertions(+), 42 deletions(-) create mode 100644 .snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 create mode 100644 .snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p diff --git a/.snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 b/.snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 new file mode 100644 index 000000000..bf402e398 --- /dev/null +++ b/.snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 @@ -0,0 +1 @@ +{"rule": "covariance_onecov", "input": ["/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini"], "log": [], "shellcmd": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n\n cd /automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked\n /n23data1/n06data/lgoh/scratch/my_env_new/bin/python /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py /automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini\n ", "params": ["'/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked'", "'/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini'", "'/n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py'", "'/n23data1/n06data/lgoh/scratch/my_env_new/bin/python'"], "code": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n\n cd {params.outdir}\n {params.python_executable} {params.onecov} {params.ini_path}\n ", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8779245703989, "starttime": 1784031091.2697892, "endtime": 1784031439.5140738, "incomplete": false, "external_jobid": null, "input_checksums": {"/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini": "sha256:989b5d6ab000eb31d40ddb7ea1b27460d65203ac1402f946e4fa3bb174cd85b3"}} \ No newline at end of file diff --git a/.snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p b/.snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p new file mode 100644 index 000000000..76c66a330 --- /dev/null +++ b/.snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p @@ -0,0 +1 @@ +{"rule": "covariance_ini_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json"], "log": [], "shellcmd": "\n mkdir -p /automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked\n\n cat > \"/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = 0\nssc = 0\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = False\nggl = False\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = /automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked\nfile = covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.dat\nstyle = matrix\nsave_configs = False\ncorrmatrix_plot = covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked_corr_plot.pdf\n\n[covTHETAspace settings]\ntheta_min = 1.0\ntheta_max = 100.0\ntheta_bins = 20\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = /automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/mask\nmask_file_lensing = mask_cls_footprint_nside_4096_norm.txt\nmask_file_clust = \nmask_file_ggl = \nellipticity_dispersion = 0.28, 0.28\nsurvey_area_lensing_in_deg2 = 1000\n\n\nn_eff_lensing = 5.0, 5.0\n\n\n[redshift]\nzlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/\n# /n17data/sguerrini/UNIONS/WL/nz/v1.4.6/\nzlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii\n# nz_SP_v1.4.6_A_lens.txt\nzclust_file = \nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = 0.8111\nh = 0.6736\nomega_m = 0.3153\nomega_b = 0.0493\nomega_de = 0.6847\nw0 = -1.0\nwa = 0.0\nns = 0.9649\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = 7.75\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "params": ["''", "''", "''", "''", "''", "''", "'/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked'", "'/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/mask'", "'/n17data/sguerrini/UNIONS/WL/nz/v1.4.6/'", "'0'", "'0.28, 0.28'", "'False'", "'False'", "'covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.dat'", "'covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked_corr_plot.pdf'", "'mask_cls_footprint_nside_4096_norm.txt'", "'n_eff_lensing = 5.0, 5.0'", "'nz_SP_v1.4.6_A_lens.txt'", "'survey_area_lensing_in_deg2 = 1000'", "0.0493", "0.3153", "0.6736", "0.6847", "0.8111", "0.9649", "7.75"], "code": "\n mkdir -p {params.outdir}\n\n cat > \"{output}\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = {params.ng_value}\nssc = {params.ng_value}\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = {params.do_clustering}\nggl = {params.do_ggl}\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = {params.outdir}\nfile = {params.out_filename}\nstyle = matrix\nsave_configs = False\ncorrmatrix_plot = {params.out_plot_filename}\n\n[covTHETAspace settings]\ntheta_min = {wildcards.min_sep}\ntheta_max = {wildcards.max_sep}\ntheta_bins = {wildcards.nbins}\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = {params.mask_dir}\nmask_file_lensing = {params.mask_lens}\nmask_file_clust = {params.mask_clust}\nmask_file_ggl = {params.mask_ggl}\nellipticity_dispersion = {params.sigma_e_param}\n{params.area_lens_line}\n{params.area_clust_line}\n{params.area_ggl_line}\n{params.n_e_lens_line}\n{params.n_e_clust_line}\n\n[redshift]\nzlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/\n# {params.nz_dir}\nzlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii\n# {params.nz_lens}\nzclust_file = {params.nz_clust}\nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = {params.sigma_8}\nh = {params.h}\nomega_m = {params.omega_m}\nomega_b = {params.omega_b}\nomega_de = {params.omega_v}\nw0 = -1.0\nwa = 0.0\nns = {params.n_s}\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = {params.hmcode_logT_AGN}\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = {ONECOV_THREADS}\n\nEOF\n ", "record_format_version": 6, "conda_env": null, "container_img_url": "/n17data/cdaley/containers/containers", "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8779245704142, "starttime": 1784031075.6630247, "endtime": 1784031211.737958, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json": "sha256:c44a19ac7749152357d3bb7bb6781783c0da2f76083c7783abfc5377ec296703"}} \ No newline at end of file diff --git a/scratch/lgoh/cov_config_test.yaml b/scratch/lgoh/cov_config_test.yaml index ebe8462cd..a5eade514 100644 --- a/scratch/lgoh/cov_config_test.yaml +++ b/scratch/lgoh/cov_config_test.yaml @@ -8,7 +8,7 @@ fiducial: mock_version: SP_v1.4.6 gaussian: g -probe_3x2pt: wl +probe: wl glass_mocks: version: "v0" seed_range: [1, 350] diff --git a/workflow/Snakefile b/workflow/Snakefile index 6f59827b4..9624ed58d 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -36,11 +36,11 @@ wildcard_constraints: **WILDCARD_CONSTRAINTS # Compute rules (infrastructure — raw outputs, no evidence.json) -include: "rules/twopoint.smk" +# include: "rules/twopoint.smk" include: "rules/covariance.smk" -include: "rules/inference.smk" -include: "rules/masks.smk" -include: "rules/glass_mock.smk" +# include: "rules/inference.smk" +# include: "rules/masks.smk" +# include: "rules/glass_mock.smk" # CosmologyValidation diagnostic suite (cosmo_val.py), decomposed rule-by-rule. # Only active when the config carries a `cosmo_val` block (papers/cosmo_val); diff --git a/workflow/common.py b/workflow/common.py index c0f51e6b3..ea7210ce6 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -7,21 +7,13 @@ # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. -COSMO_VAL = Path( - os.environ.get( - "COSMO_VAL", - "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmo_val", - ) -) +SP_VALIDATION = Path(__file__).resolve().parents[1] +COSMO_VAL = Path(os.environ.get("COSMO_VAL", SP_VALIDATION / "results/cosmo_val")) COSMO_INFERENCE = Path( - os.environ.get( - "COSMO_INFERENCE", - "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference", - ) -) -CAT_CONFIG = ( - "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_val/cat_config.yaml" + os.environ.get("COSMO_INFERENCE", SP_VALIDATION / "cosmo_inference") ) +CAT_CONFIG = SP_VALIDATION / "cosmo_val/cat_config.yaml" + BLINDS = ["A", "B", "C"] BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")] @@ -51,7 +43,7 @@ FIDUCIAL = None DEFAULT_MASK_SUFFIX = "" -DEFAULT_PROBE = None +DEFAULT_PROBE = "wl" CATALOG_CONFIG = None PLANCK18 = None @@ -61,7 +53,7 @@ def configure(workflow_config): global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, DEFAULT_PROBE, FIDUCIAL, PLANCK18 CATALOG_CONFIG = workflow_config FIDUCIAL = workflow_config["fiducial"] - DEFAULT_PROBE = "wl" + DEFAULT_PROBE = workflow_config.get("probe") or "wl" DEFAULT_MASK_SUFFIX = ( "_masked" if workflow_config["covariance"].get("default_masked", False) else "" ) @@ -216,7 +208,7 @@ def build_redshift_path_lens(version, blind): base_version = re.sub(r"_ecut\d+", "", base_version) if "v1.4.11" in base_version: base_version = "SP_v1.4.6" - return f"nz_{base_version}_{blind}.txt" + return f"nz_{base_version}_{blind}_lens.txt" def build_redshift_path_source(version, blind): @@ -225,7 +217,7 @@ def build_redshift_path_source(version, blind): base_version = re.sub(r"_ecut\d+", "", base_version) if "v1.4.11" in base_version: base_version = "SP_v1.4.6" - return f"nz_{base_version}_{blind}_source.txt" # TO DO: check lens and source file format + return f"nz_{base_version}_{blind}_clust.txt" # TO DO: check lens and source file format def get_shear_catalog(wildcards): diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index 1a991e309..c823b15d1 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -1,17 +1,17 @@ # BLOCK_PAIRS, PLANCK18, COSMOLOGY_PARAMS defined in Snakefile import os -def get_cat_params(version): +def get_cat_params(w): """Extract covariance parameters (sigma_e, n_eff, area) from catalog config. Returns (sigma_e, (n_e_lens, n_e_clust), (A_lens, A_ggl, A_clust)). For probe == "wl" the clustering/ggl slots are empty strings. """ - base_version = version.replace("_leak_corr", "") + base_version = w.version.replace("_leak_corr", "") if base_version not in config: raise KeyError(f"Catalog configuration not found for {base_version}") cov_th = config[base_version]["cov_th"] - if config["probe_3x2pt"] == "wl": + if w.probe == "wl": return cov_th["sigma_e"], (cov_th["n_e"], ""), (cov_th["A"], "", "") return ( cov_th["sigma_e"], @@ -35,6 +35,9 @@ MASK_CLS_FILES = { # v1.4.8 uses the star-halo footprint; all other versions use the standard footprint STARHALO_VERSIONS = {"v1.4.8"} +# Number of cores to use to execute OneCovariance +ONECOV_THREADS = 8 + def get_mask_cls_file(version, kind="lens"): """Return the mask Cl *filename* (OneCov wants dir and file separately).""" version_dir = version.replace("_leak_corr", "").replace("SP_", "") @@ -117,33 +120,33 @@ rule covariance_ini_onecov: h=PLANCK18["h"], omega_b=PLANCK18["Omega_b"], hmcode_logT_AGN=7.75, - sigma_e_param=lambda w: get_cat_params(w.version)[0], + sigma_e_param=lambda w: get_cat_params(w)[0], n_e_lens_line=lambda w: ( - f"n_eff_lensing = {get_cat_params(w.version)[1][0]}"), + f"n_eff_lensing = {get_cat_params(w)[1][0]}"), n_e_clust_line=lambda w: ("" if w.probe == "wl" - else f"n_eff_clust = {get_cat_params(w.version)[1][1]}"), + else f"n_eff_clust = {get_cat_params(w)[1][1]}"), area_lens_line=lambda w: ( - f"survey_area_lensing_in_deg2 = {get_cat_params(w.version)[2][0]}"), + f"survey_area_lensing_in_deg2 = {get_cat_params(w)[2][0]}"), area_ggl_line=lambda w: ( "" if w.probe == "wl" - else f"survey_area_ggl_in_deg2 = {get_cat_params(w.version)[2][1]}" + else f"survey_area_ggl_in_deg2 = {get_cat_params(w)[2][1]}" ), area_clust_line=lambda w: ( "" if w.probe == "wl" - else f"survey_area_clust_in_deg2 = {get_cat_params(w.version)[2][2]}" + else f"survey_area_clust_in_deg2 = {get_cat_params(w)[2][2]}" ), mask_dir=lambda w: _onecov_mask_params(w)["dir"], mask_lens=lambda w: _onecov_mask_params(w)["lens"], mask_ggl=lambda w: _onecov_mask_params(w)["ggl"], mask_clust=lambda w: _onecov_mask_params(w)["clust"], - nz_dir=lambda w: os.path.dirname(build_redshift_dir(w.version)), - nz_lens=lambda w: os.path.basename(build_redshift_path_lens(w.version, w.blind)), + nz_dir=lambda w: build_redshift_dir(w.version), + nz_lens=lambda w: build_redshift_path_lens(w.version, w.blind), nz_clust=lambda w: ( "" if w.probe == "wl" - else os.path.basename(build_redshift_path_source(w.version, w.blind)) + else build_redshift_path_source(w.version, w.blind) ), threads: 1 shell: @@ -196,8 +199,10 @@ ellipticity_dispersion = {params.sigma_e_param} {params.n_e_clust_line} [redshift] -zlens_directory = {params.nz_dir} -zlens_file = {params.nz_lens} +zlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/ +# {params.nz_dir} +zlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii +# {params.nz_lens} zclust_file = {params.nz_clust} value_loc_in_lensbin = mid value_loc_in_clustbin = mid @@ -256,7 +261,7 @@ transfer_model = CAMB small_k_damping_for1h = damped [misc] -num_cores = 8 +num_cores = {ONECOV_THREADS} EOF """ @@ -265,7 +270,6 @@ rule covariance_onecov: input: rules.covariance_ini_onecov.output, output: - # Matches `[output settings] file = cov_tmp_onecov.dat` in the ini. cov_output(".dat") params: outdir=lambda w: covariance_dir( @@ -280,7 +284,7 @@ rule covariance_onecov: python_executable=config["tools"]["python_executable"], container: None - threads: 8 + threads: ONECOV_THREADS shell: """ module unload gcc || true @@ -300,7 +304,7 @@ GLASS_MOCK_RESULTS = config.get("glass_mocks", {}).get( ) GLASS_MOCK_COV_DIR = config.get("glass_mocks", {}).get( "covariance_dir", - "/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6", + "/automnt/n17data/cdaley/unions/analyses/shear_2d/bmodes_2d/results/covariance/glass_mock_v1.4.6", ) rule covariance_glass_mock: @@ -364,8 +368,9 @@ rule generate_glass_mock_rhotau_samples: """ -def fiducial_covariance_outputs(mask_suffix="", probe=DEFAULT_PROBE): +def fiducial_covariance_outputs(mask_suffix="", probe=None): """Return processed covariance files for fiducial version/blind.""" + probe = probe or DEFAULT_PROBE path = covariance_path( FIDUCIAL["version"], FIDUCIAL["blind"], FIDUCIAL["gaussian"], FIDUCIAL["min_sep"], FIDUCIAL["max_sep"], FIDUCIAL["nbins"], @@ -388,7 +393,7 @@ rule covariance_unmasked: rule covariance_3x2pt: input: fiducial_covariance_outputs(mask_suffix="", - probe=config["probe_3x2pt"]), + probe="3x2pt"), # ruleorder: covariance_ini_onecov > covariance_onecov > covariance From b546c4625b8b16beb8ce43d1e8022581b1d4fcd4 Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Tue, 14 Jul 2026 14:27:58 +0200 Subject: [PATCH 06/14] edits after review --- .../A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 | 1 - .../A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p | 1 - 2 files changed, 2 deletions(-) delete mode 100644 .snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 delete mode 100644 .snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p diff --git a/.snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 b/.snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 deleted file mode 100644 index bf402e398..000000000 --- a/.snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuZGF0 +++ /dev/null @@ -1 +0,0 @@ -{"rule": "covariance_onecov", "input": ["/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini"], "log": [], "shellcmd": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n\n cd /automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked\n /n23data1/n06data/lgoh/scratch/my_env_new/bin/python /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py /automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini\n ", "params": ["'/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked'", "'/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini'", "'/n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/covariance.py'", "'/n23data1/n06data/lgoh/scratch/my_env_new/bin/python'"], "code": "\n module unload gcc || true\n module load gcc\n module unload intelpython || true\n module load intelpython/3-2024.1.0\n module load openmpi\n\n cd {params.outdir}\n {params.python_executable} {params.onecov} {params.ini_path}\n ", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8779245703989, "starttime": 1784031091.2697892, "endtime": 1784031439.5140738, "incomplete": false, "external_jobid": null, "input_checksums": {"/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini": "sha256:989b5d6ab000eb31d40ddb7ea1b27460d65203ac1402f946e4fa3bb174cd85b3"}} \ No newline at end of file diff --git a/.snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p b/.snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p deleted file mode 100644 index 76c66a330..000000000 --- a/.snakemake/metadata/@L2F1dG9tbnQvbjIzZGF0YTEvbjA2ZGF0YS9sZ29oL3NjcmF0Y2gvVU5JT05TL3NwX3ZhbGlkYXRpb24vY29zbW9faW5mZXJlbmNlL2RhdGEvY292YXJpYW5jZS9jb3ZhcmlhbmNlX1NQX3YxLjQuNl9BX2dfbWluc2VwPTEuMF9tYXhzZXA9MTAwLjBfbmJpbnM9MjBfd2xfbWFza2VkL2NvdmFyaWFuY2VfU1BfdjEuNC42X0FfZ19taW5zZX/A9MS4wX21heHNlcD0xMDAuMF9uYmlucz0yMF93bF9tYXNrZWQuaW5p +++ /dev/null @@ -1 +0,0 @@ -{"rule": "covariance_ini_onecov", "input": ["/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json"], "log": [], "shellcmd": "\n mkdir -p /automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked\n\n cat > \"/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.ini\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = 0\nssc = 0\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = False\nggl = False\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = /automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked\nfile = covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.dat\nstyle = matrix\nsave_configs = False\ncorrmatrix_plot = covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked_corr_plot.pdf\n\n[covTHETAspace settings]\ntheta_min = 1.0\ntheta_max = 100.0\ntheta_bins = 20\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = /automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/mask\nmask_file_lensing = mask_cls_footprint_nside_4096_norm.txt\nmask_file_clust = \nmask_file_ggl = \nellipticity_dispersion = 0.28, 0.28\nsurvey_area_lensing_in_deg2 = 1000\n\n\nn_eff_lensing = 5.0, 5.0\n\n\n[redshift]\nzlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/\n# /n17data/sguerrini/UNIONS/WL/nz/v1.4.6/\nzlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii\n# nz_SP_v1.4.6_A_lens.txt\nzclust_file = \nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = 0.8111\nh = 0.6736\nomega_m = 0.3153\nomega_b = 0.0493\nomega_de = 0.6847\nw0 = -1.0\nwa = 0.0\nns = 0.9649\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = 7.75\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = 8\n\nEOF\n ", "params": ["''", "''", "''", "''", "''", "''", "'/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked'", "'/automnt/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/cosmo_inference/data/mask'", "'/n17data/sguerrini/UNIONS/WL/nz/v1.4.6/'", "'0'", "'0.28, 0.28'", "'False'", "'False'", "'covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked.dat'", "'covariance_SP_v1.4.6_A_g_minsep=1.0_maxsep=100.0_nbins=20_wl_masked_corr_plot.pdf'", "'mask_cls_footprint_nside_4096_norm.txt'", "'n_eff_lensing = 5.0, 5.0'", "'nz_SP_v1.4.6_A_lens.txt'", "'survey_area_lensing_in_deg2 = 1000'", "0.0493", "0.3153", "0.6736", "0.6847", "0.8111", "0.9649", "7.75"], "code": "\n mkdir -p {params.outdir}\n\n cat > \"{output}\" << 'EOF'\n[covariance terms]\ngauss = True\nsplit_gauss = False\nnongauss = {params.ng_value}\nssc = {params.ng_value}\nsn_only = False\n\n[observables]\ncosmic_shear = True\nest_shear = xi_pm\nclustering = {params.do_clustering}\nggl = {params.do_ggl}\nest_ggl = gamma_t\nest_clust = w\n\n[output settings]\ndirectory = {params.outdir}\nfile = {params.out_filename}\nstyle = matrix\nsave_configs = False\ncorrmatrix_plot = {params.out_plot_filename}\n\n[covTHETAspace settings]\ntheta_min = {wildcards.min_sep}\ntheta_max = {wildcards.max_sep}\ntheta_bins = {wildcards.nbins}\ntheta_type = log\nxi_pp = True\nxi_mm = True\ntheta_accuracy = 1e-3\nintegration_intervals = 400\n\n[survey specs]\nmask_directory = {params.mask_dir}\nmask_file_lensing = {params.mask_lens}\nmask_file_clust = {params.mask_clust}\nmask_file_ggl = {params.mask_ggl}\nellipticity_dispersion = {params.sigma_e_param}\n{params.area_lens_line}\n{params.area_clust_line}\n{params.area_ggl_line}\n{params.n_e_lens_line}\n{params.n_e_clust_line}\n\n[redshift]\nzlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/\n# {params.nz_dir}\nzlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii\n# {params.nz_lens}\nzclust_file = {params.nz_clust}\nvalue_loc_in_lensbin = mid\nvalue_loc_in_clustbin = mid\n\n[cosmo]\nsigma8 = {params.sigma_8}\nh = {params.h}\nomega_m = {params.omega_m}\nomega_b = {params.omega_b}\nomega_de = {params.omega_v}\nw0 = -1.0\nwa = 0.0\nns = {params.n_s}\nneff = 3.046\nm_nu = 0.06\ntcmb0 = 2.725\n\n[IA]\nA_IA = 0.0\neta_IA = 0.0\nz_pivot_IA = 0.3\n\n[powspec evaluation]\nnon_linear_model = mead2020_feedback\nHMCode_logT_AGN = {params.hmcode_logT_AGN}\nlog10k_bins = 300\nlog10k_min = -3.46\nlog10k_max = 3.15\n\n[hod]\nmodel_mor_cen = double_powerlaw\nmodel_mor_sat = double_powerlaw\ndpow_logm0_cen = 10.51\ndpow_logm1_cen = 11.38\ndpow_a_cen = 7.096\ndpow_b_cen = 0.2\ndpow_norm_cen = 1.0\ndpow_norm_sat = 0.56\nmodel_scatter_cen = lognormal\nmodel_scatter_sat = modschechter\nlogn_sigma_c_cen = 0.35\nmodsch_logmref_sat = 13.0\nmodsch_alpha_s_sat = -0.858\nmodsch_b_sat = -0.024, 1.149\n\n[halomodel evaluation]\nm_bins = 900\nlog10m_min = 6\nlog10m_max = 18\nhmf_model = Tinker10\nmdef_model = SOMean\nmdef_params = overdensity, 200\ndisable_mass_conversion = True\ndelta_c = 1.686\ntransfer_model = CAMB\nsmall_k_damping_for1h = damped\n\n[misc]\nnum_cores = {ONECOV_THREADS}\n\nEOF\n ", "record_format_version": 6, "conda_env": null, "container_img_url": "/n17data/cdaley/containers/containers", "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8779245704142, "starttime": 1784031075.6630247, "endtime": 1784031211.737958, "incomplete": false, "external_jobid": null, "input_checksums": {"/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json": "sha256:c44a19ac7749152357d3bb7bb6781783c0da2f76083c7783abfc5377ec296703"}} \ No newline at end of file From 54f578377f6decafaf8d1f882f9aec822ba8ee3d Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Tue, 14 Jul 2026 14:48:19 +0200 Subject: [PATCH 07/14] updated path files --- workflow/Snakefile | 8 ++++---- workflow/common.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 9624ed58d..6f59827b4 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -36,11 +36,11 @@ wildcard_constraints: **WILDCARD_CONSTRAINTS # Compute rules (infrastructure — raw outputs, no evidence.json) -# include: "rules/twopoint.smk" +include: "rules/twopoint.smk" include: "rules/covariance.smk" -# include: "rules/inference.smk" -# include: "rules/masks.smk" -# include: "rules/glass_mock.smk" +include: "rules/inference.smk" +include: "rules/masks.smk" +include: "rules/glass_mock.smk" # CosmologyValidation diagnostic suite (cosmo_val.py), decomposed rule-by-rule. # Only active when the config carries a `cosmo_val` block (papers/cosmo_val); diff --git a/workflow/common.py b/workflow/common.py index ea7210ce6..694518c82 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -21,7 +21,7 @@ # Source of truth: cs_util.cosmo.PLANCK18 # Regenerate with: snakemake results/cosmology/planck18.json # Resolved relative to the run directory at configure() time. -COSMOLOGY_PARAMS = "/n23data1/n06data/lgoh/scratch/UNIONS/sp_validation/results/cosmology/planck18.json" +COSMOLOGY_PARAMS = "results/cosmology/planck18.json" # Wildcard constraints shared by every Snakefile that composes these rules. # Patterns must match all expected values; overly restrictive patterns cause From b0e455b9b56e5708388b328b90f472f5481a7e16 Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Mon, 27 Jul 2026 18:08:25 +0200 Subject: [PATCH 08/14] update snakemake files with tomographic option --- workflow/common.py | 25 ++++++++++++++++++++++++- workflow/rules/covariance.smk | 21 ++++++++++----------- workflow/rules/twopoint.smk | 11 +++++++---- workflow/scripts/run_2pcf.py | 32 ++++++++++++++++++-------------- 4 files changed, 59 insertions(+), 30 deletions(-) diff --git a/workflow/common.py b/workflow/common.py index 694518c82..1a7273ca0 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -30,6 +30,8 @@ "version": r"SP_v[\d.]+(_w_iv)?(_ecut\d+)?(_leak_corr)?", "blind": r"[ABC]", "nbins": r"\d+", + # Without this, npatch greedily swallows a trailing {tomo_suffix}. + "npatch": r"\d+", "min_sep": r"[0-9.]+", "max_sep": r"[0-9.]+", "gaussian": r"(g|ng)", @@ -37,6 +39,8 @@ "block_pm": r"(\+\+|--|\+-)", "block_i": r"[123]", "mask_suffix": r"(_masked)?", + # Empty means non-tomographic, so existing paths are unchanged. + "tomo_suffix": r"(_tomo)?", "mock_id": r"\d{5}", "nside": r"\d+", } @@ -80,9 +84,16 @@ def resolve_covariance_version(version): # Keep the two in sync by construction: covariance_base() formats this string. COV_BASE_TEMPLATE = ( "covariance_{version}_{blind}_{gaussian}" - "_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_{probe}{mask_suffix}" + "_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_{probe}" + "{mask_suffix}{tomo_suffix}" ) +# Tomographic covariances are defined for cosmic shear only. The pattern above +# has to carry {tomo_suffix} for every probe (a wildcard cannot be conditional +# on another wildcard), so the restriction is enforced in covariance_base() +# instead — every dir/path/base helper routes through it. +TOMO_PROBES = {"wl"} + def cov_output(suffix): """Wildcard-bearing output path under the covariance tree. @@ -111,6 +122,7 @@ def covariance_base( nbins=None, probe=None, mask_suffix=None, + tomo_suffix=None, resolve_version=True, fiducial=None, default_mask_suffix=None, @@ -126,6 +138,12 @@ def covariance_base( mask_suffix = ( DEFAULT_MASK_SUFFIX if default_mask_suffix is None else default_mask_suffix ) + tomo_suffix = tomo_suffix or "" + if tomo_suffix and probe not in TOMO_PROBES: + raise ValueError( + f"Tomographic covariance is only defined for probe in {sorted(TOMO_PROBES)}, " + f"got probe={probe!r} with tomo_suffix={tomo_suffix!r}." + ) cov_version = resolve_covariance_version(version) if resolve_version else version return COV_BASE_TEMPLATE.format( version=cov_version, @@ -136,6 +154,7 @@ def covariance_base( nbins=nbins, probe=probe, mask_suffix=mask_suffix, + tomo_suffix=tomo_suffix, ) @@ -148,6 +167,7 @@ def covariance_dir( nbins=None, probe=None, mask_suffix=None, + tomo_suffix=None, resolve_version=True, ): """Construct covariance directory path.""" @@ -160,6 +180,7 @@ def covariance_dir( nbins, probe, mask_suffix, + tomo_suffix, resolve_version=resolve_version, ) return str(COSMO_INFERENCE / f"data/covariance/{base}") @@ -174,6 +195,7 @@ def covariance_path( nbins=None, probe=None, mask_suffix=None, + tomo_suffix=None, suffix=".dat", resolve_version=True, ): @@ -187,6 +209,7 @@ def covariance_path( nbins, probe, mask_suffix, + tomo_suffix, resolve_version=resolve_version, ) return str(COSMO_INFERENCE / f"data/covariance/{base}/{base}{suffix}") diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index c823b15d1..bd9ce6db0 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -100,15 +100,15 @@ rule covariance_ini_onecov: params: outdir=lambda w: covariance_dir( w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, - w.probe, w.mask_suffix, resolve_version=False, + w.probe, w.mask_suffix, w.tomo_suffix, resolve_version=False, ), out_filename=lambda w: covariance_base( w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, - w.probe, w.mask_suffix, resolve_version=False, + w.probe, w.mask_suffix, w.tomo_suffix, resolve_version=False, ) + ".dat", out_plot_filename=lambda w: covariance_base( w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, - w.probe, w.mask_suffix, resolve_version=False, + w.probe, w.mask_suffix, w.tomo_suffix, resolve_version=False, ) + "_corr_plot.pdf", ng_value=lambda w: "1" if w.gaussian == "ng" else "0", do_ggl=lambda w: str(w.probe in ("ggl", "3x2pt")), @@ -199,10 +199,8 @@ ellipticity_dispersion = {params.sigma_e_param} {params.n_e_clust_line} [redshift] -zlens_directory = /n23data1/n06data/lgoh/scratch/UNIONS/OneCovariance/input/redshift_distribution/ -# {params.nz_dir} -zlens_file = K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO1_Nz.ascii,K1000_NS_V1.0.0A_ugriZYJHKs_photoz_SG_mask_LF_svn_309c_2Dbins_v2_goldclasses_THELI_INT_DIRcols_Fid_blindC_TOMO2_Nz.ascii -# {params.nz_lens} +zlens_directory = {params.nz_dir} +zlens_file = {params.nz_lens} zclust_file = {params.nz_clust} value_loc_in_lensbin = mid value_loc_in_clustbin = mid @@ -274,11 +272,12 @@ rule covariance_onecov: params: outdir=lambda w: covariance_dir( w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, - w.probe, w.mask_suffix, resolve_version=False, + w.probe, w.mask_suffix, w.tomo_suffix, resolve_version=False, ), ini_path=lambda w: covariance_path( w.version, w.blind, w.gaussian, w.min_sep, w.max_sep, w.nbins, - w.probe, w.mask_suffix, suffix=".ini", resolve_version=False, + w.probe, w.mask_suffix, w.tomo_suffix, suffix=".ini", + resolve_version=False, ), onecov=config["tools"]["onecov_executable"], python_executable=config["tools"]["python_executable"], @@ -368,13 +367,13 @@ rule generate_glass_mock_rhotau_samples: """ -def fiducial_covariance_outputs(mask_suffix="", probe=None): +def fiducial_covariance_outputs(mask_suffix="", probe=None, tomo_suffix=""): """Return processed covariance files for fiducial version/blind.""" probe = probe or DEFAULT_PROBE path = covariance_path( FIDUCIAL["version"], FIDUCIAL["blind"], FIDUCIAL["gaussian"], FIDUCIAL["min_sep"], FIDUCIAL["max_sep"], FIDUCIAL["nbins"], - probe, mask_suffix, + probe, mask_suffix, tomo_suffix, ) return path diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 22c09db2f..1b8a22a76 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -5,17 +5,20 @@ rule xi: input: catalog=get_shear_catalog, output: - str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + # Restore output files when a sacc_io writer is available. + # str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), + # str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + # str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + touch(str(CV_SENTINELS / "xi_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}{tomo_suffix}.done")), threads: 24 params: ver="{version}", + compute_tomography=lambda w: w.tomo_suffix == "_tomo", min_sep="{min_sep}", max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", - fits=False, + cat_config=CAT_CONFIG, resources: mem_mb=30000, disk_mb=20000, diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 2e1ccabf7..d91aa8029 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -12,10 +12,11 @@ --cat-config /path/to/cosmo_val/cat_config.yaml \ --out -The measurement itself is unchanged — ``CosmologyValidation.calculate_2pcf`` -does the TreeCorr work and writes the ``.txt`` dump plus ξ+/ξ- FITS files into -``output_dir``. ``output_dir`` is passed explicitly (rather than via the -``COSMO_VAL`` env hook) so lc can point each run at its own ``{output}`` tree. +``CosmologyValidation.calculate_2pcf`` does the TreeCorr work and returns the +ξ± ``GGCorrelation`` objects in memory; it no longer writes the ``.txt`` dump +or the ξ+/ξ- FITS files. ``output_dir`` is still passed explicitly (rather than +via the ``COSMO_VAL`` env hook) so the patch-centre file lands in each run's own +``{output}`` tree. """ import argparse @@ -25,15 +26,15 @@ def run_2pcf( ver, + npatch, + compute_tomography, min_sep, max_sep, nbins, - npatch, cat_config, output_dir, - save_fits=True, ): - """Measure ξ±(θ) for ``ver`` and write it under ``output_dir``. + """Measure ξ±(θ) for ``ver``. Parameters mirror the TreeCorr reporting/integration grids: ``min_sep`` / ``max_sep`` in arcmin, ``nbins`` logarithmic bins, ``npatch`` spatial @@ -47,9 +48,8 @@ def run_2pcf( output_dir=output_dir, ) return cv.calculate_2pcf( - ver=ver, npatch=npatch, - save_fits=save_fits, + compute_tomography=compute_tomography, min_sep=min_sep, max_sep=max_sep, nbins=nbins, @@ -60,17 +60,17 @@ def _from_snakemake(smk): p = smk.params run_2pcf( ver=p["ver"], + npatch=int(p["npatch"]), + compute_tomography=p.get("compute_tomography", False), min_sep=float(p["min_sep"]), max_sep=float(p["max_sep"]), nbins=int(p["nbins"]), - npatch=int(p["npatch"]), # cat_config / output_dir were previously resolved via an os.chdir into # the cosmo_val dir + the COSMO_VAL env var; expose them as optional # params so the rule can pass them explicitly, falling back to the # class defaults (./cat_config.yaml, COSMO_VAL env) otherwise. cat_config=p.get("cat_config", "./cat_config.yaml"), output_dir=p.get("output_dir", None), - save_fits=True, ) @@ -83,6 +83,11 @@ def _from_cli(argv=None): required=True, help="Catalog version key in cat_config, e.g. SP_v1.4.6.3_leak_corr", ) + ap.add_argument( + "--tomo", + action="store_true", + help="Compute tomographic ξ±(θ) measurement. True or False.", + ) ap.add_argument( "--min-sep", type=float, required=True, help="Min separation [arcmin]" ) @@ -97,17 +102,16 @@ def _from_cli(argv=None): "--cat-config", required=True, help="Absolute path to cat_config.yaml" ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") - ap.add_argument("--no-fits", action="store_true", help="Skip ξ+/ξ- FITS export") a = ap.parse_args(argv) run_2pcf( ver=a.ver, + npatch=a.npatch, + compute_tomography=a.tomo, min_sep=a.min_sep, max_sep=a.max_sep, nbins=a.nbins, - npatch=a.npatch, cat_config=a.cat_config, output_dir=a.out, - save_fits=not a.no_fits, ) From b1f50150831a6a5c7498a69cb66f955946e8b01b Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Tue, 4 Aug 2026 16:24:35 +0200 Subject: [PATCH 09/14] Write the 2PCF as a SACC file (#305) Add RealSpaceMixin.save_2pcf_sacc, which serialises the TreeCorr output of calculate_2pcf_version into the sacc_io layout: the source_{i} n(z) tracers and one xi+/xi- block per tomographic bin pair. Serialising lives on the class because it needs the version's n(z) and its tomographic bin map; run_2pcf.py only decides where the file goes, from the sacc_path param under Snakemake and from --sacc on the CLI. Insertion order is load-bearing: add_xi writes one pair as [xi+; xi-], so the data vector is pair-major and the covariance is tied to it by position alone. The bin pairs are sorted once and that list drives every subsequent loop. Catalog bin ids are mapped to 0-based SACC indices from the ids actually present, rather than by subtracting 1, so a catalog that starts elsewhere or skips a bin still lands on the right source_{i} tracer. A mismatch between the n(z) column count and the measured bin count raises: the columns are never summed or truncated to make up the difference. Co-Authored-By: Claude Opus 5 (1M context) --- src/sp_validation/cosmo_val/real_space.py | 139 +++++++++++++++++++++- workflow/rules/twopoint.smk | 4 +- workflow/scripts/run_2pcf.py | 75 +++++++++--- 3 files changed, 197 insertions(+), 21 deletions(-) diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 96b7a077b..94786dc64 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -7,12 +7,33 @@ """ import os +import re import matplotlib.pyplot as plt import numpy as np import treecorr from astropy.io import fits +from .. import sacc_io + +# calculate_2pcf_version keys its results "tomo_bin_{b1}_tomo_bin_{b2}"; this +# reads the bin pair back out ("all" for the non-tomographic single pair). +_PAIR_KEY = re.compile(r"tomo_bin_(.+?)_tomo_bin_(.+)") + + +def _sacc_bin_index(ggs): + """Map each catalog tomographic bin id to its 0-based SACC bin index. + + The catalog numbers its bins however the ``tomo_bin_col`` column does + (1-based in practice, and the single id ``"all"`` for a non-tomographic + run), while SACC tracers are ``source_{i}`` counted from zero. The map is + built from the ids actually present, in ascending order, so it never + assumes the catalog starts at 1 or numbers its bins without gaps. + """ + ids = {b for key in ggs for b in _PAIR_KEY.fullmatch(key).groups()} + order = sorted(ids, key=lambda b: 0 if b == "all" else int(b)) + return {b: i for i, b in enumerate(order)} + class RealSpaceMixin: def calculate_2pcf_version( @@ -130,8 +151,8 @@ def calculate_2pcf_version( def calculate_2pcf( self, - npatch=None, compute_tomography=False, + npatch=None, **treecorr_config, ): """ @@ -159,15 +180,125 @@ def calculate_2pcf( for ver in self.versions: self.cat_ggs[ver] = self.calculate_2pcf_version( ver, - npatch=npatch, compute_tomography=compute_tomography, + npatch=npatch, **treecorr_config, ) - # LG TO-DO: No longer writing out text file, change to sacc_io method - return self.cat_ggs + def save_2pcf_sacc( + self, + ver, + path, + ggs=None, + *, + type, + grid="reporting", + metadata=None, + ): + """Write the measured ξ± for ``ver`` to ``path`` as a SACC file. + + Serialises the TreeCorr output of :meth:`calculate_2pcf_version` (or of + :meth:`calculate_2pcf`, via ``self.cat_ggs``) into the standard layout + of :mod:`sp_validation.sacc_io`: the ``source_{i}`` n(z) tracers, one + ξ+/ξ− block per tomographic bin pair, and the covariance. + + Insertion order is load-bearing. ``sacc_io.add_xi`` writes one bin pair + as ``[ξ+; ξ−]``, so the data vector is *pair-major*, and the covariance + is tied to it by position alone. Both the ξ insertion and the + covariance therefore iterate the same ``pairs`` list, sorted by SACC + bin index — never recomputed independently. + + The covariance follows what the measurement can support, read off the + correlations themselves (``var_method``) rather than off ``npatch``, + which the caller may have overridden per call: + + - jackknife (npatch > 1): ``treecorr.estimate_multi_cov`` over the + correlations in ``pairs`` order. TreeCorr concatenates each one as + ``[ξ+; ξ−]``, which is exactly the pair-major insertion order, so + the result is one contiguous block spanning every ξ point — + including the cross-pair covariance, which a per-pair ``gg.cov`` + would drop. + - shot noise (npatch == 1): the ``varxip``/``varxim`` diagonal, stored + as a SACC ``DiagonalCovariance``. + + Parameters + ---------- + ver : str + Catalog version, used for the n(z) lookup and stamped as metadata. + path : str + Output path (overwritten). Passed in by the caller rather than + derived here, so a Snakemake rule gets the exact filename it + declared as its output. + ggs : dict, optional + ``{"tomo_bin_{b1}_tomo_bin_{b2}": treecorr.GGCorrelation}`` as + returned by :meth:`calculate_2pcf_version`. Defaults to + ``self.cat_ggs[ver]``, i.e. the last :meth:`calculate_2pcf` run. + type : {'data', 'mock'} + Provenance of the input catalog, required by ``sacc_io.save``. + No default: only the caller knows whether it ran on a mock, and + ``sacc_io.load`` refuses unblinded ``type='data'`` files. + grid : str, optional + The ``grid`` tag on every point (default ``'reporting'``). Use + ``'integration'`` for the fine grid COSEBIs / pure-EB integrate + over, which shares this data type and tracer pair and is told + apart by nothing else. + metadata : dict, optional + Extra key/value pairs stored on the file, merged over the + version/npatch/blind stamped here. + + Returns + ------- + sacc.Sacc + The written data set. + """ + ggs = self.cat_ggs[ver] if ggs is None else ggs + if not ggs: + raise ValueError(f"{ver}: no ξ± measurements to write") + + index = _sacc_bin_index(ggs) + pairs = sorted( + ggs, key=lambda key: [index[b] for b in _PAIR_KEY.fullmatch(key).groups()] + ) + + z, *nz_cols = self.get_redshift(ver) + if len(nz_cols) != len(index): + raise ValueError( + f"{ver}: the n(z) file has {len(nz_cols)} distribution " + f"column(s) but the measurement covers {len(index)} " + f"tomographic bin(s) — every source bin needs its own n(z)" + ) + + s = sacc_io.new_sacc( + [(z, nz) for nz in nz_cols], + metadata={ + "version": ver, + "npatch": int(ggs[pairs[0]].npatch1), + **({"blind": self.blind} if self.blind is not None else {}), + **(metadata or {}), + }, + ) + + for key in pairs: + gg = ggs[key] + bins = tuple(index[b] for b in _PAIR_KEY.fullmatch(key).groups()) + sacc_io.add_xi( + s, + bins, + gg.meanr, + gg.xip, + gg.xim, + grid=grid, + theta_nom=gg.rnom, + npairs=gg.npairs, + weight=gg.weight, + ) + + sacc_io.save(s, path, type=type) + self.print_done(f"Wrote ξ± SACC for {ver} to {path}.") + return s + def calculate_aperture_mass_dispersion( self, theta_min=0.3, diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 22c09db2f..6dc71f001 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -11,11 +11,11 @@ rule xi: threads: 24 params: ver="{version}", + compute_tomography="{compute_tomography}", + npatch="{npatch}", min_sep="{min_sep}", max_sep="{max_sep}", nbins="{nbins}", - npatch="{npatch}", - fits=False, resources: mem_mb=30000, disk_mb=20000, diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 2e1ccabf7..0ba1a4521 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -10,67 +10,91 @@ --ver SP_v1.4.6.3_leak_corr \ --min-sep 1.0 --max-sep 250.0 --nbins 20 --npatch 1 \ --cat-config /path/to/cosmo_val/cat_config.yaml \ - --out + --out --type mock -The measurement itself is unchanged — ``CosmologyValidation.calculate_2pcf`` -does the TreeCorr work and writes the ``.txt`` dump plus ξ+/ξ- FITS files into -``output_dir``. ``output_dir`` is passed explicitly (rather than via the -``COSMO_VAL`` env hook) so lc can point each run at its own ``{output}`` tree. +``CosmologyValidation.calculate_2pcf`` does the TreeCorr work and +``save_2pcf_sacc`` writes the result as a SACC file. Both live on the class: +serialising needs the version's n(z) and its tomographic bin map, so this +script only decides *where* the file goes — from the ``sacc_path`` param under +Snakemake, from ``--sacc`` on the CLI (defaulting to the untagged +``xi_{ver}.sacc`` under ``--out``, since each lc recipe gets its own output +tree). Snakemake requires the file to appear at exactly the path the rule +declared as an output, which is why the path is passed in rather than derived +inside the class. ``output_dir`` is passed explicitly +(rather than via the ``COSMO_VAL`` env hook) so lc can point each run at its +own ``{output}`` tree. """ import argparse +import os from sp_validation.cosmo_val import CosmologyValidation def run_2pcf( ver, + compute_tomography, + npatch, min_sep, max_sep, nbins, - npatch, cat_config, output_dir, - save_fits=True, + sacc_path, + data_type="data", ): - """Measure ξ±(θ) for ``ver`` and write it under ``output_dir``. + """Measure ξ±(θ) for ``ver`` and write it to ``sacc_path``. Parameters mirror the TreeCorr reporting/integration grids: ``min_sep`` / ``max_sep`` in arcmin, ``nbins`` logarithmic bins, ``npatch`` spatial patches (1 for the paper fiducial). ``cat_config`` is an absolute path to the catalog configuration; ``output_dir`` overrides ``cat_config['paths']['output']`` so products land where lc expects. + + ``data_type`` is the ``'data'``/``'mock'`` provenance stamp SACC requires. + It defaults to ``'data'`` because that is the fail-safe direction: a mock + mislabelled as data is merely refused by ``sacc_io.load`` until it carries + a blinding stamp, whereas real data mislabelled as a mock would load + unblinded. Callers running on mocks pass ``'mock'`` explicitly. """ cv = CosmologyValidation( versions=[ver], catalog_config=cat_config, output_dir=output_dir, ) - return cv.calculate_2pcf( - ver=ver, + ggs = cv.calculate_2pcf( + compute_tomography=compute_tomography, npatch=npatch, - save_fits=save_fits, min_sep=min_sep, max_sep=max_sep, nbins=nbins, ) + # calculate_2pcf keys its result by version; save_2pcf_sacc takes the + # per-version {pair: GGCorrelation} dict. + cv.save_2pcf_sacc(ver, sacc_path, ggs[ver], type=data_type) def _from_snakemake(smk): p = smk.params run_2pcf( ver=p["ver"], + compute_tomography=bool(p["compute_tomography"]), + npatch=int(p["npatch"]), min_sep=float(p["min_sep"]), max_sep=float(p["max_sep"]), nbins=int(p["nbins"]), - npatch=int(p["npatch"]), # cat_config / output_dir were previously resolved via an os.chdir into # the cosmo_val dir + the COSMO_VAL env var; expose them as optional # params so the rule can pass them explicitly, falling back to the # class defaults (./cat_config.yaml, COSMO_VAL env) otherwise. cat_config=p.get("cat_config", "./cat_config.yaml"), output_dir=p.get("output_dir", None), - save_fits=True, + # Taken from params, so the rule must keep this in sync with the SACC + # path it declares as an output — Snakemake checks for that exact file + # once the job returns, and a mismatch fails the job as a missing + # output after the measurement has already run. + sacc_path=p.get("sacc_path", None), + data_type=p.get("data_type", "data"), ) @@ -83,6 +107,11 @@ def _from_cli(argv=None): required=True, help="Catalog version key in cat_config, e.g. SP_v1.4.6.3_leak_corr", ) + ap.add_argument( + "--compute-tomography", + action="store_true", + help="Compute tomographic 2PCF (default: non-tomographic)", + ) ap.add_argument( "--min-sep", type=float, required=True, help="Min separation [arcmin]" ) @@ -97,17 +126,33 @@ def _from_cli(argv=None): "--cat-config", required=True, help="Absolute path to cat_config.yaml" ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") - ap.add_argument("--no-fits", action="store_true", help="Skip ξ+/ξ- FITS export") + ap.add_argument( + "--sacc", + default=None, + help="SACC output path (default: xi_{ver}.sacc under --out)", + ) + ap.add_argument( + "--type", + choices=("data", "mock"), + default="data", + help="Catalog provenance stamped on the SACC file, default=%(default)s", + ) a = ap.parse_args(argv) + # Each lc/ASTRA recipe gets its own output directory, so the untagged + # native name is unambiguous there — same convention as + # generate_pseudo_cl.py, which leaves its native product under --out. + sacc_path = a.sacc or os.path.join(a.out, f"xi_{a.ver}.sacc") run_2pcf( ver=a.ver, + compute_tomography=a.compute_tomography, min_sep=a.min_sep, max_sep=a.max_sep, nbins=a.nbins, npatch=a.npatch, cat_config=a.cat_config, output_dir=a.out, - save_fits=not a.no_fits, + sacc_path=sacc_path, + data_type=a.type, ) From ba253612567ed4853b6aa1663fcf455bbc4aa151 Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Tue, 4 Aug 2026 18:22:57 +0200 Subject: [PATCH 10/14] added sacc_io to 2pcf calculation Co-Authored-By: Claude Opus 5 (1M context) --- src/sp_validation/cosmo_val/pure_eb.py | 2 + src/sp_validation/cosmo_val/real_space.py | 16 +++----- workflow/common.py | 18 ++++----- workflow/rules/cosmo_val.smk | 20 ++++++---- workflow/rules/twopoint.smk | 14 +++---- workflow/scripts/cv_cosebis.py | 10 +++-- workflow/scripts/cv_plot_2pcf.py | 1 + workflow/scripts/cv_pure_eb.py | 12 +++--- workflow/scripts/cv_ratio_xi_sys_xi.py | 1 + workflow/scripts/run_2pcf.py | 48 +++++++++-------------- 10 files changed, 69 insertions(+), 73 deletions(-) diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 095dd83e9..76247476d 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -164,6 +164,7 @@ def plot_pure_eb( max_sep_int=300, nbins_int=1000, npatch=None, + compute_tomography=False, # LG: Hardcoded to False for now var_method="jackknife", cov_path_int=None, cosmo_cov=None, @@ -280,6 +281,7 @@ def plot_pure_eb( min_sep_int=min_sep_int, max_sep_int=max_sep_int, nbins_int=nbins_int, + compute_tomography=False, # LG: Hardcoded to False for now npatch=npatch, var_method=var_method, cov_path_int=cov_path_int, diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 94786dc64..2ed032387 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -92,8 +92,6 @@ def calculate_2pcf_version( ggs = {f"tomo_bin_{b1}_tomo_bin_{b2}": None for b1, b2 in tomo_bin_pairs} - # LG TO-DO: Change to sacc_io method - patch_file = self._output_path(f"{ver}_patches_npatch={npatch}.dat") cat_gal = fits.getdata(self.cc[ver]["shear"]["path"]) @@ -190,14 +188,14 @@ def calculate_2pcf( def save_2pcf_sacc( self, ver, - path, + sacc_path, ggs=None, *, type, grid="reporting", metadata=None, ): - """Write the measured ξ± for ``ver`` to ``path`` as a SACC file. + """Write the measured ξ± for ``ver`` to ``sacc_path`` as a SACC file. Serialises the TreeCorr output of :meth:`calculate_2pcf_version` (or of :meth:`calculate_2pcf`, via ``self.cat_ggs``) into the standard layout @@ -227,10 +225,8 @@ def save_2pcf_sacc( ---------- ver : str Catalog version, used for the n(z) lookup and stamped as metadata. - path : str - Output path (overwritten). Passed in by the caller rather than - derived here, so a Snakemake rule gets the exact filename it - declared as its output. + sacc_path : str + Path to the SACC product. ggs : dict, optional ``{"tomo_bin_{b1}_tomo_bin_{b2}": treecorr.GGCorrelation}`` as returned by :meth:`calculate_2pcf_version`. Defaults to @@ -295,8 +291,8 @@ def save_2pcf_sacc( weight=gg.weight, ) - sacc_io.save(s, path, type=type) - self.print_done(f"Wrote ξ± SACC for {ver} to {path}.") + sacc_io.save(s, sacc_path, type=type) + self.print_done(f"Wrote ξ± SACC for {ver} to {sacc_path}.") return s def calculate_aperture_mass_dispersion( diff --git a/workflow/common.py b/workflow/common.py index 3df3413be..ea2b1cb1a 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -5,19 +5,13 @@ import re from pathlib import Path -# Output roots are env-overridable so a reproduction run can write into a -# fresh tree without clobbering (or silently reusing) prior products. -COSMO_VAL = Path( - os.environ.get( - "COSMO_VAL", "/n17data/cdaley/unions/code/sp_validation/cosmo_val/output" - ) -) +SP_VALIDATION = Path(__file__).resolve().parents[1] +COSMO_VAL = Path(os.environ.get("COSMO_VAL", SP_VALIDATION / "results/cosmo_val")) COSMO_INFERENCE = Path( - os.environ.get( - "COSMO_INFERENCE", "/n17data/cdaley/unions/code/sp_validation/cosmo_inference" - ) + os.environ.get("COSMO_INFERENCE", SP_VALIDATION / "cosmo_inference") ) -CAT_CONFIG = "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" +CAT_CONFIG = SP_VALIDATION / "cosmo_val/cat_config.yaml" + BLINDS = ["A", "B", "C"] BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")] @@ -34,12 +28,14 @@ "version": r"SP_v[\d.]+(_w_iv)?(_ecut\d+)?(_leak_corr)?", "blind": r"[ABC]", "nbins": r"\d+", + "npatch": r"\d+", "min_sep": r"[0-9.]+", "max_sep": r"[0-9.]+", "gaussian": r"(g|ng)", "block_pm": r"(\+\+|--|\+-)", "block_i": r"[123]", "mask_suffix": r"(_masked)?", + "tomo_suffix": r"(_tomo)?", "mock_id": r"\d{5}", "nside": r"\d+", } diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 704b4fb02..a5650135b 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -45,7 +45,7 @@ CV_BINNING = ( f"_nbins={CV['nbins']}_npatch={CV['npatch']}" ) - +# LG: DEPRACATED, xi_pm vectors no longer written to txt files. def cv_xi_txt(version): """Path to the 2pcf data vector calculate_2pcf writes for a version. @@ -238,8 +238,9 @@ rule cv_additive_bias: "../scripts/cv_additive_bias.py" +# LG: DEPRACATED since we do not plot unblinded data vectors. rule cv_plot_2pcf: - """n_pairs / xi± overlay across versions.""" + """Tomographic xi± across versions.""" input: xi=[cv_xi_txt(v) for v in CV_VERSIONS], output: @@ -251,11 +252,15 @@ rule cv_plot_2pcf: script: "../scripts/cv_plot_2pcf.py" - +# LG: DEPRACATED since we do not plot unblinded data vectors. rule cv_ratio_xi_sys_xi: """Ratio of PSF systematics (xi_psf_sys) to the cosmic-shear signal (xi+).""" input: - xi=[cv_xi_txt(v) for v in CV_VERSIONS], + # The xi txt this used to declare is written by nothing since + # calculate_2pcf stopped emitting it; cv_xi_txt is gone with it. The + # rule recomputes xi_psf_sys itself, so the entry is simply dropped — + # left in place it is a parse-time NameError that takes the whole + # workflow down, deprecated or not. rho=[cv_rho_stats(v) for v in CV_VERSIONS], tau=[cv_tau_stats(v) for v in CV_VERSIONS], output: @@ -294,8 +299,6 @@ rule cv_pseudo_cl: rule cv_pure_eb: """Pure E/B-mode decomposition for one version (config-space).""" - input: - xi=lambda w: cv_xi_txt(w.version), output: npz=cv_pure_eb_npz("{version}"), params: @@ -316,8 +319,9 @@ rule cv_pure_eb: rule cv_cosebis: """COSEBIs E/B decomposition for one version (config-space, fine binning).""" - input: - xi=lambda w: cv_xi_txt(w.version), + # No xi input, for the same reason as cv_pure_eb: calculate_cosebis runs + # its own TreeCorr over the fine integration binning, and the xi txt it + # used to declare is no longer written by anything. output: npz=cv_cosebis_npz("{version}"), params: diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 6dc71f001..72c10247c 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -1,21 +1,21 @@ # Two-point data-vector rules: xi, rho/tau, and pseudo-Cl products. - +# LG: currently calculates the unblinded 2PCF. Don't look! rule xi: - input: - catalog=get_shear_catalog, + """ξ±(θ) two-point correlation for one catalog version, as a SACC file.""" output: - str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + sacc=str(COSMO_VAL / "xi_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}{tomo_suffix}.sacc"), threads: 24 params: ver="{version}", - compute_tomography="{compute_tomography}", + compute_tomography=lambda w: w.tomo_suffix == "_tomo", npatch="{npatch}", min_sep="{min_sep}", max_sep="{max_sep}", nbins="{nbins}", + cat_config=CAT_CONFIG, + sacc_path=output.sacc, + data_type="data" resources: mem_mb=30000, disk_mb=20000, diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index 182cda997..3facd067f 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -2,9 +2,13 @@ Compute + plot rule (per version). plot_cosebis calls calculate_cosebis over a fine integration binning (the 2000-bin TreeCorr is the dominant cost) and -evaluates the configured scale cuts. Writes the {version}_eb_..._data.npz -COSEBIs data product (declared output) plus figures, and the per-version -COSEBIs PTE that cv_summarize_bmodes collects. +evaluates the configured scale cuts. The TreeCorr run is its own, so this rule +declares no xi input. Writes the {version}_eb_..._data.npz COSEBIs data product +(declared output) plus figures, and the per-version COSEBIs PTE that +cv_summarize_bmodes collects. + +Non-tomographic: plot_cosebis passes compute_tomography=False and then reads +the single "tomo_bin_all_tomo_bin_all" entry back out of the result. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs diff --git a/workflow/scripts/cv_plot_2pcf.py b/workflow/scripts/cv_plot_2pcf.py index 9d5c8900d..c004f38bc 100644 --- a/workflow/scripts/cv_plot_2pcf.py +++ b/workflow/scripts/cv_plot_2pcf.py @@ -9,6 +9,7 @@ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels from snakemake.script import snakemake +# LG : This is now DEPRECATED, NOT TO BE USED, since we are no longer plotting unblinded data vectors. _unbuffer_streams() cv = make_cv(snakemake) cv.plot_2pcf() diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index d15a763f1..6ca999845 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -1,11 +1,13 @@ """Rule cv_pure_eb: pure E/B-mode decomposition for one version. Compute + plot rule (per version). plot_pure_eb calls calculate_pure_eb, which -runs two TreeCorr correlations (reporting + integration binning); the reporting -binning reuses the cv_2pcf data vector via calculate_2pcf's skip-if-exists -path. Writes the {version}_eb_..._data.npz data product (declared output) plus -companion figures, and the per-version E/B PTEs that cv_summarize_bmodes -collects. +runs two TreeCorr correlations of its own (reporting + integration binning) — +it reads no data vector from disk, so this rule declares no xi input. Writes +the {version}_eb_..._data.npz data product (declared output) plus companion +figures, and the per-version E/B PTEs that cv_summarize_bmodes collects. + +Non-tomographic: plot_pure_eb does not forward compute_tomography to +calculate_pure_eb, so the measurement runs on the single "all" bin pair. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs diff --git a/workflow/scripts/cv_ratio_xi_sys_xi.py b/workflow/scripts/cv_ratio_xi_sys_xi.py index ba737d7ee..810d55781 100644 --- a/workflow/scripts/cv_ratio_xi_sys_xi.py +++ b/workflow/scripts/cv_ratio_xi_sys_xi.py @@ -10,6 +10,7 @@ from cv_runner import _unbuffer_streams, make_cv, verify_outputs from snakemake.script import snakemake +# LG : This is now DEPRECATED, NOT TO BE USED, since we are no longer plotting unblinded data vectors. _unbuffer_streams() cv = make_cv(snakemake) cv.plot_ratio_xi_sys_xi(offset=snakemake.params.get("offset", 0.1)) diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 0ba1a4521..586225383 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -15,18 +15,16 @@ ``CosmologyValidation.calculate_2pcf`` does the TreeCorr work and ``save_2pcf_sacc`` writes the result as a SACC file. Both live on the class: serialising needs the version's n(z) and its tomographic bin map, so this -script only decides *where* the file goes — from the ``sacc_path`` param under -Snakemake, from ``--sacc`` on the CLI (defaulting to the untagged -``xi_{ver}.sacc`` under ``--out``, since each lc recipe gets its own output -tree). Snakemake requires the file to appear at exactly the path the rule -declared as an output, which is why the path is passed in rather than derived -inside the class. ``output_dir`` is passed explicitly -(rather than via the ``COSMO_VAL`` env hook) so lc can point each run at its -own ``{output}`` tree. +script only names the file — from the ``sacc_name`` param under Snakemake, from +``--sacc`` on the CLI (defaulting to the untagged ``xi_{ver}.sacc``, since each +lc recipe gets its own output tree). The *directory* is not this script's to +choose: ``save_2pcf_sacc`` joins the name onto the version's output directory, +so the SACC lands beside every other product of the run. Under Snakemake that +directory comes from the catalog config; on the CLI ``--out`` overrides it, so +lc can point each run at its own ``{output}`` tree. """ import argparse -import os from sp_validation.cosmo_val import CosmologyValidation @@ -43,13 +41,16 @@ def run_2pcf( sacc_path, data_type="data", ): - """Measure ξ±(θ) for ``ver`` and write it to ``sacc_path``. + """Measure ξ±(θ) for ``ver`` and write it as the SACC file ``sacc_name``. Parameters mirror the TreeCorr reporting/integration grids: ``min_sep`` / ``max_sep`` in arcmin, ``nbins`` logarithmic bins, ``npatch`` spatial patches (1 for the paper fiducial). ``cat_config`` is an absolute path to the catalog configuration; ``output_dir`` overrides - ``cat_config['paths']['output']`` so products land where lc expects. + ``cat_config['paths']['output']`` so products land where lc expects, and + is left ``None`` under Snakemake so the directory comes from the catalog + config. ``sacc_name`` is a bare filename — ``save_2pcf_sacc`` joins it onto + whichever of the two that resolves to. ``data_type`` is the ``'data'``/``'mock'`` provenance stamp SACC requires. It defaults to ``'data'`` because that is the fail-safe direction: a mock @@ -78,22 +79,14 @@ def _from_snakemake(smk): p = smk.params run_2pcf( ver=p["ver"], - compute_tomography=bool(p["compute_tomography"]), + compute_tomography=str(p["compute_tomography"]).lower() == "true", npatch=int(p["npatch"]), min_sep=float(p["min_sep"]), max_sep=float(p["max_sep"]), nbins=int(p["nbins"]), - # cat_config / output_dir were previously resolved via an os.chdir into - # the cosmo_val dir + the COSMO_VAL env var; expose them as optional - # params so the rule can pass them explicitly, falling back to the - # class defaults (./cat_config.yaml, COSMO_VAL env) otherwise. cat_config=p.get("cat_config", "./cat_config.yaml"), - output_dir=p.get("output_dir", None), - # Taken from params, so the rule must keep this in sync with the SACC - # path it declares as an output — Snakemake checks for that exact file - # once the job returns, and a mismatch fails the job as a missing - # output after the measurement has already run. - sacc_path=p.get("sacc_path", None), + output_dir=None, + sacc_path=p["sacc_path"], data_type=p.get("data_type", "data"), ) @@ -128,8 +121,8 @@ def _from_cli(argv=None): ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( "--sacc", - default=None, - help="SACC output path (default: xi_{ver}.sacc under --out)", + required=True, + help="Path to SACC file, written under --out (default: xi_{ver}.sacc)", ) ap.add_argument( "--type", @@ -138,10 +131,7 @@ def _from_cli(argv=None): help="Catalog provenance stamped on the SACC file, default=%(default)s", ) a = ap.parse_args(argv) - # Each lc/ASTRA recipe gets its own output directory, so the untagged - # native name is unambiguous there — same convention as - # generate_pseudo_cl.py, which leaves its native product under --out. - sacc_path = a.sacc or os.path.join(a.out, f"xi_{a.ver}.sacc") + run_2pcf( ver=a.ver, compute_tomography=a.compute_tomography, @@ -151,7 +141,7 @@ def _from_cli(argv=None): npatch=a.npatch, cat_config=a.cat_config, output_dir=a.out, - sacc_path=sacc_path, + sacc_path=a.sacc, data_type=a.type, ) From d5ab40c651c38dd848875a95419510ddbfbc7a57 Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Thu, 6 Aug 2026 13:34:23 +0200 Subject: [PATCH 11/14] pass unit test Co-Authored-By: Claude Opus 5 (1M context) --- src/sp_validation/cosmo_val/pure_eb.py | 7 ++++++- src/sp_validation/tests/test_cosmo_val.py | 7 +++++++ src/sp_validation/tests/test_glass_mock.py | 22 +++++++++++++++++++++- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 76247476d..52a83bc8a 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -82,7 +82,12 @@ def calculate_pure_eb( Returns ------- dict - A dictionary containing the following keys: + Mapping of ``"tomo_bin_{b1}_tomo_bin_{b2}"`` to that bin pair's pure + E/B results, following the keys of :meth:`calculate_2pcf_version`. + With ``compute_tomography=False`` the single key is + ``"tomo_bin_all_tomo_bin_all"``. + + Each value is a dictionary containing the following keys: - "xip_E": Pure E-mode correlation function for xi+. - "xim_E": Pure E-mode correlation function for xi-. diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 9536cbeec..de7f62366 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -631,6 +631,13 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): nbins_int=600, ) + # calculate_pure_eb keys its results by tomographic bin pair, mirroring + # calculate_2pcf_version. Non-tomographic (compute_tomography=False, the + # default) means exactly one pair -- the all-galaxy auto-correlation -- + # so assert that and unwrap it to the per-bin mode dict. + assert list(results) == ["tomo_bin_all_tomo_bin_all"] + results = results["tomo_bin_all_tomo_bin_all"] + # Reference mode vectors from the seeded synthetic catalog + Schneider # transform. Deterministic (full-sample treecorr, no RNG); regenerate by # running calculate_pure_eb with the setup above and printing repr() of diff --git a/src/sp_validation/tests/test_glass_mock.py b/src/sp_validation/tests/test_glass_mock.py index a056318b7..56a264440 100644 --- a/src/sp_validation/tests/test_glass_mock.py +++ b/src/sp_validation/tests/test_glass_mock.py @@ -40,7 +40,23 @@ camb = pytest.importorskip("camb") -HAVE_GLASS = importlib.util.find_spec("glass") is not None +def _have(module): + """True if ``module`` is importable. False if it or a parent is missing.""" + try: + return importlib.util.find_spec(module) is not None + except ModuleNotFoundError: + # find_spec imports the parent package, which raises if it is absent. + return False + + +HAVE_GLASS = _have("glass") + +# The map path needs more than GLASS itself: build_shells and +# Cosmology_from_camb import ``cosmology.compat.camb``, which ships with the +# ``cosmology`` API package rather than with GLASS. Gate on it separately so a +# missing dependency reports as a skip instead of masquerading as the (real, +# but different) glass/cosmology API incompatibility tracked in the xfail below. +HAVE_COSMOLOGY_CAMB = _have("cosmology.compat.camb") REFERENCE = Path(__file__).parent / "data" / "glass_mock_camb_reference.npz" @@ -131,6 +147,10 @@ def test_config_change_breaks_reference(): @pytest.mark.skipif(not HAVE_GLASS, reason="GLASS not installed in this image") +@pytest.mark.skipif( + not HAVE_COSMOLOGY_CAMB, + reason="cosmology.compat.camb not installed in this image", +) @pytest.mark.xfail( reason=( "glass_mock map path is incompatible with the installed glass/cosmology " From d5dcd39a86b34d8b042f1d0c9c57a20d85773ddf Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Thu, 6 Aug 2026 17:25:00 +0200 Subject: [PATCH 12/14] update plotting scripts --- src/sp_validation/cosmo_val/pure_eb.py | 40 ++++++++++++++++---------- workflow/scripts/cv_pure_eb.py | 5 ++-- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 52a83bc8a..272f154e5 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -278,21 +278,31 @@ def plot_pure_eb( ) # Get or calculate results for this version - version_results = results_list[idx] or self.calculate_pure_eb( - version, - min_sep=min_sep, - max_sep=max_sep, - nbins=nbins, - min_sep_int=min_sep_int, - max_sep_int=max_sep_int, - nbins_int=nbins_int, - compute_tomography=False, # LG: Hardcoded to False for now - npatch=npatch, - var_method=var_method, - cov_path_int=cov_path_int, - cosmo_cov=cosmo_cov, - n_samples=n_samples, - ) + version_results = results_list[idx] + if version_results is None: + version_results_tomo = self.calculate_pure_eb( + version, + min_sep=min_sep, + max_sep=max_sep, + nbins=nbins, + min_sep_int=min_sep_int, + max_sep_int=max_sep_int, + nbins_int=nbins_int, + compute_tomography=False, # LG: Hardcoded to False for now + npatch=npatch, + var_method=var_method, + cov_path_int=cov_path_int, + cosmo_cov=cosmo_cov, + n_samples=n_samples, + ) + version_results = version_results_tomo[ + "tomo_bin_all_tomo_bin_all" + ] # LG: Extract non-tomographic results + elif ( + isinstance(version_results, dict) + and "tomo_bin_all_tomo_bin_all" in version_results + ): + version_results = version_results["tomo_bin_all_tomo_bin_all"] # Calculate E/B statistics for all bin combinations version_results = calculate_eb_statistics( diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index 6ca999845..ff34c1ea0 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -6,8 +6,9 @@ the {version}_eb_..._data.npz data product (declared output) plus companion figures, and the per-version E/B PTEs that cv_summarize_bmodes collects. -Non-tomographic: plot_pure_eb does not forward compute_tomography to -calculate_pure_eb, so the measurement runs on the single "all" bin pair. +Non-tomographic: plot_pure_eb forwards a hardcoded compute_tomography=False to +calculate_pure_eb and then reads the single "tomo_bin_all_tomo_bin_all" entry +back out of the result, so the measurement runs on the single "all" bin pair. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs From 63b3914cb8960df0a8b1a21464f2e553aee80464 Mon Sep 17 00:00:00 2001 From: LisaGoh Date: Tue, 20 Oct 2026 16:31:04 +0200 Subject: [PATCH 13/14] Write tomographic xi and the OneCovariance covariance to SACC (#305, #376) - sacc_writers.xi_pairs_to_sacc: one xi part over any set of bin pairs, pair-major; xi_to_sacc is its single-pair form. - sacc_writers.onecov_xi_covariance: reorder OneCovariance's type-major real-space matrix to a part's pair-major points; sacc_io.read_onecov_matrix reads the .dat. - calculate_2pcf_version seals every tomographic bin pair into one part (one source_i tracer and n(z) per bin, jackknife covariance across pairs), kept in xi_tomo_parts; blinded catalogues now conceal tomographic xi instead of refusing it. core.sacc_nz gains tomography=. - Workflow: run_2pcf --tomo, {tomo_suffix} on the xi rule, new xi_onecov rule and xi_onecov_sacc.py; assemble_sacc reorders --xi-cov the same way. - Drops the earlier save_2pcf_sacc, written against the old sacc_io API. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/sp_validation/cosmo_val/core.py | 33 ++- src/sp_validation/cosmo_val/real_space.py | 278 ++++++------------ src/sp_validation/cosmo_val/sacc_writers.py | 127 ++++++-- src/sp_validation/sacc_io.py | 26 ++ ...st_tomo_plot_cosebis_ignores_tomography.py | 5 + src/sp_validation/tests/test_cosmo_val.py | 101 ++++++- src/sp_validation/tests/test_sacc_io.py | 15 + src/sp_validation/tests/test_sacc_writers.py | 64 ++++ workflow/rules/twopoint.smk | 38 ++- workflow/scripts/assemble_sacc.py | 11 +- workflow/scripts/run_2pcf.py | 37 ++- workflow/scripts/xi_onecov_sacc.py | 74 +++++ 12 files changed, 555 insertions(+), 254 deletions(-) create mode 100644 workflow/scripts/xi_onecov_sacc.py diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 8159a1a03..f3cca8480 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -471,8 +471,10 @@ def ensure_version_exists(ver): # B-mode results storage for summarize_bmodes() self._pure_eb_results = {} self._cosebis_results = {} - # The sealed ξ± parts calculate_2pcf_version made, by (version, grid). + # The sealed ξ± parts calculate_2pcf_version made, by (version, grid): + # the ("all", "all") pair, and every tomographic bin pair. self.xi_parts = {} + self.xi_tomo_parts = {} def _output_path(self, *parts): """Absolute path under the catalog config's output directory. @@ -484,13 +486,18 @@ def _output_path(self, *parts): """ return os.path.abspath(os.path.join(self.cc["paths"]["output"], *parts)) - def sacc_nz(self, version): - """Single-bin ``nz`` mapping ``{0: (z, nz)}`` for the SACC writers. + def sacc_nz(self, version, tomography=False): + """``nz`` mapping ``{i: (z, nz_i)}`` for the SACC writers. - The SACC parts carry the ("all", "all") pair only, so the whole-survey - n(z) of :meth:`get_redshift` is bin 0. + Non-tomographic parts carry the ("all", "all") pair only, so the + whole-survey n(z) of :meth:`get_redshift` is bin 0. With + ``tomography``, bin ``i`` is column ``i + 1`` of the ``redshift_path`` + file (column 0 is z), one per tomographic bin. """ - return {0: tuple(self.get_redshift(version))} + if not tomography: + return {0: tuple(self.get_redshift(version))} + z, dndz = self.read_redshift_distribution(version, is_tomography=True) + return {i: (z, dndz[:, i]) for i in range(dndz.shape[1])} def sacc_metadata(self, version): """Provenance metadata stored on every SACC part for ``version``.""" @@ -601,17 +608,19 @@ def blind(self, version): def _refuse_blinded_tomography(self, version): """Refuse to measure tomographic shear signal on a blinded catalogue. - Only the ``("all", "all")`` pair is born as a SACC part, whose tracer - carries the n(z) the blind's theory shifts it by. The tomographic ξ±, - ⟨M_ap²⟩ and pseudo-Cℓ are TreeCorr objects and FITS tables with no - per-bin tracer, so on a blinded catalogue they would hold true values. + A statistic is blinded by sealing its SACC part, whose tracers carry + the n(z) the blind's theory shifts it by. ξ± (and ⟨M_ap²⟩, derived from + it) is born as such a part for every tomographic bin pair; the + tomographic pseudo-Cℓ are still FITS tables with no per-bin tracer, so + on a blinded catalogue they would hold true values. """ blind = self.blind(version) if blind.name != blinding.NONE.name: raise blinding.BlindingError( f"{version} is blinded under {blind.name}, and blinding covers " - "its non-tomographic pair only; tomographic shear statistics are " - "measured on public (`blind: none`) catalogues" + "its tomographic ξ± and non-tomographic statistics only; other " + "tomographic shear statistics are measured on public " + "(`blind: none`) catalogues" ) def basename( diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 8f34eb203..05a1c3437 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -6,7 +6,6 @@ """ import os -import re import matplotlib.pyplot as plt import numpy as np @@ -15,29 +14,7 @@ from sp_validation.statistics import jackknife_patch_centers from .. import sacc_io -from .sacc_writers import xi_to_sacc - -ALL_PAIR = "tomo_bin_all_tomo_bin_all" - -from .. import sacc_io - -# calculate_2pcf_version keys its results "tomo_bin_{b1}_tomo_bin_{b2}"; this -# reads the bin pair back out ("all" for the non-tomographic single pair). -_PAIR_KEY = re.compile(r"tomo_bin_(.+?)_tomo_bin_(.+)") - - -def _sacc_bin_index(ggs): - """Map each catalog tomographic bin id to its 0-based SACC bin index. - - The catalog numbers its bins however the ``tomo_bin_col`` column does - (1-based in practice, and the single id ``"all"`` for a non-tomographic - run), while SACC tracers are ``source_{i}`` counted from zero. The map is - built from the ids actually present, in ascending order, so it never - assumes the catalog starts at 1 or numbers its bins without gaps. - """ - ids = {b for key in ggs for b in _PAIR_KEY.fullmatch(key).groups()} - order = sorted(ids, key=lambda b: 0 if b == "all" else int(b)) - return {b: i for i, b in enumerate(order)} +from .sacc_writers import xi_pairs_to_sacc class RealSpaceMixin: @@ -53,23 +30,28 @@ def calculate_2pcf_version( ): """ξ± of one catalogue version on one binning, for each bin pair. - The non-tomographic ``("all", "all")`` pair is born as its SACC part, - sealed under the version's blind (:func:`sp_validation.sacc_io.seal`) - before it is kept in ``self.xi_parts[ver, grid]``, returned or written, - so on a blinded catalogue its true values never leave this method. A - part already held for ``(ver, grid)`` is reused when no ``npatch``, - binning or ``out`` is asked for, so the figure rules draw the parts they - are handed. Tomographic pairs are measured only on a public catalogue - (:meth:`_refuse_blinded_tomography`). Use :meth:`calculate_2pcf` to run - over every version in ``self.versions``. + The measured pairs are born as one SACC part — the ``("all", "all")`` + pair, or every tomographic bin pair with one ``source_i`` tracer and + n(z) per bin — sealed under the version's blind + (:func:`sp_validation.sacc_io.seal`) before it is kept in + ``self.xi_parts[ver, grid]`` (``self.xi_tomo_parts`` for tomography), + returned or written, so on a blinded catalogue its true values never + leave this method. A part already held for ``(ver, grid)`` is reused + when no ``npatch``, binning or ``out`` is asked for, so the figure rules + draw the parts they are handed. Use :meth:`calculate_2pcf` to run over + every version in ``self.versions``. + + Tomographic bin ids map to SACC bins in ascending order, so bin ``i`` + takes column ``i + 1`` of the ``redshift_path`` file, and pairs go in + as ``(0, 0), (0, 1), …, (n-1, n-1)`` — OneCovariance's bin-pair order. Parameters: ver (str): The catalogue version to measure. npatch (int, optional): Jackknife patches; the instance's ``npatch`` by default. With patches the part carries the - jackknife covariance, without them the shot-noise diagonal. - Seeded patch centres come from the full catalogue and are - shared by every bin pair. + jackknife covariance (across bin pairs too), without them the + shot-noise diagonal. Seeded patch centres come from the full + catalogue and are shared by every bin pair. compute_tomography (bool, optional): Measure the tomographic bin pairs instead of ``("all", "all")``. grid (str): The grid tag the part's points carry. @@ -78,38 +60,30 @@ def calculate_2pcf_version( e.g. ``min_sep=1``. Returns: - dict: ``"tomo_bin_{b1}_tomo_bin_{b2}"`` → ξ± shaped like a TreeCorr - ``GGCorrelation``: the part's - :func:`~sp_validation.sacc_io.xi_correlation` view for - ``("all", "all")``, the TreeCorr object for a tomographic pair. + dict: ``"tomo_bin_{b1}_tomo_bin_{b2}"`` → the part's + :func:`~sp_validation.sacc_io.xi_correlation` view of that pair, + shaped like a TreeCorr ``GGCorrelation``. """ reuse = npatch is None and out is None and not treecorr_config npatch = int(npatch or self.npatch) blind = self.blind(ver) + parts = self.xi_tomo_parts if compute_tomography else self.xi_parts if compute_tomography: - self._refuse_blinded_tomography(ver) tomo_bin_ids, tomo_bin_pairs = self._get_tomo_bins(ver) if tomo_bin_ids is None or tomo_bin_pairs is None: raise ValueError(f"Version {ver} does not have tomography information.") + index = {b: i for i, b in enumerate(sorted(tomo_bin_ids))} self.print_magenta( f"Computing tomographic ξ± for {ver} with {len(tomo_bin_pairs)} bins." ) else: self.print_magenta(f"Computing non-tomographic ξ± for {ver}.") tomo_bin_pairs = [("all", "all")] + index = {"all": 0} - ggs = {f"tomo_bin_{b1}_tomo_bin_{b2}": None for b1, b2 in tomo_bin_pairs} - held = self.xi_parts.get((ver, grid)) if reuse else None - if ALL_PAIR in ggs and held is not None: - ggs[ALL_PAIR] = sacc_io.xi_correlation(held) - to_compute = [ - (b1, b2) - for b1, b2 in tomo_bin_pairs - if ggs[f"tomo_bin_{b1}_tomo_bin_{b2}"] is None - ] - - if to_compute: + part = parts.get((ver, grid)) if reuse else None + if part is None: jackknife = npatch > 1 gg_config = { **self._binning(**treecorr_config), @@ -118,7 +92,8 @@ def calculate_2pcf_version( cols = self._shear_columns(ver, compute_tomography) patch_centers = self._patch_centers(cols, npatch) - for bin1, bin2 in to_compute: + measured = [] + for bin1, bin2 in tomo_bin_pairs: gg = treecorr.GGCorrelation(gg_config) cat_gal1 = self._bin_catalog(cols, bin1, npatch, patch_centers) @@ -129,31 +104,60 @@ def calculate_2pcf_version( ) gg.process(cat_gal1, cat2=cat_gal2) - - if (bin1, bin2) != ("all", "all"): - ggs[f"tomo_bin_{bin1}_tomo_bin_{bin2}"] = gg - continue - s = xi_to_sacc( - self.sacc_nz(ver), - {**self.sacc_metadata(ver), "npatch": npatch}, - gg.meanr, - gg.xip, - gg.xim, - grid=grid, - theta_nom=gg.rnom, - npairs=gg.npairs, - weight=gg.weight, - covariance=gg.cov if jackknife else None, - variances=( - None if jackknife else np.concatenate([gg.varxip, gg.varxim]) - ), + measured.append(((index[bin1], index[bin2]), gg)) + + nz = self.sacc_nz(ver, tomography=compute_tomography) + if len(nz) != len(index): + raise ValueError( + f"{ver}: the n(z) file has {len(nz)} distribution column(s) " + f"but the catalogue has {len(index)} tomographic bin(s); " + "every bin needs its own n(z)" ) - if out: - part = sacc_io.save(s, out, blind=blind) - else: - part = sacc_io.seal(s, blind) - self.xi_parts[ver, grid] = part - ggs[ALL_PAIR] = sacc_io.xi_correlation(part) + correlations = [gg for _, gg in measured] + covariance = variances = None + if not jackknife: + variances = np.concatenate( + [np.concatenate([gg.varxip, gg.varxim]) for gg in correlations] + ) + elif len(correlations) == 1: + covariance = correlations[0].cov + else: + # [ξ+; ξ−] per correlation, in list order: the part's pair-major + # order, with the cross-pair covariance a per-pair gg.cov drops. + covariance = treecorr.estimate_multi_cov(correlations, "jackknife") + s = xi_pairs_to_sacc( + nz, + {**self.sacc_metadata(ver), "npatch": npatch}, + [ + ( + bins, + { + "theta": gg.meanr, + "xip": gg.xip, + "xim": gg.xim, + "theta_nom": gg.rnom, + "npairs": gg.npairs, + "weight": gg.weight, + }, + ) + for bins, gg in measured + ], + grid=grid, + covariance=covariance, + variances=variances, + ) + if out: + part = sacc_io.save(s, out, blind=blind) + else: + part = sacc_io.seal(s, blind) + parts[ver, grid] = part + + ggs = { + f"tomo_bin_{b1}_tomo_bin_{b2}": sacc_io.xi_correlation( + part, bins=(index[b1], index[b2]), grid=grid + ) + for b1, b2 in tomo_bin_pairs + } self.print_done(f"Done 2PCF for {ver}.") @@ -212,8 +216,8 @@ def _bin_catalog(self, cols, tomo_bin_id, npatch, patch_centers=None): def calculate_2pcf( self, - compute_tomography=False, npatch=None, + compute_tomography=False, **treecorr_config, ): """ @@ -241,123 +245,13 @@ def calculate_2pcf( for ver in self.versions: self.cat_ggs[ver] = self.calculate_2pcf_version( ver, - compute_tomography=compute_tomography, npatch=npatch, + compute_tomography=compute_tomography, **treecorr_config, ) return self.cat_ggs - def save_2pcf_sacc( - self, - ver, - sacc_path, - ggs=None, - *, - type, - grid="reporting", - metadata=None, - ): - """Write the measured ξ± for ``ver`` to ``sacc_path`` as a SACC file. - - Serialises the TreeCorr output of :meth:`calculate_2pcf_version` (or of - :meth:`calculate_2pcf`, via ``self.cat_ggs``) into the standard layout - of :mod:`sp_validation.sacc_io`: the ``source_{i}`` n(z) tracers, one - ξ+/ξ− block per tomographic bin pair, and the covariance. - - Insertion order is load-bearing. ``sacc_io.add_xi`` writes one bin pair - as ``[ξ+; ξ−]``, so the data vector is *pair-major*, and the covariance - is tied to it by position alone. Both the ξ insertion and the - covariance therefore iterate the same ``pairs`` list, sorted by SACC - bin index — never recomputed independently. - - The covariance follows what the measurement can support, read off the - correlations themselves (``var_method``) rather than off ``npatch``, - which the caller may have overridden per call: - - - jackknife (npatch > 1): ``treecorr.estimate_multi_cov`` over the - correlations in ``pairs`` order. TreeCorr concatenates each one as - ``[ξ+; ξ−]``, which is exactly the pair-major insertion order, so - the result is one contiguous block spanning every ξ point — - including the cross-pair covariance, which a per-pair ``gg.cov`` - would drop. - - shot noise (npatch == 1): the ``varxip``/``varxim`` diagonal, stored - as a SACC ``DiagonalCovariance``. - - Parameters - ---------- - ver : str - Catalog version, used for the n(z) lookup and stamped as metadata. - sacc_path : str - Path to the SACC product. - ggs : dict, optional - ``{"tomo_bin_{b1}_tomo_bin_{b2}": treecorr.GGCorrelation}`` as - returned by :meth:`calculate_2pcf_version`. Defaults to - ``self.cat_ggs[ver]``, i.e. the last :meth:`calculate_2pcf` run. - type : {'data', 'mock'} - Provenance of the input catalog, required by ``sacc_io.save``. - No default: only the caller knows whether it ran on a mock, and - ``sacc_io.load`` refuses unblinded ``type='data'`` files. - grid : str, optional - The ``grid`` tag on every point (default ``'reporting'``). Use - ``'integration'`` for the fine grid COSEBIs / pure-EB integrate - over, which shares this data type and tracer pair and is told - apart by nothing else. - metadata : dict, optional - Extra key/value pairs stored on the file, merged over the - version/npatch/blind stamped here. - - Returns - ------- - sacc.Sacc - The written data set. - """ - ggs = self.cat_ggs[ver] if ggs is None else ggs - if not ggs: - raise ValueError(f"{ver}: no ξ± measurements to write") - - index = _sacc_bin_index(ggs) - pairs = sorted( - ggs, key=lambda key: [index[b] for b in _PAIR_KEY.fullmatch(key).groups()] - ) - - z, *nz_cols = self.get_redshift(ver) - if len(nz_cols) != len(index): - raise ValueError( - f"{ver}: the n(z) file has {len(nz_cols)} distribution " - f"column(s) but the measurement covers {len(index)} " - f"tomographic bin(s) — every source bin needs its own n(z)" - ) - - s = sacc_io.new_sacc( - [(z, nz) for nz in nz_cols], - metadata={ - "version": ver, - "npatch": int(ggs[pairs[0]].npatch1), - **({"blind": self.blind} if self.blind is not None else {}), - **(metadata or {}), - }, - ) - - for key in pairs: - gg = ggs[key] - bins = tuple(index[b] for b in _PAIR_KEY.fullmatch(key).groups()) - sacc_io.add_xi( - s, - bins, - gg.meanr, - gg.xip, - gg.xim, - grid=grid, - theta_nom=gg.rnom, - npairs=gg.npairs, - weight=gg.weight, - ) - - sacc_io.save(s, sacc_path, type=type) - self.print_done(f"Wrote ξ± SACC for {ver} to {sacc_path}.") - return s - def calculate_aperture_mass_dispersion( self, theta_min=0.3, @@ -371,9 +265,9 @@ def calculate_aperture_mass_dispersion( Both are linear in ξ± (TreeCorr's ``calculateMapSq`` sum, Schneider et al. 2002 filter), so they and their covariance T·C·Tᵀ come from the ξ± - of :meth:`calculate_2pcf_version` on a fine grid: for ``("all", - "all")`` its sealed part, concealed on a blinded catalogue, whose shift, - the same in every patch, leaves C unchanged. + of :meth:`calculate_2pcf_version` on a fine grid: its sealed part, for + every bin pair concealed on a blinded catalogue, whose shift, the same + in every patch, leaves C unchanged. """ theta_map = np.geomspace(theta_min * 5, theta_max / 2, nbins_map) self._map2 = {"theta_map": theta_map} diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index b10bfe814..f6c860449 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -7,8 +7,11 @@ own covariance as its one block. :func:`assemble_analysis_sacc` rebuilds the single ``{version}.sacc`` analysis file from these parts. -Everything here is single-bin today (``bins=(0, 0)``); the interface is -tomography-native so a future round supplies real bin pairs unchanged. +ξ± is written over any set of bin pairs (:func:`xi_pairs_to_sacc`), and +:func:`onecov_xi_covariance` puts OneCovariance's ξ± matrix in a part's +order. The other statistics are single-bin today (``bins=(0, 0)``); their +interface is tomography-native so a future round supplies real bin pairs +unchanged. """ import numpy as np @@ -27,6 +30,39 @@ BIN = (0, 0) +def xi_pairs_to_sacc(nz, metadata, pairs, *, grid, variances=None, covariance=None): + """One ξ± part over one or more bin pairs, on a named angular grid. + + ``pairs`` is a sequence of ``(bins, xi)``: ``bins`` a source bin pair + ``(i, j)`` and ``xi`` a mapping with ``theta``, ``xip``, ``xim`` and, + optionally, ``theta_nom``, ``npairs``, ``weight``. Each pair goes in as + ``[ξ+; ξ−]`` in the order given, so the data vector is pair-major, and the + covariance is tied to that order by position: ``covariance`` is the dense + block in it, ``variances`` the matching diagonal. At most one may be given. + ``nz`` must hold an n(z) for every bin the pairs name. + """ + s = sio.new_sacc(nz, metadata) + for bins, xi in pairs: + sio.add_xi( + s, + bins, + xi["theta"], + xi["xip"], + xi["xim"], + grid=grid, + theta_nom=xi.get("theta_nom"), + npairs=xi.get("npairs"), + weight=xi.get("weight"), + ) + if covariance is not None and variances is not None: + raise ValueError("give the ξ± writer a dense covariance or variances, not both") + if covariance is not None: + s.add_covariance(np.asarray(covariance)) + elif variances is not None: + sio.add_diagonal_covariance(s, np.asarray(variances)) + return s + + def xi_to_sacc( nz, metadata, @@ -43,29 +79,74 @@ def xi_to_sacc( ): """One ξ± part (``bins=(0, 0)``) on a named angular grid. - The grid's covariance comes in one of two shapes: ``covariance``, the dense - ``[ξ+; ξ−]``-ordered block (a jackknife estimate), or ``variances``, the - concatenated ``[varxip; varxim]`` diagonal. At most one may be given. + The single-pair form of :func:`xi_pairs_to_sacc`. The grid's covariance + comes in one of two shapes: ``covariance``, the dense ``[ξ+; ξ−]``-ordered + block (a jackknife estimate), or ``variances``, the concatenated + ``[varxip; varxim]`` diagonal. At most one may be given. """ - s = sio.new_sacc(nz, metadata) - sio.add_xi( - s, - BIN, - theta, - xip, - xim, - grid=grid, - theta_nom=theta_nom, - npairs=npairs, - weight=weight, + xi = { + "theta": theta, + "xip": xip, + "xim": xim, + "theta_nom": theta_nom, + "npairs": npairs, + "weight": weight, + } + return xi_pairs_to_sacc( + nz, metadata, [(BIN, xi)], grid=grid, variances=variances, covariance=covariance ) - if covariance is not None and variances is not None: - raise ValueError("give xi_to_sacc a dense covariance or variances, not both") - if covariance is not None: - s.add_covariance(np.asarray(covariance)) - elif variances is not None: - sio.add_diagonal_covariance(s, np.asarray(variances)) - return s + + +def onecov_xi_covariance(s, cov): + """OneCovariance's real-space ξ± matrix ``cov`` in the insertion order of ``s``. + + OneCovariance writes ξ± type-major: every ξ+ point, then every ξ− point; + within each, bin pairs ``(i, j)`` with ``i <= j`` and ``i`` outermost, then + θ ascending. A ξ± part is pair-major (:func:`xi_pairs_to_sacc`), so this + finds each point of ``s`` in OneCovariance's order and permutes ``cov`` + to match. The two orders agree for a single bin pair. OneCovariance's bin + ``k`` is column ``k`` of the n(z) file, which must be the part's + ``source_k``. + + ``s`` must hold ξ± on one grid only, with the same θ points for every bin + pair, as a part does. + """ + cov = np.asarray(cov, float) + types = {sio.XI_PLUS: 0, sio.XI_MINUS: 1} + if any(dp.data_type not in types for dp in s.data): + raise ValueError("onecov_xi_covariance takes a part holding ξ± only") + grids = {dp.tags.get("grid") for dp in s.data} + if len(grids) != 1: + raise ValueError(f"the part's ξ± lies on several grids {sorted(grids)}") + if cov.shape != (len(s.data), len(s.data)): + raise ValueError( + f"OneCovariance matrix is {cov.shape}, but the part has " + f"{len(s.data)} ξ± points; check the binning and tomography match" + ) + + def bin_index(tracer): + return int(tracer.removeprefix("source_")) + + pairs = sorted({tuple(sorted(map(bin_index, dp.tracers))) for dp in s.data}) + thetas = {} + for dp in s.data: + key = (dp.data_type, tuple(sorted(map(bin_index, dp.tracers)))) + thetas.setdefault(key, []).append(dp.tags["theta"]) + ntheta = {len(t) for t in thetas.values()} + if len(ntheta) != 1 or len(thetas) != 2 * len(pairs): + raise ValueError("every bin pair must carry ξ+ and ξ− on the same θ points") + (ntheta,) = ntheta + + order = np.empty(len(s.data), dtype=int) + for k, dp in enumerate(s.data): + pair = tuple(sorted(map(bin_index, dp.tracers))) + theta_rank = sorted(thetas[dp.data_type, pair]).index(dp.tags["theta"]) + order[k] = ( + types[dp.data_type] * len(pairs) * ntheta + + pairs.index(pair) * ntheta + + theta_rank + ) + return cov[np.ix_(order, order)] def pseudo_cl_to_sacc(nz, metadata, ell_eff, cl_all, wsp, covariance=None, nside=None): diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 2c2e7919b..a2f3541c8 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -1524,6 +1524,32 @@ def read_nz(path): return table[:, 0], table[:, 1:] +def read_onecov_matrix(path): + """Read a OneCovariance ``style = matrix`` covariance (the workflow's ``.dat``). + + OneCovariance writes the dense matrix with ``np.savetxt`` under a ``#`` + header; for real-space ξ± its rows are type-major (all ξ+, then all ξ−), + which :func:`sp_validation.cosmo_val.sacc_writers.onecov_xi_covariance` + reorders to a part's insertion order. + + Parameters + ---------- + path : str or pathlib.Path + The matrix file. + + Returns + ------- + numpy.ndarray + The square, symmetric covariance matrix. + """ + cov = np.atleast_2d(np.loadtxt(str(path))) + if cov.shape[0] != cov.shape[1]: + raise ValueError(f"{path}: covariance is not square, shape {cov.shape}") + if not np.allclose(cov, cov.T, rtol=1e-6, atol=0): + raise ValueError(f"{path}: covariance is not symmetric") + return cov + + def covariance_blocks(cov_list, selectors, *, gaussian=True): """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. diff --git a/src/sp_validation/tests/regression/test_tomo_plot_cosebis_ignores_tomography.py b/src/sp_validation/tests/regression/test_tomo_plot_cosebis_ignores_tomography.py index 645a0bb61..fcf1bea2e 100644 --- a/src/sp_validation/tests/regression/test_tomo_plot_cosebis_ignores_tomography.py +++ b/src/sp_validation/tests/regression/test_tomo_plot_cosebis_ignores_tomography.py @@ -45,12 +45,17 @@ def _make_cv(tmp_path): fits.BinTableHDU(t).writeto(cat) out = tmp_path / "out" out.mkdir() + # One n(z) column per bin: the tomographic ξ± part gives each bin a tracer. + z = np.linspace(0.05, 2.0, 20) + nz_path = tmp_path / "nz_tomo.txt" + np.savetxt(nz_path, np.column_stack([z, np.exp(-((z - 0.5) ** 2)), z * 0 + 1])) cfg = { VER: { "blind": "none", "subdir": str(tmp_path), "shear": { "path": str(cat), + "redshift_path": str(nz_path), "e1_col": "e1", "e2_col": "e2", "w_col": "w", diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 25cf0865b..1351abb70 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -592,6 +592,7 @@ def test_cross_tomographic_pair_shares_patch_centres(self, tmp_path, monkeypatch params, version = self._write_synthetic_catalogs( tmp_path, n_gal=400, with_tomography=True ) + self._use_tomographic_nz(params, version, tmp_path) cv = CosmologyValidation(versions=[version], npatch=4, **params) cols = cv._shear_columns(version, compute_tomography=True) full_catalog = treecorr.Catalog( @@ -629,9 +630,96 @@ def recording_bin_catalog(cols, bin_id, npatch, patch_centers=None): cross_pair_catalogs[1][2], expected_centers, rtol=0, atol=1e-14 ) - def test_a_blinded_catalogue_refuses_tomographic_signal(self, tmp_path): - """Blinding shifts the ("all", "all") parts only, so a blinded - catalogue's tomographic ξ± and pseudo-Cℓ are refused before measuring.""" + @staticmethod + def _use_tomographic_nz(params, version, tmp_path, nbin=2): + """Point ``version`` at an n(z) file with one column per tomographic bin.""" + z = np.linspace(0.05, 3.0, 31) + columns = [np.exp(-(((z - 0.4 - 0.5 * i) / 0.3) ** 2)) for i in range(nbin)] + nz_path = tmp_path / "nz_tomo.txt" + np.savetxt(nz_path, np.column_stack([z, *columns])) + config = yaml.safe_load(open(params["catalog_config"])) + config[version]["shear"]["redshift_path"] = str(nz_path) + with open(params["catalog_config"], "w") as f: + yaml.dump(config, f, sort_keys=False) + return z, columns + + def test_tomographic_xi_is_one_part_with_a_tracer_per_bin(self, tmp_path): + """Every bin pair goes into one sealed part: source_i carries column i + 1 + of the n(z) file, pairs are pair-major in OneCovariance's order, and the + jackknife covariance spans the pairs.""" + params, version = self._write_synthetic_catalogs( + tmp_path, n_gal=600, with_tomography=True + ) + z, columns = self._use_tomographic_nz(params, version, tmp_path) + cv = CosmologyValidation(versions=[version], npatch=4, **params) + + ggs = cv.calculate_2pcf_version(version, compute_tomography=True) + + part = cv.xi_tomo_parts[version, "reporting"] + assert sacc_io.stamp(part) == "none" + assert sorted(ggs) == [ + "tomo_bin_1_tomo_bin_1", + "tomo_bin_1_tomo_bin_2", + "tomo_bin_2_tomo_bin_2", + ] + for i, column in enumerate(columns): + z_read, nz_read = sacc_io.get_nz(part, i) + np.testing.assert_allclose(z_read, z) + np.testing.assert_allclose(nz_read, column) + nbins = cv.treecorr_config["nbins"] + pairs = [dp.tracers for dp in part.data[:: 2 * nbins]] + assert pairs == [ + ("source_0", "source_0"), + ("source_0", "source_1"), + ("source_1", "source_1"), + ] + np.testing.assert_allclose( + ggs["tomo_bin_1_tomo_bin_2"].xip, + sacc_io.get_xi(part, (0, 1), grid="reporting")[1], + ) + cov = part.covariance.dense + assert cov.shape == (6 * nbins, 6 * nbins) + assert np.any(cov[: 2 * nbins, 2 * nbins : 4 * nbins] != 0) + + def test_tomographic_xi_needs_an_nz_per_bin(self, tmp_path): + """A single-column n(z) cannot describe two bins; the part is refused.""" + params, version = self._write_synthetic_catalogs( + tmp_path, n_gal=400, with_tomography=True + ) + cv = CosmologyValidation(versions=[version], npatch=1, **params) + with pytest.raises(ValueError, match="n\\(z\\)"): + cv.calculate_2pcf_version(version, compute_tomography=True) + + def test_a_blinded_catalogue_conceals_tomographic_xi(self, tmp_path): + """On a blinded catalogue every tomographic pair is sealed under the + blind, so the ξ± handed back differs from the public measurement.""" + pytest.importorskip("smokescreen") + from sp_validation import blinding + + params, version = self._write_synthetic_catalogs( + tmp_path, n_gal=400, with_tomography=True + ) + self._use_tomographic_nz(params, version, tmp_path) + public = CosmologyValidation( + versions=[version], npatch=1, **params + ).calculate_2pcf_version(version, compute_tomography=True) + + config = yaml.safe_load(open(params["catalog_config"])) + config["paths"]["blinds"] = str(tmp_path / "blinds") + config[version]["blind"] = "toy" + with open(params["catalog_config"], "w") as f: + yaml.dump(config, f, sort_keys=False) + blinding.init("toy", config) + cv = CosmologyValidation(versions=[version], npatch=1, **params) + blinded = cv.calculate_2pcf_version(version, compute_tomography=True) + + assert sacc_io.stamp(cv.xi_tomo_parts[version, "reporting"]) == "toy" + for key in public: + assert not np.allclose(blinded[key].xip, public[key].xip) + + def test_a_blinded_catalogue_refuses_tomographic_pseudo_cl(self, tmp_path): + """Blinding covers the tomographic ξ± parts but not the pseudo-Cℓ FITS + tables, so a blinded catalogue's tomographic pseudo-Cℓ are refused.""" from sp_validation import blinding params, version = self._write_synthetic_catalogs( @@ -645,8 +733,6 @@ def test_a_blinded_catalogue_refuses_tomographic_signal(self, tmp_path): blinding.init("toy", config) cv = CosmologyValidation(versions=[version], npatch=1, **params) - with pytest.raises(blinding.BlindingError, match="tomographic"): - cv.calculate_2pcf_version(version, compute_tomography=True) with pytest.raises(blinding.BlindingError, match="tomographic"): cv.calculate_pseudo_cl(compute_tomography=True) @@ -885,6 +971,11 @@ def test_sacc_nz_is_the_summed_tomographic_nz(self, tmp_path): np.testing.assert_allclose(z_read, z) np.testing.assert_allclose(nz_read, nz_bins.sum(axis=0)) + tomographic = cv.sacc_nz(version, tomography=True) + assert sorted(tomographic) == [0, 1, 2] + for i, nz_bin in enumerate(nz_bins): + np.testing.assert_allclose(tomographic[i][1], nz_bin) + @pytest.mark.parametrize("pol_factor", [True, False, 0, 2]) def test_pol_factor_must_be_plus_or_minus_one(self, tmp_path, pol_factor): """pol_factor is a ±1 e2 multiplier; a bool would pass `in (-1, 1)`.""" diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index d22799691..096dc3ceb 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -1074,3 +1074,18 @@ def test_merge_partial_files(): assert cl_bb is not None and cl_eb is None n, E, B = sio.get_cosebis(merged, (0, 0)) assert B is None + + +def test_read_onecov_matrix_skips_the_header(tmp_path): + """OneCovariance's `style = matrix` file reads back as the dense matrix.""" + cov = np.array([[2.0, 0.5], [0.5, 3.0]]) + path = tmp_path / "cov.dat" + np.savetxt(path, cov, fmt="%.6e", header="OneCovariance covariance matrix") + np.testing.assert_allclose(sio.read_onecov_matrix(path), cov) + + +def test_read_onecov_matrix_refuses_a_non_square_file(tmp_path): + path = tmp_path / "cov.dat" + np.savetxt(path, np.ones((2, 3))) + with pytest.raises(ValueError, match="not square"): + sio.read_onecov_matrix(path) diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index a3eb66188..92460f1c7 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -400,3 +400,67 @@ def test_xi_part_rejects_two_covariances(): covariance=_spd(12, 42), variances=np.ones(12), ) + + +# --------------------------------------------------------------------------- # +# Tomographic ξ± and the OneCovariance covariance +# --------------------------------------------------------------------------- # +def _tomo_xi_part(nbin=2, ntheta=4): + """A pair-major ξ± part over every pair of ``nbin`` bins, values labelled.""" + theta = _theta(ntheta) + pairs = [(i, j) for i in range(nbin) for j in range(i, nbin)] + measured = [ + ( + bins, + { + "theta": theta, + "xip": 1e-5 * (10 * k + np.arange(ntheta)), + "xim": -1e-5 * (10 * k + np.arange(ntheta)), + }, + ) + for k, bins in enumerate(pairs) + ] + nz = {i: _nz(seed=i) for i in range(nbin)} + return sw.xi_pairs_to_sacc(nz, META, measured, grid="reporting"), pairs, theta + + +def test_xi_pairs_to_sacc_writes_every_pair_pair_major(tmp_path): + """Each pair goes in as [ξ+; ξ−], pairs in the order given, one tracer per bin.""" + s, pairs, theta = _tomo_xi_part(nbin=3) + s2 = _roundtrip(s, tmp_path, "xitomo") + assert {sio.source_name(i) for i in range(3)} <= set(s2.tracers) + assert len(s2.mean) == len(pairs) * 2 * len(theta) + for k, bins in enumerate(pairs): + th, p, m = sio.get_xi(s2, bins, grid="reporting") + np.testing.assert_array_equal(th, theta) + np.testing.assert_allclose(p, 1e-5 * (10 * k + np.arange(len(theta)))) + block = slice(k * 2 * len(theta), (k + 1) * 2 * len(theta)) + np.testing.assert_allclose(s2.mean[block], np.concatenate([p, m])) + + +def test_onecov_xi_covariance_reorders_type_major_to_pair_major(): + """OneCovariance's [all ξ+; all ξ−] matrix lands on the part's pair-major points.""" + s, pairs, theta = _tomo_xi_part(nbin=2, ntheta=4) + part_order = [(t, b, n) for b in pairs for t in (0, 1) for n in range(len(theta))] + onecov_order = sorted(part_order) # type, then pair, then θ + expected = _spd(len(part_order), seed=7) + position = {label: k for k, label in enumerate(part_order)} + take = [position[label] for label in onecov_order] + onecov = expected[np.ix_(take, take)] + + np.testing.assert_array_equal(sw.onecov_xi_covariance(s, onecov), expected) + + +def test_onecov_xi_covariance_is_the_identity_for_one_bin(): + """For the single (0, 0) pair OneCovariance's order already is the part's.""" + theta = _theta() + s = sw.xi_to_sacc({0: _nz()}, META, theta, theta * 0, theta * 0, grid="reporting") + cov = _spd(2 * len(theta), seed=3) + np.testing.assert_array_equal(sw.onecov_xi_covariance(s, cov), cov) + + +def test_onecov_xi_covariance_refuses_a_mismatched_matrix(): + """A matrix from another binning or tomography is refused, not truncated.""" + s, _, _ = _tomo_xi_part(nbin=2, ntheta=4) + with pytest.raises(ValueError, match="binning and tomography"): + sw.onecov_xi_covariance(s, np.eye(len(s.mean) - 2)) diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 893e22e8b..9f627b2d2 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -17,17 +17,17 @@ rule xi: """TreeCorr ξ±(θ) for one version on one angular grid, as its sealed SACC part. One rule for every grid: outputs are named by their binning, so a request - binds the wildcards and the grid label resolves from them. + binds the wildcards and the grid label resolves from them. A `_tomo` + suffix measures every tomographic bin pair into the one part. """ input: catalog=get_shear_catalog, output: - sacc=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), + sacc=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}{tomo_suffix}.sacc"), threads: 24 params: ver="{version}", compute_tomography=lambda w: w.tomo_suffix == "_tomo", - npatch="{npatch}", min_sep="{min_sep}", max_sep="{max_sep}", nbins="{nbins}", @@ -46,6 +46,38 @@ rule xi: "../scripts/run_2pcf.py" +def xi_onecov_covariance(w): + """The OneCovariance matrix on a ξ± request's binning and tomography: + Gaussian on the integration grid, non-Gaussian otherwise, as cosmo_val's + covariance inputs are.""" + gaussian = "g" if grid_of(XI_GRIDS, w) == "integration" else "ng" + return covariance_path( + w.version, gaussian, w.min_sep, w.max_sep, w.nbins, + DEFAULT_MASK_SUFFIX, w.tomo_suffix, + ) + + +rule xi_onecov: + """A ξ± part with the OneCovariance matrix of its binning as its covariance. + + The matrix replaces the estimate the part was born with (jackknife or shot + noise), reordered to the part's points; the result is a derivation of the + part, so it keeps the part's blind. + """ + input: + xi=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}{tomo_suffix}.sacc"), + cov=xi_onecov_covariance, + output: + sacc=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}{tomo_suffix}_onecov.sacc"), + params: + blind=lambda w: blind_of(w.version), + resources: + mem_mb=4000, + runtime=10, + script: + "../scripts/xi_onecov_sacc.py" + + rule rho_tau_stats: output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_tomo_bin_all_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 2cff0a6c2..566431cbf 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -19,7 +19,10 @@ from sp_validation import sacc_io from sp_validation.blinding import Blind -from sp_validation.cosmo_val.sacc_writers import assemble_analysis_sacc +from sp_validation.cosmo_val.sacc_writers import ( + assemble_analysis_sacc, + onecov_xi_covariance, +) # NaMaster iNKA covariance FITS: per-spectrum HDU names, in SACC insertion order. _CL_HDUS = ("COVAR_EE_EE", "COVAR_BB_BB", "COVAR_EB_EB") @@ -74,7 +77,7 @@ def _attach_cov(part, name, xi_cov, pseudo_cl_cov): f"--{_INJECTED[name].replace('_', '-')}, which was not supplied" ) block = ( - np.loadtxt(supplied) + onecov_xi_covariance(part, sacc_io.read_onecov_matrix(supplied)) if name == "xi_reporting" else _pseudo_cl_cov_block(supplied) ) @@ -178,7 +181,9 @@ def _from_cli(argv=None): ap.add_argument( f"--{name.replace('_', '-')}", default=None, help=f"{name} part" ) - ap.add_argument("--xi-cov", default=None, help="OneCovariance ξ± covariance matrix (.dat)") + ap.add_argument( + "--xi-cov", default=None, help="OneCovariance ξ± covariance matrix (.dat)" + ) ap.add_argument( "--pseudo-cl-cov", default=None, help="NaMaster pseudo-Cℓ covariance FITS" ) diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 15e1f9f1b..9625c092b 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -10,12 +10,14 @@ --ver SP_v1.4.6.3_leak_corr \ --min-sep 1.0 --max-sep 250.0 --nbins 20 --npatch 1 \ --cat-config /path/to/cosmo_val/cat_config.yaml \ - --out --type mock + --out [--tomo] The measurement is binning-agnostic: the reporting and the fine integration grids are the same compute with different ``--min-sep/--max-sep/--nbins``. The -non-tomographic ``("all", "all")`` ξ± is born as a SACC part, named by its -binning, tagged with its ``--grid`` and sealed under the catalogue's blind by +ξ± — the non-tomographic ``("all", "all")`` pair, or with ``--tomo`` every +tomographic bin pair with one n(z) tracer per bin — is born as a SACC part, +named by its binning (``_tomo`` appended for tomography), tagged with its +``--grid`` and sealed under the catalogue's blind by ``CosmologyValidation.calculate_2pcf_version``. The part carries the covariance the measurement estimated: the dense jackknife covariance when it had patches, the shot-noise ``varxip``/``varxim`` diagonal when it had none. @@ -32,15 +34,15 @@ def run_2pcf( ver, - compute_tomography, - npatch, min_sep, max_sep, nbins, + npatch, cat_config, output_dir, sacc_out=None, grid="reporting", + compute_tomography=False, ): """Measure ξ±(θ) for ``ver`` and write its sealed SACC part. @@ -51,6 +53,8 @@ def run_2pcf( ``cat_config['paths']['output']``. ``sacc_out`` is the exact destination for the part (the Snakemake-declared output); it defaults to a binning-derived name under the resolved output directory for the CLI path. + ``compute_tomography`` measures every tomographic bin pair instead of + ``("all", "all")``. Returns ------- @@ -64,18 +68,21 @@ def run_2pcf( ) out_path = sacc_out or os.path.join( output_dir or cv.cc["paths"]["output"], - f"{ver}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc", + f"{ver}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}" + f"{'_tomo' if compute_tomography else ''}.sacc", ) cv.calculate_2pcf_version( ver, npatch=npatch, + compute_tomography=compute_tomography, grid=grid, out=out_path, min_sep=min_sep, max_sep=max_sep, nbins=nbins, ) - part = cv.xi_parts[ver, grid] + parts = cv.xi_tomo_parts if compute_tomography else cv.xi_parts + part = parts[ver, grid] print(f"Wrote {grid} ξ± SACC part: {out_path}") return part @@ -84,8 +91,6 @@ def _from_snakemake(smk): p = smk.params run_2pcf( ver=p["ver"], - compute_tomography=str(p["compute_tomography"]).lower() == "true", - npatch=int(p["npatch"]), min_sep=float(p["min_sep"]), max_sep=float(p["max_sep"]), nbins=int(p["nbins"]), @@ -94,6 +99,7 @@ def _from_snakemake(smk): output_dir=p["output_dir"], grid=p.get("grid", "reporting"), sacc_out=smk.output["sacc"], + compute_tomography=bool(p.get("compute_tomography", False)), ) @@ -106,11 +112,6 @@ def _from_cli(argv=None): required=True, help="Catalog version key in cat_config, e.g. SP_v1.4.6.3_leak_corr", ) - ap.add_argument( - "--compute-tomography", - action="store_true", - help="Compute tomographic 2PCF (default: non-tomographic)", - ) ap.add_argument( "--min-sep", type=float, required=True, help="Min separation [arcmin]" ) @@ -128,11 +129,14 @@ def _from_cli(argv=None): ap.add_argument( "--grid", default="reporting", help="SACC grid tag for the measured points" ) + ap.add_argument( + "--tomo", + action="store_true", + help="Measure every tomographic bin pair instead of the whole sample", + ) a = ap.parse_args(argv) - run_2pcf( ver=a.ver, - compute_tomography=a.compute_tomography, min_sep=a.min_sep, max_sep=a.max_sep, nbins=a.nbins, @@ -140,6 +144,7 @@ def _from_cli(argv=None): cat_config=a.cat_config, output_dir=a.out, grid=a.grid, + compute_tomography=a.tomo, ) diff --git a/workflow/scripts/xi_onecov_sacc.py b/workflow/scripts/xi_onecov_sacc.py new file mode 100644 index 000000000..4ee75c569 --- /dev/null +++ b/workflow/scripts/xi_onecov_sacc.py @@ -0,0 +1,74 @@ +"""Give a ξ± SACC part the OneCovariance matrix of its binning as its covariance. + +Dual-mode. Under Snakemake (``script:`` directive) the injected ``snakemake`` +object supplies the paths; as a standalone CLI the same step runs from flags: + + python xi_onecov_sacc.py \ + --xi _xi_..._npatch=1_tomo.sacc \ + --cov /.dat \ + --blind none \ + --out _xi_..._npatch=1_tomo_onecov.sacc + +OneCovariance writes ξ± type-major; the part is pair-major, so the matrix is +reordered to the part's points +(:func:`sp_validation.cosmo_val.sacc_writers.onecov_xi_covariance`) before it +replaces the part's own estimate. The output is a derivation of the part and +carries its blind stamp, which must be the catalogue's declared blind. +""" + +import argparse + +from sp_validation import sacc_io +from sp_validation.blinding import Blind +from sp_validation.cosmo_val.sacc_writers import onecov_xi_covariance + + +def xi_onecov_sacc(xi_path, cov_path, out_path, *, blind): + """Write the part at ``xi_path`` with the matrix at ``cov_path`` to ``out_path``. + + Returns + ------- + sacc.Sacc + The data set as written. + """ + part = sacc_io.load(xi_path) + s = part.copy() + s.add_covariance( + onecov_xi_covariance(part, sacc_io.read_onecov_matrix(cov_path)), + overwrite=True, + ) + out = sacc_io.save(s, out_path, derived_from=[part], blind=Blind(blind)) + print(f"Wrote ξ± with the OneCovariance covariance {cov_path}: {out_path}") + return out + + +def _from_snakemake(smk): + xi_onecov_sacc( + smk.input["xi"], + smk.input["cov"], + smk.output["sacc"], + blind=smk.params["blind"], + ) + + +def _from_cli(argv=None): + ap = argparse.ArgumentParser( + description="Attach a OneCovariance ξ± matrix to a ξ± SACC part." + ) + ap.add_argument("--xi", required=True, help="ξ± SACC part") + ap.add_argument("--cov", required=True, help="OneCovariance matrix (.dat)") + ap.add_argument( + "--blind", required=True, help="The blind the catalogue is declared under" + ) + ap.add_argument("--out", required=True, help="Output SACC path") + a = ap.parse_args(argv) + xi_onecov_sacc(a.xi, a.cov, a.out, blind=a.blind) + + +if __name__ == "__main__": + try: + snakemake # noqa: F821 — injected by Snakemake's script: directive + except NameError: + _from_cli() + else: + _from_snakemake(snakemake) # noqa: F821 From bc5faba2643276840a6fde3915085833858b82c0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:55:45 +0000 Subject: [PATCH 14/14] ruff autofix (format + safe lint fixes) Pushed by the lint gate. --- src/sp_validation/tests/test_glass_mock.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sp_validation/tests/test_glass_mock.py b/src/sp_validation/tests/test_glass_mock.py index a3cbd0035..1ce1f5804 100644 --- a/src/sp_validation/tests/test_glass_mock.py +++ b/src/sp_validation/tests/test_glass_mock.py @@ -40,6 +40,7 @@ camb = pytest.importorskip("camb") + def _have(module): """True if ``module`` is importable. False if it or a parent is missing.""" try: