diff --git a/config/calibration/mask_v1.X.10.yaml b/config/calibration/mask_v1.X.10.yaml index 6a926216..703ea91b 100644 --- a/config/calibration/mask_v1.X.10.yaml +++ b/config/calibration/mask_v1.X.10.yaml @@ -42,17 +42,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.11.yaml b/config/calibration/mask_v1.X.11.yaml index 9562d524..88e01643 100644 --- a/config/calibration/mask_v1.X.11.yaml +++ b/config/calibration/mask_v1.X.11.yaml @@ -42,17 +42,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.2.yaml b/config/calibration/mask_v1.X.2.yaml index b50e2ed2..846f14ff 100644 --- a/config/calibration/mask_v1.X.2.yaml +++ b/config/calibration/mask_v1.X.2.yaml @@ -42,17 +42,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.3.yaml b/config/calibration/mask_v1.X.3.yaml index 9f8442ff..bcbbc620 100644 --- a/config/calibration/mask_v1.X.3.yaml +++ b/config/calibration/mask_v1.X.3.yaml @@ -42,17 +42,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.4.yaml b/config/calibration/mask_v1.X.4.yaml index 077a9f6d..903b9098 100644 --- a/config/calibration/mask_v1.X.4.yaml +++ b/config/calibration/mask_v1.X.4.yaml @@ -42,17 +42,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.5.yaml b/config/calibration/mask_v1.X.5.yaml index e2d9e871..5f56b135 100644 --- a/config/calibration/mask_v1.X.5.yaml +++ b/config/calibration/mask_v1.X.5.yaml @@ -42,17 +42,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.6.yaml b/config/calibration/mask_v1.X.6.yaml index e01d3d3f..1b5f7da6 100644 --- a/config/calibration/mask_v1.X.6.yaml +++ b/config/calibration/mask_v1.X.6.yaml @@ -42,17 +42,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.7.yaml b/config/calibration/mask_v1.X.7.yaml index bbe52699..cbc802be 100644 --- a/config/calibration/mask_v1.X.7.yaml +++ b/config/calibration/mask_v1.X.7.yaml @@ -42,17 +42,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.8.yaml b/config/calibration/mask_v1.X.8.yaml index 2071f651..58234d64 100644 --- a/config/calibration/mask_v1.X.8.yaml +++ b/config/calibration/mask_v1.X.8.yaml @@ -42,17 +42,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.9.yaml b/config/calibration/mask_v1.X.9.yaml index 95234bcd..4a448ce7 100644 --- a/config/calibration/mask_v1.X.9.yaml +++ b/config/calibration/mask_v1.X.9.yaml @@ -42,17 +42,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/cosmo_inference/notebooks/cfis_analysis.ipynb b/cosmo_inference/notebooks/cfis_analysis.ipynb index 74816206..ee93f4ec 100644 --- a/cosmo_inference/notebooks/cfis_analysis.ipynb +++ b/cosmo_inference/notebooks/cfis_analysis.ipynb @@ -776,8 +776,8 @@ "cat_psf = treecorr.Catalog(\n", " ra=df_psf[\"RA\"],\n", " dec=df_psf[\"DEC\"],\n", - " g1=df_psf[\"E1_PSF_HSM\"],\n", - " g2=df_psf[\"E2_PSF_HSM\"],\n", + " g1=df_psf[\"HSM_G1_PSF\"],\n", + " g2=df_psf[\"HSM_G2_PSF\"],\n", " ra_units=\"degrees\",\n", " dec_units=\"degrees\",\n", " npatch=50,\n", diff --git a/cosmo_inference/scripts/masking.py b/cosmo_inference/scripts/masking.py index 08a04feb..ba2f3c4c 100644 --- a/cosmo_inference/scripts/masking.py +++ b/cosmo_inference/scripts/masking.py @@ -118,9 +118,9 @@ def apply_masks(data, data_ext, mask_config, footprint_only=False): if not footprint_only: rel_size = np.divide( data["NGMIX_T_NOSHEAR"], - data["NGMIX_Tpsf_NOSHEAR"], + data["NGMIX_T_PSF_RECONV_NOSHEAR"], out=np.zeros_like(data["NGMIX_T_NOSHEAR"]), - where=(data["NGMIX_Tpsf_NOSHEAR"] > 0), + where=(data["NGMIX_T_PSF_RECONV_NOSHEAR"] > 0), ) rel_min = mask_config["metacal"]["gal_rel_size_min"] diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index 0b949372..cc1326df 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -12,7 +12,6 @@ DES: sigma_e: 0.29 psf: PSF_size: piff_T - square_size: false star_size: obs_T hdu: 1 path: psf_y3a1-v29.fits @@ -52,32 +51,31 @@ SP_axel_v0.0: n_psf: 0.47 sigma_e: 0.3 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: star_cat.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: shapepipe_1500_goldshape_v1.fits w_col: w e1_col: g1 - e1_PSF_col: E1_PSF_HSM + e1_PSF_col: HSM_G1_PSF e2_col: g2 - e2_PSF_col: E2_PSF_HSM + e2_PSF_col: HSM_G2_PSF star: ra_col: RA dec_col: DEC - e1_col: E1_STAR_HSM - e2_col: E2_STAR_HSM + e1_col: HSM_G1_STAR + e2_col: HSM_G2_STAR path: star_cat.fits SP_v0.1.1: subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 @@ -87,18 +85,18 @@ SP_v0.1.1: ls: dashdot marker: v psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: star_cat.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: unions_shapepipe_extended_2022_v0.0_recon_theli_8192.fits @@ -109,8 +107,8 @@ SP_v0.1.1: star: ra_col: RA dec_col: DEC - e1_col: E1_STAR_HSM - e2_col: E2_STAR_HSM + e1_col: HSM_G1_STAR + e2_col: HSM_G2_STAR path: star_cat.fits SP_v1.3: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe @@ -120,19 +118,18 @@ SP_v1.3: ls: solid marker: p psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_star_2022_v1.3.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR label: SP_LFmask_psf shear: R: 1.0 @@ -162,19 +159,18 @@ SP_v1.3.6: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2022_v1.3.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.3.6/unions_shapepipe_cut_struc_2022_v1.3.6.fits @@ -206,19 +202,18 @@ SP_v1.4.5: sigma_e: 0.39740419232655594 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n17data/sguerrini/unions_shapepipe_cut_struc_2024_v1.4.5_rerun.fits @@ -249,19 +244,18 @@ SP_v1.4.5.A: n_psf: 0.752316232272063 sigma_e: 0.30961528707207325 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: shapepipe_SPv1.fits @@ -290,19 +284,18 @@ SP_v1.4.5_bright: n_psf: 0.752316232272063 sigma_e: 0.30961528707207325 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits @@ -331,19 +324,18 @@ SP_v1.4.5_faint: n_psf: 0.752316232272063 sigma_e: 0.30961528707207325 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits @@ -373,19 +365,18 @@ SP_v1.4.6_glass_mock: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n09data/guerrini/glass_mock_v1.4.6/results/unions_glass_sim_00001_4096.fits @@ -415,19 +406,18 @@ SP_v1.4.5_intermediate: n_psf: 0.752316232272063 sigma_e: 0.30961528707207325 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits @@ -456,19 +446,18 @@ SP_v1.4.6: n_psf: 0.752316232272063 sigma_e: 0.379587601488189 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits @@ -501,19 +490,18 @@ SP_v1.4.6.3: n_psf: 0.752316232272063 sigma_e: 0.3783979556382064 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n17data/UNIONS/WL/v1.4.x/v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits @@ -545,19 +533,18 @@ SP_v1.3.6: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2022_v1.3.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.3.6/unions_shapepipe_cut_struc_2022_v1.3.6.fits @@ -589,19 +576,18 @@ SP_v1.4.6: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 0.92 path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits @@ -633,19 +619,18 @@ SP_v1.4.6.3_B: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits @@ -677,19 +662,18 @@ SP_v1.4.6.3_C: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits @@ -721,19 +705,18 @@ SP_v1.4.6.3_A: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits @@ -765,19 +748,18 @@ SP_v1.4.6_ecut07: sigma_e: 0.34187904441900674 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n17data/cdaley/unions/pure_eb/results/ecut/SP_v1.4.6_ecut07.fits @@ -809,19 +791,18 @@ SP_v1.4.6.1: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.6.1/unions_shapepipe_cut_struc_2024_v1.4.6.1.fits @@ -854,19 +835,18 @@ SP_v1.4.7: sigma_e: 0.3797419081703805 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits @@ -900,19 +880,18 @@ SP_v1.4.8: sigma_e: 0.3797397825805285 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_starhalo_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits @@ -944,19 +923,18 @@ SP_v1.4.11.2: sigma_e: 0.380499698062307 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n17data/UNIONS/WL/v1.4.x/v1.4.11.2/unions_shapepipe_cut_struc_2024_v1.4.11.2.fits @@ -988,19 +966,18 @@ SP_v1.4.11.3: sigma_e: 0.380499698062307 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n17data/UNIONS/WL/v1.4.x/v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits @@ -1032,19 +1009,18 @@ SP_v1.4.11.3_ecut07: sigma_e: 0.34058513153783426 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n17data/cdaley/unions/pure_eb/results/ecut/SP_v1.4.11.3_ecut07.fits @@ -1165,19 +1141,18 @@ SP_v1.4_LFmask_8k: n_psf: 0.5434016250405327 sigma_e: 0.31509572849714534 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR label: SP_LFmask_psf shear: R: 1.0 @@ -1206,19 +1181,18 @@ SP_v1.4_LFmask_8k_noalpha: n_psf: 0.5434016250405327 sigma_e: 0.31509572849714534 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR label: SP_LFmask_psf shear: R: 1.0 @@ -1255,19 +1229,18 @@ SP_v1.6.6: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.6.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.6.6/unions_shapepipe_cut_struc_2024_v1.6.6.fits diff --git a/cosmo_val/compute_theory_cov.py b/cosmo_val/compute_theory_cov.py index b65fadb2..2868355f 100644 --- a/cosmo_val/compute_theory_cov.py +++ b/cosmo_val/compute_theory_cov.py @@ -18,7 +18,6 @@ def get_params_rho_tau(cat, survey="other"): params["e2_star_col"] = cat["psf"]["e2_star_col"] params["PSF_size"] = cat["psf"]["PSF_size"] params["star_size"] = cat["psf"]["star_size"] - params["square_size"] = cat["psf"]["square_size"] params["R11"] = np.array([1]) params["R22"] = np.array([1]) if survey != "DES": diff --git a/docs/ngmix_psf_column_migration.md b/docs/ngmix_psf_column_migration.md new file mode 100644 index 00000000..b24e9f83 --- /dev/null +++ b/docs/ngmix_psf_column_migration.md @@ -0,0 +1,200 @@ +# ShapePipe-v2 column-grammar migration (shapepipe → sp_validation) + +**Status:** code-complete on `migrate/ngmix-psf-column-names` (draft PR +[#201](https://github.com/CosmoStat/sp_validation/pull/201)). Every shape-column +read in the live package, configs, calibration scripts, paper figures, and +notebooks uses the ShapePipe-v2 grammar; the σ→T units change and the +`spread_model` removal are in; the dead `galsim` estimator path is removed; and +the suite is green against synthetic catalogues carrying the new columns. + +**No code work is left waiting on a regenerated catalogue.** The one *value* +change — `*_PSF_ORIG` now holds a true original-PSF fit (shapepipe#749) rather +than the reconvolved-kernel alias the old columns silently held — is a straight +column rename at the code level and is already in place; the code does not care +that the numbers moved. All a v2 catalogue enables is a *look-at-the-numbers* +sanity check (do the α-leakage / size-ratio cuts still behave), which is analysis, +not code, and does not gate the PR. **The real merge gate is cutover timing:** +merging this branch makes `develop` *require* v2 columns and stop reading today's +catalogues, so #201 should land together with — or just after — +[shapepipe#761](https://github.com/CosmoStat/shapepipe/pull/761)→#741 and the +first v2 catalogue. + +shapepipe#761 turns the shape-measurement output into **one column grammar for +the whole catalogue**: every estimator names its outputs +`ESTIMATOR_COMPONENT[_ERR][_OBJECT]_SHEAR` (uppercase), stores a single size +`T = 2σ²` sourced from `cs_util.size`, and splits ellipticity into named scalar +components. The galaxy is the implicit default object and carries **no token** +(`NGMIX_G1_NOSHEAR`, never `..._GAL_...`). This document is the authoritative +old→new map for the sp_validation consumer side. + +## What is *not* a value change (safe renames) + +Both shape estimators now store native **`g`** (reduced shear). HSM always stored +galsim's `observed_shape.g1/.g2` under the `E*` names — name and value disagreed; +#761 renamed the columns to match the value (`HSM_G1/G2_*`), moving **zero +numbers**. So there is **no e→g conversion anywhere in this migration** — the ρ/τ +leakage code (`shear_psf_leakage`) reads `g` straight into treecorr, which is what +it was always implicitly validated against. `cs_util.shape` exists for whichever +repo needs a genuine e↔g conversion; sp_validation does not. + +## Column map + +`{shear} ∈ {NOSHEAR, 1P, 1M, 2P, 2M}` (uppercase). Moments branch prefixes `NGMIXm_`. + +### ngmix — galaxy (implicit object, no token) + +| Old | New | Note | +|---|---|---| +| `NGMIX_ELL_{shear}` (2-vec) | `NGMIX_G1_{shear}`, `NGMIX_G2_{shear}` | split into named scalars | +| `NGMIX_ELL_ERR_{shear}` (2-vec) | `NGMIX_G1_ERR_{shear}`, `NGMIX_G2_ERR_{shear}` | split; value-safe | +| `NGMIX_T_{shear}` / `_T_ERR_` | unchanged | no `_GAL` token added | +| `NGMIX_FLUX_{shear}` / `_ERR` | unchanged | | +| `NGMIX_FLAGS_{shear}`, `NGMIX_SNR_{shear}` | unchanged | | +| `NGMIX_MCAL_FLAGS`, `NGMIX_N_EPOCH` | unchanged | OBJECT/SHEAR-less metadata | +| `NGMIX_MOM_FAIL` | `NGMIX_MCAL_TYPES_FAIL` | **renamed + semantics change** (see below) | + +`NGMIX_MOM_FAIL` → `NGMIX_MCAL_TYPES_FAIL` is more than a rename: in ngmix v1 the +column counted moments-initial-guess failures from `get_guess`, which no longer +exists in v2, so the producer reused the slot for a *failed-metacal-types* count. +sp_validation only ever cuts on it as `== 0` (keep objects with no failure), and +that cut stays correct — but the underlying failure mode changed, so if the +post-cut galaxy count looks off against a regenerated v2 catalogue, this is the +first line to check. The cut lives in `galaxy.classification_galaxy_ngmix`; the +column is also carried through `params.add_cols_pre_cal` and every +`config/calibration/mask_v1.X.*.yaml`. + +### ngmix — reconvolved PSF (metacal kernel; value-safe rename) + +| Old | New | Note | +|---|---|---| +| `NGMIX_Tpsf_{shear}` | `NGMIX_T_PSF_RECONV_{shear}` | same value (the `T/Tpsf` size-ratio cut) | + +### ngmix — original PSF (value change — shapepipe#749 fix — *not a code blocker*) + +`*_PSF_ORIG` now carries a *true* fit to the original image PSF, no longer the +reconvolved-kernel alias the old `ELL_PSFo`/`T_PSFo` columns silently held. The +rename is a straight column rename in sp_validation and is correct as-is — the +code does not care that the numbers moved. The only thing a regenerated catalogue +buys is a *look-at-the-numbers* check that the α-leakage / size-ratio cuts still +behave; that is analysis, not code, and it does not gate this PR (see Status). + +| Old | New | +|---|---| +| `NGMIX_ELL_PSFo_{shear}_0` (or `[:, 0]`) | `NGMIX_G1_PSF_ORIG_{shear}` | +| `NGMIX_ELL_PSFo_{shear}_1` (or `[:, 1]`) | `NGMIX_G2_PSF_ORIG_{shear}` | +| `NGMIX_T_PSFo_{shear}` | `NGMIX_T_PSF_ORIG_{shear}` | + +### HSM — star / PSF validation catalogue + +`SIGMA_*_HSM` → `HSM_T_*` is a **units change**: the new column stores the area +`T = 2σ²` directly (the producer applied `sigma_to_T`). Downstream must stop +squaring — the stored value is already `T`. + +| Old | New | Note | +|---|---|---| +| `E1_PSF_HSM` / `E2_PSF_HSM` | `HSM_G1_PSF` / `HSM_G2_PSF` | native `g`, pure rename | +| `E1_STAR_HSM` / `E2_STAR_HSM` | `HSM_G1_STAR` / `HSM_G2_STAR` | native `g`, pure rename | +| `SIGMA_PSF_HSM` | `HSM_T_PSF` | **units:** now `T = 2σ²`, an area | +| `SIGMA_STAR_HSM` | `HSM_T_STAR` | **units:** now `T = 2σ²`, an area | +| `FLAG_PSF_HSM` | `HSM_FLAG_PSF` | singular `FLAG` for HSM | +| `FLAG_STAR_HSM` | `HSM_FLAG_STAR` | singular `FLAG` for HSM | + +The `piff_T` / DES block in `cat_config.yaml` uses Piff columns, **not** this +grammar — left untouched. + +### `spread_model` — removed + +shapepipe no longer writes `SPREAD_MODEL` / `SPREADERR_MODEL` / `SPREAD_CLASS` +(Axel's review: these must never be cut on). The `do_spread_model` branch, +`classification_galaxy_base`'s parameter, and `catalog.match_spread_class` are +removed; star/galaxy classification uses the size-based path. + +## Size arithmetic routes through `cs_util.size` + +`T = 2σ²` is the only stored size. Anything needing σ / FWHM / r50 calls +`cs_util.size` (`T_to_fwhm`, `T_to_r50`, …) — never inline `SIGMA ** 2` or +`sqrt(T)`. The pre-migration `papers/catalog/2025_12_*` hand-rolled +`T = SIGMA_*_HSM ** 2` (a factor-of-2 error even then, since `SIGMA_*_HSM` never +truly held σ) now reads `HSM_T_*` directly. + +## `square_size` is retired + +The σ→T change makes the old squaring dead: `HSM_T_*` (and DES's `piff_T`) already +hold `T`, so nothing squares. The per-dataset `square_size:` flags in +`cat_config.yaml` are dropped, the `not_square_size` list in `rho_tau.py` is +removed, and the `square_size` key is gone from both param builders +(`rho_tau.get_params_rho_tau`, `cosmo_val/compute_theory_cov.py`). The parameter +is also removed from `shear_psf_leakage`'s `build_cat_to_compute_{rho,tau}` and +`CovTauTh` ([PR #27](https://github.com/CosmoStat/shear_psf_leakage/pull/27)), so +sp_validation no longer *passes* it — the two migrations are coordinated. (Passing +`square_size=False` into the post-#27 leakage handlers would raise `TypeError`; +dropping the argument here is what keeps the container green once #27 lands.) + +## Consumer sites in sp_validation + +- **Core** — `src/sp_validation/{galaxy.py, calibration.py, catalog.py}`. +- **Configs** — `cosmo_val/cat_config.yaml` (HSM blocks; DES/piff spared); + `config/calibration/mask_v1.X.*.yaml` ×10 (`NGMIX_ELL_PSFo_NOSHEAR_0/_1`). +- **rho/τ + covariance** — `src/sp_validation/rho_tau.py`, + `src/sp_validation/cosmo_val/psf_systematics.py`, `cosmo_val/compute_theory_cov.py`, + `src/sp_validation/glass_mock.py`. +- **Scripts** — `scripts/calibration/{extract_info,params,calibrate_comprehensive_cat}.py`, + `scripts/apply_alpha_snr_size_bin.py`, `scripts/examples/demo_*.py`. +- **Papers** — `papers/catalog/2025_12_*`, `papers/catalog/{hist_mag,2025_09_19_alpha_leakage_correction}.py`, + `papers/harmonic/2025_09_11_psf_leakage_cell.py`. +- **Notebooks** — `cosmo_inference/notebooks/cfis_analysis.ipynb` + (`E1/E2_PSF_HSM` → `HSM_G1/G2_PSF`); a sweep of all tracked notebooks found no + other old tokens. +- **Tests** — `src/sp_validation/tests/{test_calibration,test_cosmo_val}.py` + (synthetic catalogues + configs updated in lock-step; this is the migration's + internal-consistency check). + +## galsim estimator path — removed as dead code + +**ShapePipe cannot produce galsim shapes.** There is no galsim shape-measurement +runner in the pipeline — `ngmix_runner` measures shapes and writes `NGMIX_*`, but +no galsim analogue exists. The only trace of galsim-as-shapes is a dormant +serialization hook in `make_cat` (`SHAPE_MEASUREMENT_TYPE=galsim` reads shapes from +an optional 5th input catalogue that nothing upstream fills), and every production +config sets `SHAPE_MEASUREMENT_TYPE=ngmix` with galsim commented out. shapepipe#761 +did rename the `GALSIM_*` family onto the grammar (`GALSIM_GAL_ELL_*` / +`GALSIM_*_SIGMA_*` → scalar `GALSIM_G1/G2_*`, `GALSIM_T*`), but that is cheap +serialization symmetry, not a live capability. + +On the consumer side the galsim reader was **dead and already broken**: `shape` is +hardcoded to `"ngmix"`, `extract_info.py` raises for any other value, nothing +outside `scratch/` instantiates `metacal(prefix="GALSIM")` or calls +`classification_galaxy_galsim`, and the shared +`col_1p = f"{prefix}_T_PSF_RECONV_1P"` read in `metacal._read_data` never matched +the galsim producer output (`GALSIM_T_PSF_*`, not `..._T_PSF_RECONV_*`). Carrying +an untestable, already-broken path onto the new grammar is a worse end state than +deleting it, so this branch **removes** it: + +- `calibration.metacal._read_data_galsim`, the `prefix == "GALSIM"` dispatch + branch (now `else: raise` — unknown prefixes fail loudly), and the two galsim + ellipticity-sign flips in `_shear_response` / `_selection_response`; +- `galaxy.classification_galaxy_galsim`; +- the `sh == "galsim"` branch in `catalog.get_snr` (now `else: raise`); +- the unused `shape_method` argument on `get_calibrated_quantities` / + `get_calibrated_m_c`, and the `galsim` mentions in `params.py` / `extract_info.py`. + +ngmix is the sole estimator sp_validation supports; the `prefix` parameter stays +(it names the column family and could serve a future `NGMIXm` moments family). +To revive galsim shapes, the capability must return **end-to-end**: a galsim +shape-measurement runner in ShapePipe (so the columns are producible at all), then +a migrated-and-tested consumer here — restore commit `e0eaa9f`, rename onto the +grammar, and add coverage — not the dead stub that was removed. + +## Coordinated repos + +- **`shear_psf_leakage`** ([PR #27](https://github.com/CosmoStat/shear_psf_leakage/pull/27), + "Adopt ShapePipe-v2 HSM column grammar, retire square_size") is the sibling + consumer migration. It lands on the same HSM grammar + (`HSM_G1/G2_{PSF,STAR}`, `HSM_T_*`, `HSM_FLAG_*`) and removed the `square_size` + parameter from `build_cat_to_compute_{rho,tau}` and `CovTauTh`; this PR drops the + matching argument, so the two land together (see "`square_size` is retired"). +- **shapepipe#761** (producer) still renames the `GALSIM_*` family onto the grammar + for columns nothing can create. If the goal is to simplify the grammar, + retiring that serialization is a producer-side follow-up worth raising there. + +— Claude on behalf of Cail diff --git a/papers/catalog/2025_09_19_alpha_leakage_correction.py b/papers/catalog/2025_09_19_alpha_leakage_correction.py index de411ed8..135b1aab 100644 --- a/papers/catalog/2025_09_19_alpha_leakage_correction.py +++ b/papers/catalog/2025_09_19_alpha_leakage_correction.py @@ -33,7 +33,7 @@ cat_gal = fits.getdata(path_cat) # %% -psf_size = "NGMIX_Tpsf_NOSHEAR" +psf_size = "NGMIX_T_PSF_RECONV_NOSHEAR" gal_size = "NGMIX_T_NOSHEAR" size_ratio = cat_gal[psf_size] / (cat_gal[gal_size] + cat_gal[psf_size]) diff --git a/papers/catalog/2025_12_09_residual_star_plot.py b/papers/catalog/2025_12_09_residual_star_plot.py index 30085eef..79e2f521 100644 --- a/papers/catalog/2025_12_09_residual_star_plot.py +++ b/papers/catalog/2025_12_09_residual_star_plot.py @@ -27,12 +27,12 @@ cat_star = fits.getdata(path_star) # To be checked -e1_star = cat_star["E1_STAR_HSM"] -e2_star = cat_star["E2_STAR_HSM"] -T_star = cat_star["SIGMA_STAR_HSM"] ** 2 -e1_psf = cat_star["E1_PSF_HSM"] -e2_psf = cat_star["E2_PSF_HSM"] -T_psf = cat_star["SIGMA_PSF_HSM"] ** 2 +e1_star = cat_star["HSM_G1_STAR"] +e2_star = cat_star["HSM_G2_STAR"] +T_star = cat_star["HSM_T_STAR"] +e1_psf = cat_star["HSM_G1_PSF"] +e2_psf = cat_star["HSM_G2_PSF"] +T_psf = cat_star["HSM_T_PSF"] M_4_1_star = cat_star["M_4_STAR_1"] M_4_2_star = cat_star["M_4_STAR_2"] M_4_1_psf = cat_star["M_4_PSF_1"] diff --git a/papers/catalog/2025_12_10_plot_maps.py b/papers/catalog/2025_12_10_plot_maps.py index 2ad0be7d..ad8da0ba 100644 --- a/papers/catalog/2025_12_10_plot_maps.py +++ b/papers/catalog/2025_12_10_plot_maps.py @@ -599,9 +599,9 @@ def get_kappa_kaiser_squire(gamma_1, gamma_2): # %% # Maps of star/PSF information ra_star, dec_star = cat_star["RA"], cat_star["DEC"] -e1_star, e2_star = cat_star["E1_STAR_HSM"], cat_star["E2_STAR_HSM"] -e1_psf, e2_psf = cat_star["E1_PSF_HSM"], cat_star["E2_PSF_HSM"] -t_star, t_psf = cat_star["SIGMA_STAR_HSM"] ** 2, cat_star["SIGMA_PSF_HSM"] ** 2 +e1_star, e2_star = cat_star["HSM_G1_STAR"], cat_star["HSM_G2_STAR"] +e1_psf, e2_psf = cat_star["HSM_G1_PSF"], cat_star["HSM_G2_PSF"] +t_star, t_psf = cat_star["HSM_T_STAR"], cat_star["HSM_T_PSF"] weights_star = np.ones_like(ra_star) # %% diff --git a/papers/catalog/2025_12_11_plot_star_properties.py b/papers/catalog/2025_12_11_plot_star_properties.py index be992521..2aef02f4 100644 --- a/papers/catalog/2025_12_11_plot_star_properties.py +++ b/papers/catalog/2025_12_11_plot_star_properties.py @@ -34,9 +34,9 @@ # Load catalog cat_stars = fits.open(path_cat_star)[1].data -e1_star, e2_star = cat_stars["E1_STAR_HSM"], cat_stars["E2_STAR_HSM"] -e1_psf, e2_psf = cat_stars["E1_PSF_HSM"], cat_stars["E2_PSF_HSM"] -t_star, t_psf = cat_stars["SIGMA_STAR_HSM"] ** 2, cat_stars["SIGMA_PSF_HSM"] ** 2 +e1_star, e2_star = cat_stars["HSM_G1_STAR"], cat_stars["HSM_G2_STAR"] +e1_psf, e2_psf = cat_stars["HSM_G1_PSF"], cat_stars["HSM_G2_PSF"] +t_star, t_psf = cat_stars["HSM_T_STAR"], cat_stars["HSM_T_PSF"] mag = cat_stars["MAG"] # %% diff --git a/papers/catalog/2025_12_11_plot_tau_stats_dependence_mag.py b/papers/catalog/2025_12_11_plot_tau_stats_dependence_mag.py index 4b359e15..911a8449 100644 --- a/papers/catalog/2025_12_11_plot_tau_stats_dependence_mag.py +++ b/papers/catalog/2025_12_11_plot_tau_stats_dependence_mag.py @@ -65,8 +65,8 @@ tc_psf = treecorr.Catalog( ra=cat_star["RA"], dec=cat_star["DEC"], - g1=cat_star["E1_PSF_HSM"], - g2=cat_star["E2_PSF_HSM"], + g1=cat_star["HSM_G1_PSF"], + g2=cat_star["HSM_G2_PSF"], ra_units="degrees", dec_units="degrees", npatch=100, diff --git a/papers/catalog/hist_mag.py b/papers/catalog/hist_mag.py index 262c38cd..b0bc33ef 100644 --- a/papers/catalog/hist_mag.py +++ b/papers/catalog/hist_mag.py @@ -299,9 +299,9 @@ def plot_all_hists( [ "FLAGS", "IMAFLAGS_ISO", - "NGMIX_MOM_FAIL", - "NGMIX_ELL_PSFo_NOSHEAR_0", - "NGMIX_ELL_PSFo_NOSHEAR_1", + "NGMIX_MCAL_TYPES_FAIL", + "NGMIX_G1_PSF_ORIG_NOSHEAR", + "NGMIX_G2_PSF_ORIG_NOSHEAR", "4_Stars", "8_Manual", "1024_Maximask", @@ -315,9 +315,9 @@ def plot_all_hists( [ "IMAFLAGS_ISO", "FLAGS", - "NGMIX_MOM_FAIL", - "NGMIX_ELL_PSFo_NOSHEAR_0", - "NGMIX_ELL_PSFo_NOSHEAR_1", + "NGMIX_MCAL_TYPES_FAIL", + "NGMIX_G1_PSF_ORIG_NOSHEAR", + "NGMIX_G2_PSF_ORIG_NOSHEAR", "4_Stars", "8_Manual", "1024_Maximask", @@ -329,9 +329,9 @@ def plot_all_hists( combine_cols = { "ngmix failures": [ - "NGMIX_MOM_FAIL", - "NGMIX_ELL_PSFo_NOSHEAR_0", - "NGMIX_ELL_PSFo_NOSHEAR_1", + "NGMIX_MCAL_TYPES_FAIL", + "NGMIX_G1_PSF_ORIG_NOSHEAR", + "NGMIX_G2_PSF_ORIG_NOSHEAR", ] } @@ -415,7 +415,7 @@ def get_info_for_metacal_masking(dat, mask, prefix="NGMIX", name_shear="NOSHEAR" for key in ("flux", "flux_err", "T"): res[key] = dat[mask][f"{prefix}_{key.upper()}_{name_shear}"] - res["Tpsf"] = dat[mask][f"{prefix}_Tpsf_{name_shear}"] + res["Tpsf"] = dat[mask][f"{prefix}_T_PSF_RECONV_{name_shear}"] return res @@ -439,7 +439,6 @@ def get_info_for_metacal_masking(dat, mask, prefix="NGMIX", name_shear="NOSHEAR" size_corr_ell=cm["gal_size_corr_ell"], sigma_eps=cm["sigma_eps_prior"], global_R_weight=cm["global_R_weight"], - col_2d=False, verbose=True, ) diff --git a/papers/harmonic/2025_09_11_psf_leakage_cell.py b/papers/harmonic/2025_09_11_psf_leakage_cell.py index c388affb..45f13362 100644 --- a/papers/harmonic/2025_09_11_psf_leakage_cell.py +++ b/papers/harmonic/2025_09_11_psf_leakage_cell.py @@ -103,7 +103,7 @@ def get_shape_noise(e1, e2, w): f_psf = nmt.NmtFieldCatalog( positions=[cat_star["RA"], cat_star["DEC"]], weights=np.ones_like(cat_star["RA"]), - field=[cat_star["E1_PSF_HSM"], -cat_star["E2_PSF_HSM"]], + field=[cat_star["HSM_G1_PSF"], -cat_star["HSM_G2_PSF"]], lmax=b_lmax, lmax_mask=b_lmax, spin=2, @@ -235,7 +235,7 @@ def compute_iNKA_covariance(field1, field2, coupled=False): # %% -field_psf_map = get_field_catalog(cat_star, "E1_PSF_HSM", "E2_PSF_HSM", nside=1024) +field_psf_map = get_field_catalog(cat_star, "HSM_G1_PSF", "HSM_G2_PSF", nside=1024) field_gal_map = get_field_catalog(cat_gal, "e1", "e2", nside=1024, w_col="w_des") field_gal_corrected_map = get_field_catalog( cat_gal, "e1_leak_corrected", "e2_leak_corrected", nside=1024, w_col="w_des" diff --git a/scripts/apply_alpha_snr_size_bin.py b/scripts/apply_alpha_snr_size_bin.py index abb541bf..12261e31 100644 --- a/scripts/apply_alpha_snr_size_bin.py +++ b/scripts/apply_alpha_snr_size_bin.py @@ -40,7 +40,7 @@ def params_default(): "e2_PSF_col": "e2_PSF", "snr_col": "snr", "w_col": "w", - "TPSF_col": "NGMIX_Tpsf_NOSHEAR", + "TPSF_col": "NGMIX_T_PSF_RECONV_NOSHEAR", "T_col": "NGMIX_T_NOSHEAR", "input_path_shear": "SP/unions_shapepipe_extended_2022_v1.0.fits", "output_path": "shape_cat_cor_alpha.fits", diff --git a/scripts/calibration/calibrate_comprehensive_cat.py b/scripts/calibration/calibrate_comprehensive_cat.py index 38d67a53..4b15ed89 100644 --- a/scripts/calibration/calibrate_comprehensive_cat.py +++ b/scripts/calibration/calibrate_comprehensive_cat.py @@ -92,7 +92,6 @@ size_corr_ell=cm["gal_size_corr_ell"], sigma_eps=cm["sigma_eps_prior"], global_R_weight=cm["global_R_weight"], - col_2d=False, verbose=True, ) @@ -185,7 +184,7 @@ "FLUX_APER", "FLUXERR_APER", "NGMIX_T_NOSHEAR", - "NGMIX_Tpsf_NOSHEAR", + "NGMIX_T_PSF_RECONV_NOSHEAR", "fwhm_PSF", ] add_cols_data = {} @@ -197,8 +196,10 @@ "FHP/MK hack: explicit copying of the metacal no-shear (updated from 1p)" + " PSF size" ) -add_cols_data["NGMIX_Tpsf_NOSHEAR_orig"] = add_cols_data["NGMIX_Tpsf_NOSHEAR"] -add_cols_data["NGMIX_Tpsf_NOSHEAR"] = gal_metacal.ns["Tpsf"][mask_metacal] +add_cols_data["NGMIX_T_PSF_RECONV_NOSHEAR_orig"] = add_cols_data[ + "NGMIX_T_PSF_RECONV_NOSHEAR" +] +add_cols_data["NGMIX_T_PSF_RECONV_NOSHEAR"] = gal_metacal.ns["Tpsf"][mask_metacal] # %% # Additional post-processing columns to write to output cat diff --git a/scripts/calibration/extract_info.py b/scripts/calibration/extract_info.py index 3868080b..1cbf836f 100644 --- a/scripts/calibration/extract_info.py +++ b/scripts/calibration/extract_info.py @@ -79,14 +79,21 @@ # PSF keys key_base = shape.upper() -key_PSF_ell = f"{key_base}_ELL_PSFo_NOSHEAR" -key_PSF_size = f"{key_base}_T_PSFo_NOSHEAR" +key_PSF_g1 = f"{key_base}_G1_PSF_ORIG_NOSHEAR" +key_PSF_g2 = f"{key_base}_G2_PSF_ORIG_NOSHEAR" +key_PSF_size = f"{key_base}_T_PSF_ORIG_NOSHEAR" size_to_fwhm = T_to_fwhm print_stats("Galaxies:", stats_file, verbose=verbose) n_tot = spv_cat.print_some_quantities(dd, stats_file, verbose=verbose) spv_cat.print_mean_ellipticity( - dd, f"{key_base}_ELL_NOSHEAR", 2, n_tot, stats_file, invalid=-10, verbose=verbose + dd, + [f"{key_base}_G1_NOSHEAR", f"{key_base}_G2_NOSHEAR"], + 1, + n_tot, + stats_file, + invalid=-10, + verbose=verbose, ) # #### Survey area and potential missing tiles @@ -110,7 +117,7 @@ n_tot = spv_cat.print_some_quantities(d_star, stats_file, verbose=verbose) spv_cat.print_mean_ellipticity( d_star, - ["E1_PSF_HSM", "E2_PSF_HSM"], + ["HSM_G1_PSF", "HSM_G2_PSF"], 1, n_tot, stats_file, @@ -121,7 +128,7 @@ # ### 3. Matching of stars # ### Matching of star catalogues -# Match the star catalogue `d_star` (selected on individual exposures using size-magnitude diagram) to catalogue from tile. Uses some simple criteria to select stars from tile catalogue such as SPREAD_CLASS. +# Match the star catalogue `d_star` (selected on individual exposures using size-magnitude diagram) to catalogue from tile. Uses some simple criteria to select stars from tile catalogue. # # This is mainly for testing, this match will not be used later. @@ -148,7 +155,7 @@ (dd["FLAGS"][ind_star] == 0) & (dd["IMAFLAGS_ISO"][ind_star] == 0) & (dd["NGMIX_MCAL_FLAGS"][ind_star] == 0) - & (dd["NGMIX_ELL_PSFo_NOSHEAR"][:, 0][ind_star] != -10) + & (dd["NGMIX_G1_PSF_ORIG_NOSHEAR"][ind_star] != -10) ) ra_star, dec_star, g_star_psf = spv_cat.match_subsample( @@ -156,7 +163,8 @@ ind_star, m_star, [col_name_ra, col_name_dec], - key_PSF_ell, + key_PSF_g1, + key_PSF_g2, n_star_tot, stats_file, verbose=verbose, @@ -175,24 +183,11 @@ g_star_psf[1], ) -#### Refine: Match to SPREAD_CLASS samples -if "SPREAD_CLASS" in dd.dtype.names: - spv_cat.match_spread_class( - dd, ind_star, m_star, stats_file, len(ra_star), verbose=verbose - ) -else: - print_stats( - "No SPREAD_CLASS in input, skipping star-gal matching", - stats_file, - verbose=verbose, - ) - # ## Check for objects with invalid PSF spv_cat.check_invalid( dd, - [key_PSF_ell, f"{key_base}_ELL_NOSHEAR"], - [0, 0], + [key_PSF_g1, f"{key_base}_G1_NOSHEAR"], [-10, -10], stats_file, name=["`PSF", "galaxy ellipticity"], @@ -243,16 +238,16 @@ add_cols_pre_cal_format["overlap"] = "I" # Additional columns {e1, e2, size}_PSF -ext_cols_pre_cal["e1_PSF"] = dd[key_PSF_ell][:, 0] -ext_cols_pre_cal["e2_PSF"] = dd[key_PSF_ell][:, 1] +ext_cols_pre_cal["e1_PSF"] = dd[key_PSF_g1] +ext_cols_pre_cal["e2_PSF"] = dd[key_PSF_g2] ext_cols_pre_cal["fwhm_PSF"] = size_to_fwhm(dd[key_PSF_size]) -_, _, iv_w = metacal.get_variance_ivweights(dd, sigma_eps_prior, mask=None, col_2d=True) +_, _, iv_w = metacal.get_variance_ivweights(dd, sigma_eps_prior, mask=None) mag = spv_cat.get_col(dd, "MAG_AUTO", None, None) snr = spv_cat.get_snr(shape, dd, None, None) -g1_uncal = dd[f"{key_base}_ELL_NOSHEAR"][:, 0] -g2_uncal = dd[f"{key_base}_ELL_NOSHEAR"][:, 1] +g1_uncal = dd[f"{key_base}_G1_NOSHEAR"] +g2_uncal = dd[f"{key_base}_G2_NOSHEAR"] # Comprehensive catalogue without cuts nor mask applied if verbose: @@ -284,8 +279,7 @@ # ## 4. Select galaxies # #### Common flags and cuts -# First, set cuts common to ngmix and galsim: -# - spread model: select objects well larger than the PSF +# First, set cuts common to all objects: # - magnitude: cut galaxies that are too faint (= too noisy, likely to be # artefacts), and too bright (might be too large for postage stamp) # - flags: cut objects that were flagged as invalid or masked @@ -300,7 +294,6 @@ gal_mag_faint=gal_mag_faint, flags_keep=flags_keep, n_epoch_min=n_epoch_min, - do_spread_model=do_spread_model, ) if shape == "ngmix": m_gal = classification_galaxy_ngmix( @@ -608,7 +601,7 @@ labels = [] if shape == "ngmix": # Do not apply `mask_ns`, so use all galaxies - xs = [dd["NGMIX_T_NOSHEAR"][m_gal] / dd["NGMIX_Tpsf_NOSHEAR"][m_gal]] + xs = [dd["NGMIX_T_NOSHEAR"][m_gal] / dd["NGMIX_T_PSF_RECONV_NOSHEAR"][m_gal]] labels.append("size ratio") else: @@ -878,8 +871,7 @@ n_bin = 250 # + -key = key_PSF_ell -xs = [dd[key][:, 0][mask_ns_stars], dd[key][:, 1][mask_ns_stars]] +xs = [dd[key_PSF_g1][mask_ns_stars], dd[key_PSF_g2][mask_ns_stars]] title = "PSF" out_name = f"ell_PSF_{shape}.pdf" out_path = os.path.join(plot_dir, out_name) @@ -990,8 +982,8 @@ # + # Additional columns: # {e1, e2, size}_PSF -ext_cols["e1_PSF"] = dd[key_PSF_ell][:, 0][m_gal][mask] -ext_cols["e2_PSF"] = dd[key_PSF_ell][:, 1][m_gal][mask] +ext_cols["e1_PSF"] = dd[key_PSF_g1][m_gal][mask] +ext_cols["e2_PSF"] = dd[key_PSF_g2][m_gal][mask] ext_cols["fwhm_PSF"] = size_to_fwhm(dd[key_PSF_size][m_gal][mask]) if mask_external_path: ext_cols["mask_extern"] = m_extern diff --git a/scripts/calibration/params.py b/scripts/calibration/params.py index 363b36f7..2d0bd45f 100644 --- a/scripts/calibration/params.py +++ b/scripts/calibration/params.py @@ -35,9 +35,8 @@ ## Pixel size in arcsec pixel_size = 0.187 -## Shape measurement method, implemented is -## 'ngix': multi-epoch model fitting -## 'galsim': stacked-image moments (experimental) +## Shape measurement method (only 'ngmix' is supported): +## 'ngmix': multi-epoch model fitting shape = "ngmix" # Paths @@ -119,7 +118,7 @@ "FLUX_APER", "FLUXERR_APER", "NGMIX_T_NOSHEAR", - "NGMIX_Tpsf_NOSHEAR", + "NGMIX_T_PSF_RECONV_NOSHEAR", ] ## Pre-calibration catalogue, including masked objects and mask flags @@ -129,11 +128,13 @@ "IMAFLAGS_ISO", "FLAGS", "NGMIX_MCAL_FLAGS", - "NGMIX_MOM_FAIL", + "NGMIX_MCAL_TYPES_FAIL", "N_EPOCH", "NGMIX_N_EPOCH", - "NGMIX_ELL_PSFo_NOSHEAR", - "NGMIX_ELL_ERR_NOSHEAR", + "NGMIX_G1_PSF_ORIG_NOSHEAR", + "NGMIX_G2_PSF_ORIG_NOSHEAR", + "NGMIX_G1_ERR_NOSHEAR", + "NGMIX_G2_ERR_NOSHEAR", ] ### Set flag columns as integer format @@ -143,7 +144,7 @@ "IMAFLAGS_ISO", "FLAGS", "NGMIX_MCAL_FLAGS", - "NGMIX_MOM_FAIL", + "NGMIX_MCAL_TYPES_FAIL", "N_EPOCH", "NGMIX_N_EPOCH", ): @@ -155,7 +156,7 @@ # Create key names for metacal information prefix = "NGMIX" suffixes = ["1M", "1P", "2M", "2P", "NOSHEAR"] -centers = ["FLAGS", "ELL", "FLUX", "FLUX_ERR", "T", "T_ERR", "Tpsf"] +centers = ["FLAGS", "G1", "G2", "FLUX", "FLUX_ERR", "T", "T_ERR", "T_PSF_RECONV"] for center in centers: for suffix in suffixes: add_cols_pre_cal.append(f"{prefix}_{center}_{suffix}") @@ -185,9 +186,6 @@ gal_mag_bright = 15 gal_mag_faint = 30 -### Spread-model -do_spread_model = False - ### SExtractor flags to keep in addition to FLAGS=0 ### (bit-coded; list of powers of 2); ### Empty list if no flags diff --git a/scripts/examples/demo_calibrate_minimal_cat.py b/scripts/examples/demo_calibrate_minimal_cat.py index 85a0f536..a90025dd 100644 --- a/scripts/examples/demo_calibrate_minimal_cat.py +++ b/scripts/examples/demo_calibrate_minimal_cat.py @@ -52,9 +52,9 @@ "1024_Maximask", "N_EPOCH", "mag", - "NGMIX_MOM_FAIL", - "NGMIX_ELL_PSFo_NOSHEAR_0", - "NGMIX_ELL_PSFo_NOSHEAR_1", + "NGMIX_MCAL_TYPES_FAIL", + "NGMIX_G1_PSF_ORIG_NOSHEAR", + "NGMIX_G2_PSF_ORIG_NOSHEAR", ] masks, labels = sp_joint.get_masks_from_config( @@ -104,7 +104,6 @@ rel_size_max=cm["gal_rel_size_max"], size_corr_ell=cm["gal_size_corr_ell"], sigma_eps=cm["sigma_eps_prior"], - col_2d=False, verbose=True, ) # - diff --git a/scripts/examples/demo_comprehensive_to_minimal_cat.py b/scripts/examples/demo_comprehensive_to_minimal_cat.py index 243feda7..11e5ed5a 100644 --- a/scripts/examples/demo_comprehensive_to_minimal_cat.py +++ b/scripts/examples/demo_comprehensive_to_minimal_cat.py @@ -53,9 +53,9 @@ masks_to_apply = [ "overlap", "IMAFLAGS_ISO", - "NGMIX_MOM_FAIL", - "NGMIX_ELL_PSFo_NOSHEAR_0", - "NGMIX_ELL_PSFo_NOSHEAR_1", + "NGMIX_MCAL_TYPES_FAIL", + "NGMIX_G1_PSF_ORIG_NOSHEAR", + "NGMIX_G2_PSF_ORIG_NOSHEAR", "8_Manual", ] diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index b61e7261..9032bc63 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -19,7 +19,7 @@ from sp_validation.statistics import jackknif_weighted_average2 -def get_calibrated_quantities(gal_metacal, shape_method="ngmix"): +def get_calibrated_quantities(gal_metacal): """Get Calibrated Quantities. Return catalogue quantities for objects calibrated for multiplicative @@ -29,8 +29,6 @@ def get_calibrated_quantities(gal_metacal, shape_method="ngmix"): ---------- gal_metacal : dict galaxy metacalibration catalogue - shape_method : string, optional, default='ngmix' - shape measurement method, one in 'ngmix', 'galsim' Returns ------- @@ -58,7 +56,7 @@ def get_calibrated_quantities(gal_metacal, shape_method="ngmix"): return g_corr, g_uncorr, w, mask -def get_calibrated_m_c(gal_metacal, shape_method="ngmix"): +def get_calibrated_m_c(gal_metacal): """Get Calibrated C. Return catalogue quantities for objects calibrated for multiplicative and @@ -68,8 +66,6 @@ def get_calibrated_m_c(gal_metacal, shape_method="ngmix"): ---------- gal_metacal : dict galaxy metacalibration catalogue - shape_method : string, optional, default='ngmix' - shape measurement method, one in 'ngmix', 'galsim' Returns ------- @@ -182,10 +178,12 @@ def fill_cat_gal(cat_gal, dat, g_uncorr, gal_metacal, mask1, mask2, purpose="wei cat_gal["R_g22"] = gal_metacal.R22 cat_gal["NGMIX_T_NOSHEAR"] = sp_cat.get_col(dat, "NGMIX_T_NOSHEAR", mask1, mask2) - cat_gal["NGMIX_Tpsf_NOSHEAR"] = sp_cat.get_col( - dat, "NGMIX_Tpsf_NOSHEAR", mask1, mask2 + cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] = sp_cat.get_col( + dat, "NGMIX_T_PSF_RECONV_NOSHEAR", mask1, mask2 + ) + cat_gal["size_ratio"] = ( + cat_gal["NGMIX_T_NOSHEAR"] / cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] ) - cat_gal["size_ratio"] = cat_gal["NGMIX_T_NOSHEAR"] / cat_gal["NGMIX_Tpsf_NOSHEAR"] cat_gal["snr"] = sp_cat.get_col( dat, "NGMIX_FLUX_NOSHEAR", mask1, mask2 @@ -322,8 +320,8 @@ def get_alpha_leakage_per_object(cat_gal, num_bins, weight_type="des"): """ assert weight_type in ["des", "iv"], "weight_type must be either 'des' or 'iv'" # Compute the size ratio - size_ratio = cat_gal["NGMIX_Tpsf_NOSHEAR"] / ( - cat_gal["NGMIX_T_NOSHEAR"] + cat_gal["NGMIX_Tpsf_NOSHEAR"] + size_ratio = cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] / ( + cat_gal["NGMIX_T_NOSHEAR"] + cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] ) df_gal = pd.DataFrame( @@ -715,9 +713,6 @@ class metacal: sigma_eps : float, optional ellipticity dispersion (one component) for computation of weights; default is 0.34 - col_2d : bool, optional - if `True` (default, ellipticity in one 2D column; - if `False`, ellipticity in two columns ELL_0, ELL_1 verbose : bool, optional, default=False verbose output if True @@ -737,7 +732,6 @@ def __init__( size_corr_ell=True, global_R_weight=None, sigma_eps=0.34, - col_2d=True, verbose=False, ): @@ -761,7 +755,6 @@ def __init__( self._global_R_weight = global_R_weight self._sigma_eps = sigma_eps - self._col_2d = col_2d self._verbose = verbose @@ -791,19 +784,14 @@ def _read_data(self, data, mask): p2, ns, ) - elif self._prefix == "GALSIM": - m1, p1, m2, p2, ns = self._read_data_galsim( - masked_data, - m1, - p1, - m2, - p2, - ns, + else: + raise ValueError( + f"Unsupported shape prefix '{self._prefix}'; only 'NGMIX' is supported" ) print("FHP/MK hack using p1 PSF for ns in cuts") indices = np.where(mask)[0] - col_1p = f"{self._prefix}_Tpsf_1P" + col_1p = f"{self._prefix}_T_PSF_RECONV_1P" new_psf = data[col_1p][indices] # Overwriting incorrect no-shear PSF size to the one from 1p @@ -830,32 +818,24 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): dict_tmp["flag"] = masked_data[f"{self._prefix}_FLAGS_{name_shear}"] - if self._col_2d: - # Ellipticity in one 2D column - for comp in (0, 1): - dict_tmp[f"g{comp + 1}"] = masked_data[ - f"{self._prefix}_ELL_{name_shear}" - ][:, comp] - else: - # Ellipcitiy in two different columns - for comp in (0, 1): - dict_tmp[f"g{comp + 1}"] = masked_data[ - f"{self._prefix}_ELL_{name_shear}_{comp}" - ] + # Ellipticity in named scalar components (ShapePipe-v2 grammar) + for comp in (0, 1): + dict_tmp[f"g{comp + 1}"] = masked_data[ + f"{self._prefix}_G{comp + 1}_{name_shear}" + ] for key in ("flux", "flux_err", "T", "T_err"): dict_tmp[key] = masked_data[ f"{self._prefix}_{key.upper()}_{name_shear}" ] - dict_tmp["Tpsf"] = masked_data[f"{self._prefix}_Tpsf_{name_shear}"] + dict_tmp["Tpsf"] = masked_data[f"{self._prefix}_T_PSF_RECONV_{name_shear}"] ns["C11"], ns["C22"], ns["w"] = self.get_variance_ivweights( masked_data, self._sigma_eps, self._prefix, mask=None, - col_2d=self._col_2d, ) self._n_input = len(masked_data) @@ -870,7 +850,7 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): return m1, p1, m2, p2, ns @staticmethod - def get_variance_ivweights(data, sigma_eps, prefix="NGMIX", mask=None, col_2d=True): + def get_variance_ivweights(data, sigma_eps, prefix="NGMIX", mask=None): """Get Variance IVWEIGHTS. Compute variance and inverse-variance weights. @@ -886,9 +866,6 @@ def get_variance_ivweights(data, sigma_eps, prefix="NGMIX", mask=None, col_2d=Tr mask : list, optional indicates valid objects with ``True`` values; default is ``None`` = use all objects type has to be bool - col_2d : bool, optional - if ``True`` (default), ellipticity is given in single 2D column; - if ``False``, ellipticity is expected in two 1D columns. Returns ------- @@ -900,59 +877,16 @@ def get_variance_ivweights(data, sigma_eps, prefix="NGMIX", mask=None, col_2d=Tr weight """ + C11 = data[f"{prefix}_G1_ERR_NOSHEAR"] + C22 = data[f"{prefix}_G2_ERR_NOSHEAR"] if mask is not None: - if col_2d: - C11 = data[f"{prefix}_ELL_ERR_NOSHEAR"][:, 0][mask] - C22 = data[f"{prefix}_ELL_ERR_NOSHEAR"][:, 1][mask] - else: - C11 = data[f"{prefix}_ELL_ERR_NOSHEAR_0"][mask] - C22 = data[f"{prefix}_ELL_ERR_NOSHEAR_1"][mask] - else: - if col_2d: - C11 = data[f"{prefix}_ELL_ERR_NOSHEAR"][:, 0] - C22 = data[f"{prefix}_ELL_ERR_NOSHEAR"][:, 1] - else: - C11 = data[f"{prefix}_ELL_ERR_NOSHEAR_0"] - C22 = data[f"{prefix}_ELL_ERR_NOSHEAR_1"] + C11 = C11[mask] + C22 = C22[mask] iv_w = 1 / (2 * sigma_eps**2 + C11 + C22) return C11, C22, iv_w - def _read_data_galsim(self, masked_data, m1, p1, m2, p2, ns): - """Read Data Galsim. - - Read data from galsim catalogue. - - """ - prefix_mom = "GALSIM_GAL" - - for name_shear, dict_tmp in zip( - ["1m", "1p", "2m", "2p", "noshear"], [m1, p1, m2, p2, ns] - ): - if self._verbose: - print("Extracting {}".format(name_shear)) - - dict_tmp["flag"] = masked_data[f"{self._prefix}_FLAGS_{name_shear.upper()}"] - dict_tmp["g1"] = masked_data[ - f"{prefix_mom}_ELL_UNCORR_{name_shear.upper()}" - ][:, 0] - dict_tmp["g2"] = masked_data[ - f"{prefix_mom}_ELL_UNCORR_{name_shear.upper()}" - ][:, 1] - - dict_tmp["T"] = masked_data[f"{prefix_mom}_SIGMA_{name_shear.upper()}"] - dict_tmp["Tpsf"] = masked_data[ - f"{self._prefix}_PSF_SIGMA_{name_shear.upper()}" - ] - - self.snr_sextractor = masked_data["SNR_WIN"] - ns["C11"] = masked_data[f"{prefix_mom}_ELL_ERR_NOSHEAR"][:, 0] - ns["C22"] = masked_data[f"{prefix_mom}_ELL_ERR_NOSHEAR"][:, 1] - ns["w"] = 1.0 / (2 * self._sigma_eps**2 + dict_tmp["C11"] + dict_tmp["C22"]) - - return m1, p1, m2, p2, ns - def _compute_calibration(self): """Compute Calibration. @@ -1094,15 +1028,11 @@ def _shear_response(self): Compute shear response matrix """ - sign = 1 - if self._prefix == "GALSIM": - sign = -1 - ma = self.mask_dict["ns"] h2 = 2 * self._step self.R11 = (self.p1["g1"][ma] - self.m1["g1"][ma]) / h2 - self.R22 = sign * (self.p2["g2"][ma] - self.m2["g2"][ma]) / h2 + self.R22 = (self.p2["g2"][ma] - self.m2["g2"][ma]) / h2 self.R12 = (self.p2["g1"][ma] - self.m2["g1"][ma]) / h2 self.R21 = (self.p1["g2"][ma] - self.m1["g2"][ma]) / h2 @@ -1144,10 +1074,6 @@ def _selection_response(self): ... """ - sign = 1 - if self._prefix == "GALSIM": - sign = -1 - ma_p1 = self.mask_dict["p1"] ma_m1 = self.mask_dict["m1"] ma_p2 = self.mask_dict["p2"] @@ -1158,8 +1084,8 @@ def _selection_response(self): np.mean(self.ns["g1"][ma_p1]) - np.mean(self.ns["g1"][ma_m1]) ) / h2 self.R22_s = ( - sign * (np.mean(self.ns["g2"][ma_p2]) - np.mean(self.ns["g2"][ma_m2])) / h2 - ) + np.mean(self.ns["g2"][ma_p2]) - np.mean(self.ns["g2"][ma_m2]) + ) / h2 self.R12_s = ( np.mean(self.ns["g1"][ma_p2]) - np.mean(self.ns["g1"][ma_m2]) ) / h2 diff --git a/src/sp_validation/catalog.py b/src/sp_validation/catalog.py index 4fe45476..b1b175d9 100644 --- a/src/sp_validation/catalog.py +++ b/src/sp_validation/catalog.py @@ -217,7 +217,7 @@ def check_matching( return ind, mask_area_tiles, n_tot -def check_invalid(dd, key, comp, val, stats_file, name=None, verbose=False): +def check_invalid(dd, key, val, stats_file, name=None, verbose=False): """Check invalid objects. Check whether objects have invalid values. @@ -227,9 +227,7 @@ def check_invalid(dd, key, comp, val, stats_file, name=None, verbose=False): dd : dict catalog key : list - key names of columns to check - comp : array of int - components for above columns + key names of (scalar) columns to check val : array of float values for above columns indicating invalid entries stats_file : file handler @@ -245,7 +243,7 @@ def check_invalid(dd, key, comp, val, stats_file, name=None, verbose=False): n_all = len(dd) for i in range(len(key)): - w = dd[key[i]][:, comp[i]] == val[i] + w = dd[key[i]] == val[i] n_inv_psf = len(np.where(w)[0]) msg = "Invalid {} found for {}/{} = {:.1g}% objects".format( name[i], n_inv_psf, n_all, n_inv_psf / n_all @@ -258,7 +256,8 @@ def match_subsample( ind, mask, pos_key, - ell_key, + g1_key, + g2_key, n_ref, stats_file, verbose=False, @@ -277,8 +276,8 @@ def match_subsample( boolean mask pos_key : list key names for position columns - ell_key : str - key name for ellipticity column + g1_key, g2_key : str + key names for the two scalar ellipticity components n_ref : int reference number of objects stats_file : file handler @@ -302,42 +301,13 @@ def match_subsample( ra = dd[pos_key[0]][ind][mask] dec = dd[pos_key[1]][ind][mask] - g1 = dd[ell_key][:, 0][ind][mask] - g2 = dd[ell_key][:, 1][ind][mask] + g1 = dd[g1_key][ind][mask] + g2 = dd[g2_key][ind][mask] g = np.array([g1, g2]) return ra, dec, g -def match_spread_class(dd, ind, mask, stats_file, n_ref, verbose=False): - """Match spread class. - - Match - """ - tot_star = n_ref - tot_as_star = len(np.where(dd["SPREAD_CLASS"][ind][mask] == 0)[0]) - tot_as_gal = len(np.where(dd["SPREAD_CLASS"][ind][mask] == 1)[0]) - tot_as_other = len(np.where(dd["SPREAD_CLASS"][ind][mask] == 2)[0]) - - msg = ( - "Number of stars selected as star (SPREAD_CLASS=0) = " - + f"{tot_as_star}/{tot_star} = {tot_as_star / tot_star * 100:.1f}%" - ) - io.print_stats(msg, stats_file, verbose=verbose) - - msg = ( - "Number of stars selected as galaxy (SPREAD_CLASS=1) = " - + f"{tot_as_gal}/{tot_star} = {tot_as_gal / tot_star * 100:.1f}%" - ) - io.print_stats(msg, stats_file, verbose=verbose) - - msg = ( - "Number of stars selected as other (SPREAD_CLASS=2) = " - + f"{tot_as_other}/{tot_star} = {tot_as_other / tot_star * 100:.1f}%" - ) - io.print_stats(msg, stats_file, verbose=verbose) - - def match_stars2(ra_gal, dec_gal, ra_star, dec_star, thresh=0.0002): """Add docstring. @@ -885,7 +855,7 @@ def get_snr(sh, dat, m_sel, m_flg): my_snr = get_col(dat, "NGMIX_FLUX_NOSHEAR", m_sel, m_flg) / get_col( dat, "NGMIX_FLUX_ERR_NOSHEAR", m_sel, m_flg ) - elif sh == "galsim": - my_snr = get_col(dat, "SNR_WIN", m_sel, m_flg) + else: + raise ValueError(f"Unsupported shape method '{sh}'; only 'ngmix' is supported") return my_snr diff --git a/src/sp_validation/galaxy.py b/src/sp_validation/galaxy.py index 616f0ad5..3c49a96b 100644 --- a/src/sp_validation/galaxy.py +++ b/src/sp_validation/galaxy.py @@ -141,23 +141,12 @@ def classification_galaxy_base( gal_mag_faint=26, flags_keep=None, n_epoch_min=1, - do_spread_model=True, ): """Classification Galaxy Base. Return mask corresponding to basic classification for galaxies. """ - if do_spread_model: - # spread model class, add two times the uncertainty to be conservative - sm_classif = dd["SPREAD_MODEL"] + 2 * dd["SPREADERR_MODEL"] - cut_sm = sm_classif > 0.0035 - - cut_sm_all = cut_sm & (dd["SPREAD_MODEL"] > 0) & (dd["SPREAD_MODEL"] < 0.03) - else: - # Do not use spread model - cut_sm_all = True - # SExtractor flags # Keep some flags if specified if flags_keep: @@ -181,7 +170,6 @@ def classification_galaxy_base( cut_common = ( cut_overlap & cut_flags - & cut_sm_all & (dd["MAG_AUTO"] <= gal_mag_faint) & (dd["MAG_AUTO"] >= gal_mag_bright) & (dd["IMAFLAGS_ISO"] == 0) @@ -204,8 +192,8 @@ def classification_galaxy_ngmix( m_gal_ngmix = ( cut_common & (dd["NGMIX_MCAL_FLAGS"] == 0) - & (dd["NGMIX_ELL_PSFo_NOSHEAR"][:, 0] != -10) - & (dd["NGMIX_MOM_FAIL"] == 0) + & (dd["NGMIX_G1_PSF_ORIG_NOSHEAR"] != -10) + & (dd["NGMIX_MCAL_TYPES_FAIL"] == 0) ) n_gal_ngmix = len(np.where(m_gal_ngmix)[0]) @@ -223,28 +211,6 @@ def classification_galaxy_ngmix( return m_gal_ngmix -def classification_galaxy_galsim(dd, cut_common, stats_file, verbose=False): - """Classification Galaxy Galsim. - - Return mask corresponding to galsim classification of galaxies - - """ - m_gal_galsim = cut_common & (dd["GALSIM_PSF_ELL_ORIGINAL_PSF"][:, 0] != -10) - - n_gal_galsim = len(np.where(m_gal_galsim)[0]) - n_tot = len(dd) - - io.print_ratio( - "galsim: Objects selected as galaxies", - n_gal_galsim, - n_tot, - stats_file, - verbose=verbose, - ) - - return m_gal_galsim - - def mask_overlap(ra, dec, tile_id_in, region_file_path, n_jobs=-1): """Mask Overlap. diff --git a/src/sp_validation/glass_mock.py b/src/sp_validation/glass_mock.py index 7ba0057b..de8312c7 100644 --- a/src/sp_validation/glass_mock.py +++ b/src/sp_validation/glass_mock.py @@ -493,8 +493,8 @@ def compute_leakage_harmony(cat, cat_star, nside=1024, lmin=8, n_bins=32): e1 = cat["e1"] e2 = cat["e2"] - e1_psf = cat_star["E1_PSF_HSM"] - e2_psf = cat_star["E2_PSF_HSM"] + e1_psf = cat_star["HSM_G1_PSF"] + e2_psf = cat_star["HSM_G2_PSF"] ra = cat["ra"] dec = cat["dec"] diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index 0a311432..6874a08c 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -10,15 +10,6 @@ # `from sp_validation.rho_tau import SquareRootScale` keeps working. from sp_validation.plots import SquareRootScale # noqa: F401 -not_square_size = [ - "DES", - "SP_v1.3_LFmask_8k", - "SP_v1.3_LFmask_8k_no_alpha", - "SP_v1.3_LFmask_8k_li_2024", - "SP_v1.3_LFmask_8k_SN8", - "SP_v1.3_LFmask_8k_F2", -] - def _extract_xip(correlations): """Return flattened array of xip values from a list of correlations.""" @@ -49,7 +40,6 @@ def get_params_rho_tau(cat, survey="other"): params["e2_star_col"] = cat["psf"]["e2_star_col"] params["PSF_size"] = cat["psf"]["PSF_size"] params["star_size"] = cat["psf"]["star_size"] - params["square_size"] = survey not in not_square_size if survey != "DES": params["PSF_flag"] = cat["psf"]["PSF_flag"] params["star_flag"] = cat["psf"]["star_flag"] @@ -175,12 +165,10 @@ def get_rho_tau( rho_stat_handler.catalogs.set_params(params, outdir) mask = version != "DES" - square_size = params["square_size"] rho_stat_handler.build_cat_to_compute_rho( config[version]["psf"]["path"], catalog_id=catalog_id, - square_size=square_size, mask=mask, hdu=( config[version]["psf"]["hdu"] @@ -218,15 +206,12 @@ def get_rho_tau( mask = version != "DES" - square_size = params["square_size"] - # Build the different catalogs if necessary if f"psf_{version}" not in tau_stat_handler.catalogs.catalogs_dict.keys(): tau_stat_handler.build_cat_to_compute_tau( config[version]["psf"]["path"], cat_type="psf", catalog_id=version, - square_size=square_size, mask=mask, hdu=( config[version]["psf"]["hdu"] @@ -240,7 +225,6 @@ def get_rho_tau( config[version]["shear"]["path"], cat_type="gal", catalog_id=version, - square_size=square_size, mask=mask, ) @@ -354,8 +338,6 @@ def get_jackknife_cov( rho_stat_handler.catalogs.set_params(params, outdir) - square_size = params["square_size"] - tau_stat_handler = TauStat( catalogs=rho_stat_handler.catalogs, output=outdir, @@ -376,7 +358,6 @@ def get_jackknife_cov( rho_stat_handler.build_cat_to_compute_rho( config[version]["psf"]["path"], catalog_id=version + str(i), - square_size=square_size, mask=False, hdu=config[version]["psf"]["hdu"], ) @@ -390,7 +371,6 @@ def get_jackknife_cov( config[version]["shear"]["path"], cat_type="gal", catalog_id=version + str(i), - square_size=square_size, mask=False, ) diff --git a/src/sp_validation/tests/test_calibration.py b/src/sp_validation/tests/test_calibration.py index b1b0dd26..40621c19 100644 --- a/src/sp_validation/tests/test_calibration.py +++ b/src/sp_validation/tests/test_calibration.py @@ -259,20 +259,20 @@ def _build_ngmix_catalog(slope_11=2.0, slope_22=3.0, step=0.01): cols = {} for var in _VARIANTS: dg1, dg2 = shifts[var] - ell = np.zeros((n, 2)) - ell[:, 0] = g1_base + dg1 - ell[:, 1] = g2_base + dg2 - cols[f"NGMIX_ELL_{var}"] = ell + # ShapePipe-v2 grammar: ellipticity in named scalar components. + cols[f"NGMIX_G1_{var}"] = np.full(n, g1_base + dg1) + cols[f"NGMIX_G2_{var}"] = np.full(n, g2_base + dg2) cols[f"NGMIX_FLAGS_{var}"] = np.zeros(n, dtype=int) cols[f"NGMIX_FLUX_{var}"] = np.full(n, 50.0) cols[f"NGMIX_FLUX_ERR_{var}"] = np.full(n, 1.0) - # T / Tpsf = 1.0 -> rel_size inside [0.5, 3.0]. + # T / T_PSF_RECONV = 1.0 -> rel_size inside [0.5, 3.0]. cols[f"NGMIX_T_{var}"] = np.full(n, 1.0) cols[f"NGMIX_T_ERR_{var}"] = np.full(n, 0.1) - cols[f"NGMIX_Tpsf_{var}"] = np.full(n, 1.0) + cols[f"NGMIX_T_PSF_RECONV_{var}"] = np.full(n, 1.0) # Per-component ellipticity errors -> inverse-variance weights. - cols["NGMIX_ELL_ERR_NOSHEAR"] = np.full((n, 2), 0.25) + cols["NGMIX_G1_ERR_NOSHEAR"] = np.full(n, 0.25) + cols["NGMIX_G2_ERR_NOSHEAR"] = np.full(n, 0.25) return Table(cols), n diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 0a618e9a..f50992d4 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -418,14 +418,14 @@ def _write_synthetic_catalogs( { "RA": rng.uniform(*ra_range, n_star), "Dec": rng.uniform(*dec_range, n_star), - "E1_PSF_HSM": rng.normal(0, 0.03, n_star), - "E2_PSF_HSM": rng.normal(0, 0.03, n_star), - "E1_STAR_HSM": rng.normal(0, 0.03, n_star), - "E2_STAR_HSM": rng.normal(0, 0.03, n_star), - "SIGMA_PSF_HSM": rng.uniform(0.4, 0.6, n_star), - "SIGMA_STAR_HSM": rng.uniform(0.4, 0.6, n_star), - "FLAG_PSF_HSM": np.zeros(n_star, dtype=int), - "FLAG_STAR_HSM": np.zeros(n_star, dtype=int), + "HSM_G1_PSF": rng.normal(0, 0.03, n_star), + "HSM_G2_PSF": rng.normal(0, 0.03, n_star), + "HSM_G1_STAR": rng.normal(0, 0.03, n_star), + "HSM_G2_STAR": rng.normal(0, 0.03, n_star), + "HSM_T_PSF": rng.uniform(0.4, 0.6, n_star), + "HSM_T_STAR": rng.uniform(0.4, 0.6, n_star), + "HSM_FLAG_PSF": np.zeros(n_star, dtype=int), + "HSM_FLAG_STAR": np.zeros(n_star, dtype=int), } ).write(star_path, overwrite=True) @@ -449,8 +449,8 @@ def _write_synthetic_catalogs( "path": "star.fits", "ra_col": "RA", "dec_col": "Dec", - "e1_col": "E1_PSF_HSM", - "e2_col": "E2_PSF_HSM", + "e1_col": "HSM_G1_PSF", + "e2_col": "HSM_G2_PSF", } version_cfg = { "subdir": str(cat_dir), @@ -464,14 +464,14 @@ def _write_synthetic_catalogs( "hdu": 1, "ra_col": "RA", "dec_col": "Dec", - "e1_PSF_col": "E1_PSF_HSM", - "e2_PSF_col": "E2_PSF_HSM", - "e1_star_col": "E1_STAR_HSM", - "e2_star_col": "E2_STAR_HSM", - "PSF_size": "SIGMA_PSF_HSM", - "star_size": "SIGMA_STAR_HSM", - "PSF_flag": "FLAG_PSF_HSM", - "star_flag": "FLAG_STAR_HSM", + "e1_PSF_col": "HSM_G1_PSF", + "e2_PSF_col": "HSM_G2_PSF", + "e1_star_col": "HSM_G1_STAR", + "e2_star_col": "HSM_G2_STAR", + "PSF_size": "HSM_T_PSF", + "star_size": "HSM_T_STAR", + "PSF_flag": "HSM_FLAG_PSF", + "star_flag": "HSM_FLAG_STAR", } config_data = {