Skip to content

Commit db79ddf

Browse files
andrewdelmanandrewdelman
committed
updates to budget tutorials for in-cloud access
1 parent 0f6d9b2 commit db79ddf

3 files changed

Lines changed: 260 additions & 657 deletions

File tree

‎Tutorials_as_Jupyter_Notebooks/ECCO_v4_Heat_budget_closure.ipynb‎

Lines changed: 77 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,11 @@
112112
"import numpy as np\n",
113113
"import xarray as xr\n",
114114
"import sys\n",
115-
"import glob"
115+
"import glob\n",
116+
"\n",
117+
"\n",
118+
"# indicate whether you are working in a cloud instance (True if yes, False otherwise)\n",
119+
"incloud_access = True"
116120
]
117121
},
118122
{
@@ -530,6 +534,42 @@
530534
"ECCO_dir = join(user_home_dir,'Downloads','ECCO_V4r4_PODAAC')"
531535
]
532536
},
537+
{
538+
"cell_type": "code",
539+
"execution_count": null,
540+
"metadata": {},
541+
"outputs": [],
542+
"source": [
543+
"## if working in the AWS cloud, access datasets needed for this tutorial\n",
544+
"\n",
545+
"ShortNames_list = [\"ECCO_L4_GEOMETRY_LLC0090GRID_V4R4\",\\\n",
546+
" \"ECCO_L4_OCEAN_3D_TEMPERATURE_FLUX_LLC0090GRID_MONTHLY_V4R4\",\\\n",
547+
" \"ECCO_L4_HEAT_FLUX_LLC0090GRID_MONTHLY_V4R4\",\\\n",
548+
" \"ECCO_L4_SSH_LLC0090GRID_SNAPSHOT_V4R4\",\\\n",
549+
" \"ECCO_L4_TEMP_SALINITY_LLC0090GRID_SNAPSHOT_V4R4\"]\n",
550+
"if incloud_access == True:\n",
551+
" # apply fixes to xgcm (you might have to do this manually if not on a cloud instance,\n",
552+
" # see \"Horizontal convergence of advective heat flux section)\n",
553+
" import xgcm\n",
554+
" xgcm_path = xgcm.__path__[0]\n",
555+
" if xgcm.__version__ == '0.8.1':\n",
556+
" import os\n",
557+
" os.remove(join(xgcm_path,'grid_ufunc.py'))\n",
558+
" os.system('cp ../misc/xgcm_fixes/grid_ufunc.py '+xgcm_path+'/')\n",
559+
" os.remove(join(xgcm_path,'padding.py'))\n",
560+
" os.system('cp ../misc/xgcm_fixes/padding.py '+xgcm_path+'/')\n",
561+
"\n",
562+
" # re-import xgcm\n",
563+
" import importlib\n",
564+
" importlib.reload(xgcm)\n",
565+
" \n",
566+
" from ecco_s3_retrieve import ecco_podaac_s3_get_diskaware\n",
567+
" files_dict = ecco_podaac_s3_get_diskaware(ShortNames=ShortNames_list,\\\n",
568+
" StartDate='1993-01',EndDate='2016-12',\\\n",
569+
" max_avail_frac=0.5,\\\n",
570+
" download_root_dir=ECCO_dir)"
571+
]
572+
},
533573
{
534574
"cell_type": "code",
535575
"execution_count": 8,
@@ -539,7 +579,10 @@
539579
"outputs": [],
540580
"source": [
541581
"## Load the model grid\n",
542-
"ecco_grid = xr.open_dataset(glob.glob(join(ECCO_dir,'*GEOMETRY*','*.nc'))[0])"
582+
"if incloud_access == True:\n",
583+
" ecco_grid = xr.open_dataset(files_dict[ShortNames_list[0]])\n",
584+
"else:\n",
585+
" ecco_grid = xr.open_dataset(glob.glob(join(ECCO_dir,'*GEOMETRY*','*.nc'))[0])"
543586
]
544587
},
545588
{
@@ -569,20 +612,24 @@
569612
},
570613
{
571614
"cell_type": "code",
572-
"execution_count": 10,
573-
"metadata": {
574-
"scrolled": false
575-
},
615+
"execution_count": null,
616+
"metadata": {},
576617
"outputs": [],
577618
"source": [
578-
"year_start = 1993\n",
619+
"## year_start = 1993\n",
579620
"year_end = 2017\n",
580621
"\n",
581622
"# open ETAN and THETA snapshots (beginning of each month)\n",
582-
"ecco_monthly_SSH = xr.open_mfdataset(join(ECCO_dir,'*SSH*SNAPSHOT*','*-01T*.nc'),\\\n",
583-
" data_vars='minimal',coords='minimal',compat='override')\n",
584-
"ecco_monthly_TS = xr.open_mfdataset(join(ECCO_dir,'*TEMP_SALINITY*SNAPSHOT*','*-01T*.nc'),\\\n",
585-
" data_vars='minimal',coords='minimal',compat='override')\n",
623+
"if incloud_access == True:\n",
624+
" ecco_monthly_SSH = xr.open_mfdataset(files_dict[ShortNames_list[-2]],\\\n",
625+
" data_vars='minimal',coords='minimal',compat='override')\n",
626+
" ecco_monthly_TS = xr.open_mfdataset(files_dict[ShortNames_list[-1]],\\\n",
627+
" data_vars='minimal',coords='minimal',compat='override')\n",
628+
"else:\n",
629+
" ecco_monthly_SSH = xr.open_mfdataset(join(ECCO_dir,'*SSH*SNAPSHOT*','*-01T*.nc'),\\\n",
630+
" data_vars='minimal',coords='minimal',compat='override')\n",
631+
" ecco_monthly_TS = xr.open_mfdataset(join(ECCO_dir,'*TEMP_SALINITY*SNAPSHOT*','*-01T*.nc'),\\\n",
632+
" data_vars='minimal',coords='minimal',compat='override')\n",
586633
"ecco_monthly_snaps = xr.merge((ecco_monthly_SSH['ETAN'],ecco_monthly_TS['THETA']))\n",
587634
"\n",
588635
"# time mask for snapshots\n",
@@ -646,10 +693,16 @@
646693
"outputs": [],
647694
"source": [
648695
"## Open ECCO monthly mean variables\n",
649-
"ecco_vars_int = xr.open_mfdataset(join(ECCO_dir,'*_OCEAN_3D_TEMPERATURE_FLUX*MONTHLY*','*.nc'),\\\n",
650-
" data_vars='minimal',coords='minimal',compat='override')\n",
651-
"ecco_vars_sfc = xr.open_mfdataset(join(ECCO_dir,'*_HEAT_FLUX*MONTHLY*','*.nc'),\\\n",
652-
" data_vars='minimal',coords='minimal',compat='override')\n",
696+
"if incloud_access == True:\n",
697+
" ecco_vars_int = xr.open_mfdataset(files_dict[ShortNames_list[1]],\\\n",
698+
" data_vars='minimal',coords='minimal',compat='override')\n",
699+
" ecco_vars_sfc = xr.open_mfdataset(files_dict[ShortNames_list[2]],\\\n",
700+
" data_vars='minimal',coords='minimal',compat='override')\n",
701+
"else:\n",
702+
" ecco_vars_int = xr.open_mfdataset(join(ECCO_dir,'*_OCEAN_3D_TEMPERATURE_FLUX*MONTHLY*','*.nc'),\\\n",
703+
" data_vars='minimal',coords='minimal',compat='override')\n",
704+
" ecco_vars_sfc = xr.open_mfdataset(join(ECCO_dir,'*_HEAT_FLUX*MONTHLY*','*.nc'),\\\n",
705+
" data_vars='minimal',coords='minimal',compat='override')\n",
653706
"ecco_monthly_mean = xr.merge((ecco_vars_int,\\\n",
654707
" ecco_vars_sfc[['TFLUX','oceQsw']]))\n",
655708
"\n",
@@ -2808,12 +2861,13 @@
28082861
"outputs": [],
28092862
"source": [
28102863
"# Load terms from zarr dataset\n",
2811-
"G_total = xr.open_zarr(join(save_dir,'eccov4r4_budg_heat')).G_total\n",
2812-
"G_advection = xr.open_zarr(join(save_dir,'eccov4r4_budg_heat')).G_advection\n",
2813-
"G_diffusion = xr.open_zarr(join(save_dir,'eccov4r4_budg_heat')).G_diffusion\n",
2814-
"G_forcing = xr.open_zarr(join(save_dir,'eccov4r4_budg_heat')).G_forcing\n",
2815-
"Qnet = xr.open_zarr(join(save_dir,'eccov4r4_budg_heat')).Qnet\n",
2816-
"SWpen = xr.open_zarr(join(save_dir,'eccov4r4_budg_heat')).SWpen"
2864+
"G_budget = xr.open_zarr(join(save_dir,'eccov4r4_budg_heat'))\n",
2865+
"G_total = G_budget.G_total\n",
2866+
"G_advection = G_budget.G_advection\n",
2867+
"G_diffusion = G_budget.G_diffusion\n",
2868+
"G_forcing = G_budget.G_forcing\n",
2869+
"Qnet = G_budget.Qnet\n",
2870+
"SWpen = G_budget.SWpen"
28172871
]
28182872
},
28192873
{
@@ -3355,9 +3409,9 @@
33553409
"name": "python",
33563410
"nbconvert_exporter": "python",
33573411
"pygments_lexer": "ipython3",
3358-
"version": "3.9.13"
3412+
"version": "3.11.8"
33593413
}
33603414
},
33613415
"nbformat": 4,
3362-
"nbformat_minor": 2
3416+
"nbformat_minor": 4
33633417
}

‎Tutorials_as_Jupyter_Notebooks/ECCO_v4_Salt_and_salinity_budget.ipynb‎

Lines changed: 63 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,12 @@
157157
"outputs": [],
158158
"source": [
159159
"import numpy as np\n",
160-
"import xarray as xr"
160+
"import xarray as xr\n",
161+
"import glob\n",
162+
"\n",
163+
"\n",
164+
"# indicate whether you are working in a cloud instance (True if yes, False otherwise)\n",
165+
"incloud_access = True"
161166
]
162167
},
163168
{
@@ -250,6 +255,44 @@
250255
"> **Note**: Change `ECCO_dir` to the directory path where ECCOv4 output is stored."
251256
]
252257
},
258+
{
259+
"cell_type": "code",
260+
"execution_count": null,
261+
"metadata": {},
262+
"outputs": [],
263+
"source": [
264+
"## if working in the AWS cloud, access datasets needed for this tutorial\n",
265+
"\n",
266+
"ShortNames_list = [\"ECCO_L4_GEOMETRY_LLC0090GRID_V4R4\",\\\n",
267+
" \"ECCO_L4_OCEAN_3D_SALINITY_FLUX_LLC0090GRID_MONTHLY_V4R4\",\\\n",
268+
" \"ECCO_L4_OCEAN_3D_VOLUME_FLUX_LLC0090GRID_MONTHLY_V4R4\",\\\n",
269+
" \"ECCO_L4_OCEAN_BOLUS_STREAMFUNCTION_LLC0090GRID_MONTHLY_V4R4\",\\\n",
270+
" \"ECCO_L4_FRESH_FLUX_LLC0090GRID_MONTHLY_V4R4\",\\\n",
271+
" \"ECCO_L4_SSH_LLC0090GRID_SNAPSHOT_V4R4\",\\\n",
272+
" \"ECCO_L4_TEMP_SALINITY_LLC0090GRID_SNAPSHOT_V4R4\"]\n",
273+
"if incloud_access == True:\n",
274+
" # apply fixes to xgcm (you might have to do this manually if not on a cloud instance,\n",
275+
" # see \"Horizontal convergence of advective heat flux\" section)\n",
276+
" import xgcm\n",
277+
" xgcm_path = xgcm.__path__[0]\n",
278+
" if xgcm.__version__ == '0.8.1':\n",
279+
" import os\n",
280+
" os.remove(join(xgcm_path,'grid_ufunc.py'))\n",
281+
" os.system('cp ../misc/xgcm_fixes/grid_ufunc.py '+xgcm_path+'/')\n",
282+
" os.remove(join(xgcm_path,'padding.py'))\n",
283+
" os.system('cp ../misc/xgcm_fixes/padding.py '+xgcm_path+'/')\n",
284+
"\n",
285+
" # re-import xgcm\n",
286+
" import importlib\n",
287+
" importlib.reload(xgcm)\n",
288+
" \n",
289+
" from ecco_s3_retrieve import ecco_podaac_s3_get_diskaware\n",
290+
" files_dict = ecco_podaac_s3_get_diskaware(ShortNames=ShortNames_list,\\\n",
291+
" StartDate='1993-01',EndDate='2016-12',\\\n",
292+
" max_avail_frac=0.5,\\\n",
293+
" download_root_dir=ECCO_dir)"
294+
]
295+
},
253296
{
254297
"cell_type": "code",
255298
"execution_count": 7,
@@ -258,10 +301,11 @@
258301
},
259302
"outputs": [],
260303
"source": [
261-
"# Load the model grid\n",
262-
"grid_shortname = \"ECCO_L4_GEOMETRY_LLC0090GRID_V4R4\"\n",
263-
"grid_filename = \"GRID_GEOMETRY_ECCO_V4r4_native_llc0090.nc\"\n",
264-
"ecco_grid = xr.open_dataset(join(ECCO_dir,grid_shortname,grid_filename))"
304+
"## Load the model grid\n",
305+
"if incloud_access == True:\n",
306+
" ecco_grid = xr.open_dataset(files_dict[ShortNames_list[0]])\n",
307+
"else:\n",
308+
" ecco_grid = xr.open_dataset(glob.glob(join(ECCO_dir,'*GEOMETRY*','*.nc'))[0])"
265309
]
266310
},
267311
{
@@ -349,7 +393,7 @@
349393
"cell_type": "markdown",
350394
"metadata": {},
351395
"source": [
352-
"Now we can use [wget](https://ecco-v4-python-tutorial.readthedocs.io/Tutorial_wget_Command_Line_HTTPS_Downloading_ECCO_Datasets_from_PODAAC.html) to download the list of files in *etan_snaps_filenames.txt* and *salt_snaps_filenames.txt*, e.g., wget --no-verbose --no-clobber --continue -i etan_snaps_filenames.txt -P ~/Downloads/ECCO_V4r4_PODAAC/ECCO_L4_SSH_LLC0090GRID_SNAPSHOT_V4R4/"
396+
"Now we can use [wget](https://ecco-v4-python-tutorial.readthedocs.io/Tutorial_wget_Command_Line_HTTPS_Downloading_ECCO_Datasets_from_PODAAC.html) to download the list of files in *etan_snaps_filenames.txt* and *salt_snaps_filenames.txt*, e.g., ```wget --no-verbose --no-clobber --continue -i etan_snaps_filenames.txt -P ~/Downloads/ECCO_V4r4_PODAAC/ECCO_L4_SSH_LLC0090GRID_SNAPSHOT_V4R4/```"
353397
]
354398
},
355399
{
@@ -385,10 +429,16 @@
385429
"salt_snaps_in_range = files_in_year_range(salt_snaps_dir,year_start,year_end+1)\n",
386430
"\n",
387431
"# open files as two xarray datasets, then merge to create one dataset with the variables we need\n",
388-
"ds_etan_snaps = xr.open_mfdataset(etan_snaps_in_range,\\\n",
389-
" data_vars='minimal',coords='minimal',compat='override')\n",
390-
"ds_salt_snaps = xr.open_mfdataset(salt_snaps_in_range,\\\n",
391-
" data_vars='minimal',coords='minimal',compat='override')\n",
432+
"if incloud_access == True:\n",
433+
" ds_etan_snaps = xr.open_mfdataset(files_dict[ShortNames_list[-2]],\\\n",
434+
" data_vars='minimal',coords='minimal',compat='override')\n",
435+
" ds_salt_snaps = xr.open_mfdataset(files_dict[ShortNames_list[-1]],\\\n",
436+
" data_vars='minimal',coords='minimal',compat='override')\n",
437+
"else:\n",
438+
" ds_etan_snaps = xr.open_mfdataset(etan_snaps_in_range,\\\n",
439+
" data_vars='minimal',coords='minimal',compat='override')\n",
440+
" ds_salt_snaps = xr.open_mfdataset(salt_snaps_in_range,\\\n",
441+
" data_vars='minimal',coords='minimal',compat='override')\n",
392442
"ecco_monthly_snaps = xr.merge([ds_etan_snaps['ETAN'],ds_salt_snaps['SALT']])\n",
393443
"\n",
394444
"# Exclude snapshots after Jan 1 of year_end\n",
@@ -433,6 +483,7 @@
433483
"# open datasets where monthly mean files are located\n",
434484
"datasets_to_open = ['FW_surf_flux','salt_flux','vol_flux','bolus_strmfcn',\\\n",
435485
" 'etan_monthly','salt_monthly']\n",
486+
"if incloud_access == True:\n",
436487
"for dataset in datasets_to_open:\n",
437488
" curr_shortname = eval(dataset + '_shortname')\n",
438489
" curr_dir = join(ECCO_dir,curr_shortname)\n",
@@ -4262,9 +4313,7 @@
42624313
{
42634314
"cell_type": "code",
42644315
"execution_count": 132,
4265-
"metadata": {
4266-
"scrolled": false
4267-
},
4316+
"metadata": {},
42684317
"outputs": [
42694318
{
42704319
"data": {
@@ -4314,7 +4363,7 @@
43144363
"name": "python",
43154364
"nbconvert_exporter": "python",
43164365
"pygments_lexer": "ipython3",
4317-
"version": "3.9.13"
4366+
"version": "3.11.8"
43184367
}
43194368
},
43204369
"nbformat": 4,

0 commit comments

Comments
 (0)