merge master into met

Author: dlebauer

README.md

@@ -15,12 +15,12 @@ Many of these tutorials have complex dependencies. In addition, many datasets ar
 The first research-grade version of TERRA REF data products will be released in November 2018. 
 Before that, we will make evaluation releases available: the alpha version was released in November 2016 and the beta version will be released in 2017.
 
-To access data please fill out a quick [Alpha User application](https://docs.google.com/forms/d/e/1FAIpQLScBsD042RrRok70BCGCRwARTcm9etvVHqvQaz1c5X7c5y0H3w/viewform?c=0&w=1).
+To access data please fill out a quick [Beta User application](http://terraref.org/beta).
 
 ### Links
 
 TODO: add links to quick-start documentation, README's, code for learning and applied examples
 
 * Data portal: terraref.org/data
 * Docker Images on Docker Hub: hub.docker.com/terraref
-* NDS Labs Workbench: terraref.ndslabs.org
+* NDS Labs Workbench: workbench.terraref.org

sensors/01-meteorological-data.Rmd

@@ -119,14 +119,15 @@ library(jsonlite)
 
 url = ""
 mac_weather.list <- jsonlite::fromJSON('https://terraref.ncsa.illinois.edu/clowder/api/geostreams/datapoints?key=Pb3AUSqnUw&stream_id=4807&since=2017-01-02&until=2017-01-31')
+#mac_weather.list <- jsonlite::fromJSON('https://terraref.ncsa.illinois.edu/clowder/api/geostreams/datapoints?key=Pb3AUSqnUw&stream_id=3208&since=2017-01-02&until=2017-12-31', flatten = FALSE)
 
 # change time to human-readable
 mac_weather <- mac_weather.list$properties %>% 
   mutate(time = lubridate::ymd_hms(mac_weather.list$end_time))
-  
-
 ```
 
+### Using 
+
 ## Weather Summary
 
 
@@ -186,8 +187,9 @@ Did you see what we just did? We wrote some custom code to convert the units of
 
 This higher resolution weather data can be used for VNIR calibration, for example. But at 1/s it is very large!
 
+## Lets see how data are downloaded
 
-### Lets see how data are downloaded
+Here we will download the files using the Clowder API, but note that if you have access to the filesystem (on www.workbench.terraref.org or globus, you can directly access the data in the `sites/ua-mac/Level_1/EnvironmentLogger`. Folder
 
 ```{r query-clowder}
 library(jsonlite)
@@ -215,7 +217,7 @@ ncfiles <- files[grepl('environmentlogger.nc', files$filename), ]
 print(ncfiles %>% select(id, filename))
 ```
 
-#### Download netCDF 1/s data from Clowder
+## Download netCDF 1/s data from Clowder
 
 
 ```{r nc-download, echo=FALSE}

sensors/01.2-pecan-met-utilities.Rmd

@@ -0,0 +1,76 @@
+# Using the PEcAn atmospheric data utilities
+
+Explain what these are
+
+github.com/pecanproject/pecan
+
+insert slide from talks ... 
+
+## Dependencies
+
+```{r install-pecan-dependencies, message=FALSE, eval = FALSE}
+
+devtools::install_github("pecanproject/pecan",  
+                         subdir = 'base/utils', ref = 'develop', dependencies = FALSE)
+devtools::install_github("pecanproject/pecan",  
+                         subdir = 'base/db')
+devtools::install_github("rforge/reddyproc",
+                         subdir = "pkg/REddyProc")
+devtools::install_github("pecanproject/pecan",  
+                         subdir = 'modules/data.atmosphere',
+                         ref = 'develop')
+
+source("https://raw.githubusercontent.com/PecanProject/pecan/develop/models/biocro/R/met2model.BIOCRO.R")
+```
+
+
+## PEcAn Met Workflow 
+
+```{r write-clowder, eval = FALSE}
+writeLines("
+<pecan>
+  <clowder>
+    <hostname>terraref.ncsa.illinois.edu</hostname>
+    <user>user@illinois.edu</user>
+    <password>ask</password>
+  </clowder>
+</pecan>", 
+con =  "~/.pecan.clowder.xml")
+```
+
+![](pecan.clowder.xml.png)
+
+```{r pecan-met-workflow, message=FALSE, warning=FALSE, eval = FALSE}
+library("PEcAn.data.atmosphere")
+library("dplyr")
+
+## download raw data
+ne <- download.Geostreams(
+  outfolder="data",
+  sitename="EnvironmentLogger sensor_weather_station",
+  start_date="2016-02-28",
+  end_date="2016-04-01",
+  overwrite = TRUE)
+
+## convert to standard
+ne_cf <- met2CF.Geostreams(
+  in.path = "data/", 
+  in.prefix = ne$dbfile.name, 
+  outfolder = "data/cf",
+  start_date = "2016-03-01", # note date shift to avoid TZ issues
+  end_date = "2016-04-01",
+  overwrite = TRUE)
+
+## convert to model specific input
+met2model.BIOCRO(
+  overwrite = TRUE,
+  in.path = "data/cf", 
+  in.prefix = ne_cf$dbfile.name, 
+  outfolder = "data/biocromet",
+  lat = 40,
+  lon = -88,
+  start_date = "2016-03-01", 
+  end_date = "2016-03-30")
+
+met <- readr::read_csv('data/biocromet/Clowder.UIUC Energy Farm - NE.2016-02-28.2016-04-01.2016.csv')
+```

sensors/06-list-datasets-by-plot.md sensors/06-list-datasets-by-plot.Rmdsensors/06-list-datasets-by-plot.md renamed to sensors/06-list-datasets-by-plot.Rmd

@@ -7,25 +7,26 @@ from the workbench or Globus.
 ## Getting started
 
 After installing terrautils, you should be able to import the *product* module.
-```
+```{python}
 from terrautils.products import get_sensor_list, unique_sensor_names
 from terrautils.products import get_file_listing, extract_file_paths
 ```
 
-The get\_sensor\_list and get\_file\_listing both require connection, url,
+The `get_sensor_list` and `get_file_listing` functions both require connection, url,
 and key parameters. *Connection* can be 'None', the *url* (called host in the
 code) should be something like https://terraref.ncsa.illinois.edu/clowder/.
 The *key* is a unique access key for the Clowder api.
 
 ## Getting the sensor list
+
 The first thing to get is the sensor name. This can be retreived using the
-get\_sensor\_list function. This function returns the full record which may
+`get_sensor_list` function. This function returns the full record which may
 be useful in some cases but primarily includes sensor names that include
-a plot id number. The utility function unique_sensor_names accpets the 
+a plot id number. The utility function `unique_sensor_names` accpets the 
 sensor list and provides a list of names suitable for use in the 
-get_file_listing function.
+`get_file_listing` function.
 
-```
+```{python}
 sensors = get_sensor_list(None, url, key)
 names = unique_sensor_names(sensors)
 ```
@@ -36,16 +37,16 @@ geostreams API. The currently available sensors are:
 * IR Surface Temperature
 * Thermal IR GeoTIFFs Datasets
 * flirIrCamera Datasets
-* (EL) sensor\_weather\_station
+* (EL) sensor_weather_station
 * Irrigation Observations
 * Canopy Cover
 * Energy Farm Observations SE
-* (EL) sensor\_par
+* (EL) sensor_par
 * scanner3DTop Datasets
 * Weather Observations
 * Energy Farm Observations NE
 * RGB GeoTIFFs Datasets
-* (EL) sensor\_co2
+* (EL) sensor_co2
 * stereoTop Datasets
 * Energy Farm Observations CEN
 
@@ -55,28 +56,30 @@ The geostreams API can be used to get a list of datasets that overlap a
 specific plot boundary and, optionally, limited by a time range. Iterating 
 over the datasets allows the paths to all the files to be extracted.
 
-```
+```{python}
 sensor = 'Thermal IR GeoTIFFs Datasets'
 sitename = 'MAC Field Scanner Season 1 Field Plot 101 W'
 datasets = get_file_listing(None, url, key, sensor, sitename)
 files = extract_file_paths(datasets)
 ```
 
 Datasets can be further filtered using the *since* and *until* parameters
-of get\_file\_listing with a date string.
+of `get_file_listing` with a date string.
 
-```
+```{python}
 dataset = get_file_listing(None, url, key, sensor, sitename, 
         since='2016-06-01', until='2016-06-10')
 ```
 
 
 # Alternative method
+
 The following method demonstrates the same approach using the Clowder API. This
 approach is useful for understanding the data layout and when the Python
 terrautils package is not available.
 
 ## Finding plot ID
+
 ```
 SENSOR_NAME = "MAC Field Scanner Season 1 Field Plot 101 W"
 GET https://terraref.ncsa.illinois.edu/clowder/api/geostreams/sensors?sensor_name={SENSOR_NAME}
@@ -85,7 +88,9 @@ GET https://terraref.ncsa.illinois.edu/clowder/api/geostreams/sensors?sensor_nam
 This returns a JSON object with an 'id' parameter. You can use this ID parameter to specify the right data stream.
 
 ## Finding stream ID within a plot
+
 The names are formatted as "<Sensor Group> Datasets (<Sensor ID>)".
+
 ```
 SENSOR_ID = 3355
 STREAM_NAME = "Thermal IR GeoTIFFs Datasets ({SENSOR_ID})"
@@ -95,13 +100,15 @@ GET https://terraref.ncsa.illinois.edu/clowder/api/geostreams/streams?stream_nam
 This returns a JSON object with an 'id' parameter. You can use this ID parameter to get the right datapoints.
 
 ## Listing Clowder file IDs for that plot & sensor stream
+
 ```
 STREAM_ID = "11586"
 GET https://terraref.ncsa.illinois.edu/clowder/api/geostreams/datapoints?stream_id={STREAM_ID}
 ```
 
 This returns a list of datapoint JSON objects, each with a 'properties' parameter that looks like:
-```
+
+```{python}
 properties: {
     dataset_name: "Thermal IR GeoTIFFs - 2016-05-09__12-07-57-990",
     source_dataset: "https://terraref.ncsa.illinois.edu/clowder/datasets/59fc9e7d4f0c3383c73d2905"
@@ -111,19 +118,23 @@ properties: {
 The source_dataset URL can be used to view the dataset in Clowder.
 
 You can also filter the datapoints by date:
+
 ```
 GET https://terraref.ncsa.illinois.edu/clowder/api/geostreams/datapoints?stream_id={STREAM_ID}&since=2017-01-02&until=2017-06-10
 ```
 
 ## Getting ROGER file path from dataset
+
 Given a source dataset URL, we can call the API to get the files and their paths.
+
 ```
 SOURCE_DATASET = "https://terraref.ncsa.illinois.edu/clowder/datasets/59fc9e7d4f0c3383c73d2905"
 # Add /api after /clowder, and add /files at the end of the URL
 GET "https://terraref.ncsa.illinois.edu/clowder/api/datasets/59fc9e7d4f0c3383c73d2905/files"
 ```
 
 This returns a list of files in the dataset and their paths if available:
+
 ```
 [
     {

sensors/08-hyperspectral-calibration.Rmd

@@ -0,0 +1,51 @@
+
+## Hyperspectral Data
+
+### Calibration Targets
+
+These were collected on April 15 2017 every ~15 minutes
+
+
+```{r get-vnir-calibration, eval=FALSE}
+library(ncdf4)
+library(dplyr)
+
+hsi_calibration_dir <- '/data/terraref/sites/ua-mac/Level_1/hyperspectral/2017-04-15'
+hsi_calibration_files <- dir(hsi_calibration_dir, 
+                             recursive = TRUE,
+                             full.names = TRUE)
+
+fileinfo <- bind_rows(lapply(hsi_calibration_files, file.info)) %>%
+  mutate(size_gb = size/1073741824)
+
+calibration_nc <- nc_open(hsi_calibration_files[200])
+a <- calibration_nc$var$rfl_img
+
+
+#calibration_nc$dim$x$len 1600
+#calibration_nc$dim$y$len
+x_length <- round(calibration_nc$dim$x$len / 10)
+y_length <- round(calibration_nc$dim$y$len * 3/4)
+
+xstart <- ceiling(calibration_nc$dim$x$len / 2) - floor(x_length / 2) + 1
+
+ystart <- ceiling(calibration_nc$dim$y$len / 2) - floor(y_length / 2) + 1
+
+rfl <- ncvar_get(calibration_nc, 'rfl_img', 
+          #start = c(1, xstart, ystart), 
+          #count = c(955, x_length, y_length)
+          start = c(2, 2, 2),
+          count = c(1320, 10, 954)
+          )
+x <- ncvar_get(calibration_nc, 'x', start = 100, count = 160)
+y <- ncvar_get(calibration_nc, 'y', start = 100, count = 1324)
+lambda <- calibration_nc$dim$wavelength$vals
+for(i in 1 + 0:10*95){
+  image(x = x, y = y, z = rfl[i,,],
+        xlab = 'x (m)', ylab = 'y (m)',
+        col = rainbow(n=100),
+        main = paste('wavelength', 
+                      udunits2::ud.convert(lambda[i],'m','nm')))
+} 
+
+```