update stream_ids

Merge branch 'master' of github.com:terraref/tutorials into clowder-met

# Conflicts:
#	sensors/01-meteorological-data.Rmd

Author: dlebauer

sensors/01-meteorological-data.Rmd

@@ -7,7 +7,7 @@ output: html_document
 knitr::opts_chunk$set(echo = FALSE, cache = TRUE)
 
 api_url <- "https://terraref.ncsa.illinois.edu/clowder/api"
-output_dir <- "~/data/downloads/"
+output_dir <- file.path(tempdir(), "downloads")
 dir.create(output_dir, showWarnings = FALSE, recursive = TRUE)
 
 ```
@@ -124,27 +124,33 @@ Here is the json representation of a single five-minute observation:
 
 The data represent 5 minute summaries aggregated from 1/s observations.
 
+#### Using Curl
 
+First, this is what the API looks like as a URL. Try pasting it into your browser
 
-#### Using Curl
+
+https://terraref.ncsa.illinois.edu/clowder/api/geostreams/datapoints?user=USER&pass=PASSWORD&stream_id=746&since=2017-01-02&until=2017-01-31
+
+This is how you can automatically download the met data to a local file:
 
 ```{sh eval=FALSE}
-curl -O spectra.json -X GET https://terraref.ncsa.illinois.edu/clowder/api/geostreams/datapoints?user=USER&pass=PASSWORD&stream_id=743
+curl -O spectra.json -X GET https://terraref.ncsa.illinois.edu/clowder/api/geostreams/datapoints?user=USER&pass=PASSWORD&stream_id=746
 ```
 
+And this is how you can access the data in R:
+
+
 ```{r met-geostream}
 library(dplyr)
 library(ggplot2)
 library(jsonlite)
 
 url = ""
-mac_weather.list <- jsonlite::fromJSON('https://terraref.ncsa.illinois.edu/clowder/api/geostreams/datapoints?user=USER&pass=PASS&stream_id=300', flatten = FALSE)
+mac_weather.list <- jsonlite::fromJSON('https://terraref.ncsa.illinois.edu/clowder/api/geostreams/datapoints?user=USER&pass=PASSWORD&stream_id=746&since=2017-01-02&until=2017-01-31', flatten = FALSE)
 
 # change time to human-readable
 mac_weather <- mac_weather.list$properties %>% 
   mutate(time = lubridate::ymd_hms(mac_weather.list$end_time))
-  
-
 ```
 
 ### Using 

sensors/01.2-pecan-met-utilities.Rmd

@@ -0,0 +1,76 @@
+# Using the PEcAn atmospheric data utilities
+
+Explain what these are
+
+github.com/pecanproject/pecan
+
+insert slide from talks ... 
+
+## Dependencies
+
+```{r install-pecan-dependencies, message=FALSE, eval = FALSE}
+
+devtools::install_github("pecanproject/pecan",  
+                         subdir = 'base/utils', ref = 'develop', dependencies = FALSE)
+devtools::install_github("pecanproject/pecan",  
+                         subdir = 'base/db')
+devtools::install_github("rforge/reddyproc",
+                         subdir = "pkg/REddyProc")
+devtools::install_github("pecanproject/pecan",  
+                         subdir = 'modules/data.atmosphere',
+                         ref = 'develop')
+
+source("https://raw.githubusercontent.com/PecanProject/pecan/develop/models/biocro/R/met2model.BIOCRO.R")
+```
+
+
+## PEcAn Met Workflow 
+
+```{r write-clowder, eval = FALSE}
+writeLines("
+<pecan>
+  <clowder>
+    <hostname>terraref.ncsa.illinois.edu</hostname>
+    <user>user@illinois.edu</user>
+    <password>ask</password>
+  </clowder>
+</pecan>", 
+con =  "~/.pecan.clowder.xml")
+```
+
+![](pecan.clowder.xml.png)
+
+```{r pecan-met-workflow, message=FALSE, warning=FALSE, eval = FALSE}
+library("PEcAn.data.atmosphere")
+library("dplyr")
+
+## download raw data
+ne <- download.Geostreams(
+  outfolder="data",
+  sitename="EnvironmentLogger sensor_weather_station",
+  start_date="2016-02-28",
+  end_date="2016-04-01",
+  overwrite = TRUE)
+
+## convert to standard
+ne_cf <- met2CF.Geostreams(
+  in.path = "data/", 
+  in.prefix = ne$dbfile.name, 
+  outfolder = "data/cf",
+  start_date = "2016-03-01", # note date shift to avoid TZ issues
+  end_date = "2016-04-01",
+  overwrite = TRUE)
+
+## convert to model specific input
+met2model.BIOCRO(
+  overwrite = TRUE,
+  in.path = "data/cf", 
+  in.prefix = ne_cf$dbfile.name, 
+  outfolder = "data/biocromet",
+  lat = 40,
+  lon = -88,
+  start_date = "2016-03-01", 
+  end_date = "2016-03-30")
+
+met <- readr::read_csv('data/biocromet/Clowder.UIUC Energy Farm - NE.2016-02-28.2016-04-01.2016.csv')
+```

sensors/06-list-datasets-by-plot.md sensors/06-list-datasets-by-plot.Rmdsensors/06-list-datasets-by-plot.md renamed to sensors/06-list-datasets-by-plot.Rmd

@@ -7,25 +7,26 @@ from the workbench or Globus.
 ## Getting started
 
 After installing terrautils, you should be able to import the *product* module.
-```
+```{python}
 from terrautils.products import get_sensor_list, unique_sensor_names
 from terrautils.products import get_file_listing, extract_file_paths
 ```
 
-The get\_sensor\_list and get\_file\_listing both require connection, url,
+The `get_sensor_list` and `get_file_listing` functions both require connection, url,
 and key parameters. *Connection* can be 'None', the *url* (called host in the
 code) should be something like https://terraref.ncsa.illinois.edu/clowder/.
 The *key* is a unique access key for the Clowder api.
 
 ## Getting the sensor list
+
 The first thing to get is the sensor name. This can be retreived using the
-get\_sensor\_list function. This function returns the full record which may
+`get_sensor_list` function. This function returns the full record which may
 be useful in some cases but primarily includes sensor names that include
-a plot id number. The utility function unique_sensor_names accpets the 
+a plot id number. The utility function `unique_sensor_names` accpets the 
 sensor list and provides a list of names suitable for use in the 
-get_file_listing function.
+`get_file_listing` function.
 
-```
+```{python}
 sensors = get_sensor_list(None, url, key)
 names = unique_sensor_names(sensors)
 ```
@@ -36,16 +37,16 @@ geostreams API. The currently available sensors are:
 * IR Surface Temperature
 * Thermal IR GeoTIFFs Datasets
 * flirIrCamera Datasets
-* (EL) sensor\_weather\_station
+* (EL) sensor_weather_station
 * Irrigation Observations
 * Canopy Cover
 * Energy Farm Observations SE
-* (EL) sensor\_par
+* (EL) sensor_par
 * scanner3DTop Datasets
 * Weather Observations
 * Energy Farm Observations NE
 * RGB GeoTIFFs Datasets
-* (EL) sensor\_co2
+* (EL) sensor_co2
 * stereoTop Datasets
 * Energy Farm Observations CEN
 
@@ -55,28 +56,30 @@ The geostreams API can be used to get a list of datasets that overlap a
 specific plot boundary and, optionally, limited by a time range. Iterating 
 over the datasets allows the paths to all the files to be extracted.
 
-```
+```{python}
 sensor = 'Thermal IR GeoTIFFs Datasets'
 sitename = 'MAC Field Scanner Season 1 Field Plot 101 W'
 datasets = get_file_listing(None, url, key, sensor, sitename)
 files = extract_file_paths(datasets)
 ```
 
 Datasets can be further filtered using the *since* and *until* parameters
-of get\_file\_listing with a date string.
+of `get_file_listing` with a date string.
 
-```
+```{python}
 dataset = get_file_listing(None, url, key, sensor, sitename, 
         since='2016-06-01', until='2016-06-10')
 ```
 
 
 # Alternative method
+
 The following method demonstrates the same approach using the Clowder API. This
 approach is useful for understanding the data layout and when the Python
 terrautils package is not available.
 
 ## Finding plot ID
+
 ```
 SENSOR_NAME = "MAC Field Scanner Season 1 Field Plot 101 W"
 GET https://terraref.ncsa.illinois.edu/clowder/api/geostreams/sensors?sensor_name={SENSOR_NAME}
@@ -85,7 +88,9 @@ GET https://terraref.ncsa.illinois.edu/clowder/api/geostreams/sensors?sensor_nam
 This returns a JSON object with an 'id' parameter. You can use this ID parameter to specify the right data stream.
 
 ## Finding stream ID within a plot
+
 The names are formatted as "<Sensor Group> Datasets (<Sensor ID>)".
+
 ```
 SENSOR_ID = 3355
 STREAM_NAME = "Thermal IR GeoTIFFs Datasets ({SENSOR_ID})"
@@ -95,13 +100,15 @@ GET https://terraref.ncsa.illinois.edu/clowder/api/geostreams/streams?stream_nam
 This returns a JSON object with an 'id' parameter. You can use this ID parameter to get the right datapoints.
 
 ## Listing Clowder file IDs for that plot & sensor stream
+
 ```
 STREAM_ID = "11586"
 GET https://terraref.ncsa.illinois.edu/clowder/api/geostreams/datapoints?stream_id={STREAM_ID}
 ```
 
 This returns a list of datapoint JSON objects, each with a 'properties' parameter that looks like:
-```
+
+```{python}
 properties: {
     dataset_name: "Thermal IR GeoTIFFs - 2016-05-09__12-07-57-990",
     source_dataset: "https://terraref.ncsa.illinois.edu/clowder/datasets/59fc9e7d4f0c3383c73d2905"
@@ -111,19 +118,23 @@ properties: {
 The source_dataset URL can be used to view the dataset in Clowder.
 
 You can also filter the datapoints by date:
+
 ```
 GET https://terraref.ncsa.illinois.edu/clowder/api/geostreams/datapoints?stream_id={STREAM_ID}&since=2017-01-02&until=2017-06-10
 ```
 
 ## Getting ROGER file path from dataset
+
 Given a source dataset URL, we can call the API to get the files and their paths.
+
 ```
 SOURCE_DATASET = "https://terraref.ncsa.illinois.edu/clowder/datasets/59fc9e7d4f0c3383c73d2905"
 # Add /api after /clowder, and add /files at the end of the URL
 GET "https://terraref.ncsa.illinois.edu/clowder/api/datasets/59fc9e7d4f0c3383c73d2905/files"
 ```
 
 This returns a list of files in the dataset and their paths if available:
+
 ```
 [
     {

sensors/08-hyperspectral-calibration.Rmd

@@ -0,0 +1,51 @@
+
+## Hyperspectral Data
+
+### Calibration Targets
+
+These were collected on April 15 2017 every ~15 minutes
+
+
+```{r get-vnir-calibration, eval=FALSE}
+library(ncdf4)
+library(dplyr)
+
+hsi_calibration_dir <- '/data/terraref/sites/ua-mac/Level_1/hyperspectral/2017-04-15'
+hsi_calibration_files <- dir(hsi_calibration_dir, 
+                             recursive = TRUE,
+                             full.names = TRUE)
+
+fileinfo <- bind_rows(lapply(hsi_calibration_files, file.info)) %>%
+  mutate(size_gb = size/1073741824)
+
+calibration_nc <- nc_open(hsi_calibration_files[200])
+a <- calibration_nc$var$rfl_img
+
+
+#calibration_nc$dim$x$len 1600
+#calibration_nc$dim$y$len
+x_length <- round(calibration_nc$dim$x$len / 10)
+y_length <- round(calibration_nc$dim$y$len * 3/4)
+
+xstart <- ceiling(calibration_nc$dim$x$len / 2) - floor(x_length / 2) + 1
+
+ystart <- ceiling(calibration_nc$dim$y$len / 2) - floor(y_length / 2) + 1
+
+rfl <- ncvar_get(calibration_nc, 'rfl_img', 
+          #start = c(1, xstart, ystart), 
+          #count = c(955, x_length, y_length)
+          start = c(2, 2, 2),
+          count = c(1320, 10, 954)
+          )
+x <- ncvar_get(calibration_nc, 'x', start = 100, count = 160)
+y <- ncvar_get(calibration_nc, 'y', start = 100, count = 1324)
+lambda <- calibration_nc$dim$wavelength$vals
+for(i in 1 + 0:10*95){
+  image(x = x, y = y, z = rfl[i,,],
+        xlab = 'x (m)', ylab = 'y (m)',
+        col = rainbow(n=100),
+        main = paste('wavelength', 
+                      udunits2::ud.convert(lambda[i],'m','nm')))
+} 
+
+```