Merge pull request #237 from rOpenGov/development

Development

Author: MansMeg

DESCRIPTION

@@ -1,8 +1,8 @@
 Type: Package
 Package: pxweb
 Title: R Interface to PXWEB APIs
-Version: 0.12.0
-Date: 2021-10-09
+Version: 0.13.0
+Date: 2022-01-29
 Authors@R: c(
     person("Mans", "Magnusson", , "mons.magnusson@gmail.com", role = c("aut", "cre")),
     person("Markus", "Kainu", role = "aut"),
@@ -40,7 +40,8 @@ Suggests:
     knitr,
     remotes,
     rmarkdown,
-    testthat (>= 0.11)
+    testthat (>= 0.11),
+    xml2
 VignetteBuilder: 
     knitr
 Encoding: UTF-8

R/pxweb_as_data_frame.R

@@ -100,9 +100,7 @@ pxweb_as_data_frame.pxweb_levels <- function(x, optional = FALSE, ..., stringsAs
   df <- list()
   for(i in seq_along(x)){
     df[[i]] <- as.data.frame(x[[i]], optional = optional, stringsAsFactors = FALSE)
-    if(x[[i]]$type %in% c("l", "h")) {
-      df[[i]]$updated <- NA
-    }
+    if(is.null(df[[i]]$updated)) df[[i]]$updated <- NA
   }
   df <- do.call(rbind, df)
 

R/pxweb_interactive.R

@@ -960,7 +960,7 @@ pxe_interactive_get_data <- function(pxe, test_input = NULL){
   if(print_citation){
     cat("############# CITATION #############")
     pxweb_cite(dat)
-    cat("############# CITATION #############")
+    cat("############# CITATION #############\n")
   }
   if(return_df){
     dat <- as.data.frame(dat)

R/pxweb_test_api_endpoint.R

@@ -63,13 +63,13 @@ pxweb_test_api <- function(url, test_type="first", n = 1, verbose = TRUE, time_l
       break
     }
 
-    if(api_tree_df$type[i] == "t") {
-      next
-    }
-    
     if(verbose) {
-      cat(sum(api_tree_df$type == "l"), "node(s) and", sum(api_tree_df$type == "t"), "table(s)...\n")
+      cat(sum(api_tree_df$type == "l"), "node(s) and", sum(api_tree_df$type == "t"), "table(s). Checking entry", i, "of", nrow(api_tree_df), "...\n")
     }
+
+    if(api_tree_df$type[i] == "t") {
+      next
+    }    
 
     px <- try(pxweb(url = api_tree_df$path[i]), silent = TRUE)
     if(inherits(px, "try-error")) {
@@ -85,11 +85,14 @@ pxweb_test_api <- function(url, test_type="first", n = 1, verbose = TRUE, time_l
 
     api_tree_df$checked[i] <- TRUE
 
-    if(test_type == "touch"){
+    if(test_type == "touch" & !inherits(tmp_df, "try-error")){
       # Jump to next level
       i <- nrow(api_tree_df)
     }
-    api_tree_df <- rbind(api_tree_df, tmp_df)
+    
+    if(!inherits(tmp_df, "try-error")){
+      api_tree_df <- rbind(api_tree_df, tmp_df)
+    }
   }
 
   if(verbose & !test_type == "touch"){

inst/extdata/api.json

@@ -373,7 +373,7 @@
         "lang": ["fi"],
         "calls_per_period": 1000,
         "period_in_seconds": 10,
-        "max_values_to_download" : 5000000
+        "max_values_to_download": 10000000
     },
 
     "andmed.stat.ee": {
@@ -388,9 +388,21 @@
         "calls_per_period": 1000,
         "period_in_seconds": 10,
         "max_values_to_download" : 1000000
+    },
+    
+    "pxweb.nordicstatistics.org": {
+        "description" : "Nordic Statistics Database",
+        "citation" : {
+          "organization" : "Nordic Statistics database"
+          },
+        "url" : "https://pxweb.nordicstatistics.org/api/[version]/[lang]/",
+        "version": ["v1"], 
+        "lang": ["en"],
+        "calls_per_period": 100,
+        "period_in_seconds": 10,
+        "max_values_to_download" : 1000000
     }
   },
-    
   "local_apis": {}
 
 }

tests/testthat/test-pxweb_constructor.R

@@ -78,7 +78,8 @@ test_that(desc="pxweb_clear_cache() without any files and WARNING for redirect."
   expect_silent(pxweb_clear_cache())
   expect_silent(pxweb_clear_cache())
 
-  expect_warning(pxweb("http://pxweb.asub.ax/PXWeb/api/v1/sv/"), regexp = "PXWEB URL CHANGE")
+  # Now not working
+  # expect_warning(pxweb("http://pxweb.asub.ax/PXWeb/api/v1/sv/"), regexp = "PXWEB URL CHANGE")
 
 })  
 

tests_bash/pxweb.R

@@ -101,7 +101,7 @@ if(length(new_api_idx) > 0){
 
   for(i in seq_along(new_api_paths)){
     # cat(new_api_paths[i], "\n")
-    first_results[[i]] <- try(pxweb_test_api(url = new_api_paths[i], test_type = "first", verbose = TRUE, time_limit = 10*60), silent = TRUE)
+    first_results[[i]] <- try(pxweb_test_api(url = new_api_paths[i], test_type = "first", verbose = TRUE, time_limit = 15*60), silent = TRUE)
     if(inherits(first_results[[i]], "try-error")){
       new_api_errored[i] <- TRUE
     }