Merge pull request #866 from OpenKnowledgeMaps/dev

refactor: aquanavi data update and documents renaming

Author: chreman

.gitignore

@@ -38,6 +38,7 @@ local_dev/paper_preview
 # php files
 /server/classes/headstart/vendor
 /server/classes/headstart/var/*
+**.phpunit.cache
 
 # R files
 server/preprocessing/other-scripts/.Rhistory
@@ -46,6 +47,9 @@ server/preprocessing/other-scripts/renv
 .Rprofile
 .Rproj*
 /*.Rproj
+*/**/*.Rproj
+*/**/*.RData
+server/workers/*/renv
 .Rproj.user
 
 # python files
@@ -66,6 +70,9 @@ server/preprocessing/other-scripts/renv
 # temporal
 local_dev/config_local_headstart.ini
 local_dev/config_local_searchflow.ini
+local_dev/tools/**/*.csv
+local_dev/tools/**/*.json
+local_dev/tools/**/*.log
 
 # mac os
 .DS_Store

examples/public/main_integration/base_sg.html

@@ -20,7 +20,7 @@
     <meta name="keywords" content="knowldege visualization, open knowledge, open science">
 
     <meta name="description"
-        content="Get an overview of digital education, find relevant papers, and identify important concepts.">
+        content="Get an overview of digital education, find relevant research outputs, and identify important concepts.">
 
     <!-- FAVICONS -->
     <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png?v=xQz6nej7eR">

examples/public/main_integration/openaire.html

@@ -20,7 +20,7 @@
     <meta name="keywords" content="knowldege visualization, open knowledge, open science">
 
     <meta name="description"
-          content="Get an overview of digital education, find relevant papers, and identify important concepts.">
+          content="Get an overview of digital education, find relevant research outputs, and identify important concepts.">
 
     <!-- FAVICONS -->
     <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png?v=xQz6nej7eR">

examples/templates/base.html

@@ -22,7 +22,7 @@
     <link rel="icon" type="image/png" sizes="16x16" href="./main_integration/img/favicon-16x16.png?v=xQz6nej7eR">
 
     <meta name="description"
-        content="Get an overview of digital education, find relevant papers, and identify important concepts.">
+        content="Get an overview of digital education, find relevant research outputs, and identify important concepts.">
 
     <!-- FAVICONS -->
     <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png?v=xQz6nej7eR">

examples/templates/pubmed.html

@@ -22,7 +22,7 @@
     <link rel="icon" type="image/png" sizes="16x16" href="./main_integration/img/favicon-16x16.png?v=xQz6nej7eR">
 
     <meta name="description"
-        content="Get an overview of &quot;climate change&quot;, find relevant papers, and identify important concepts.">
+        content="Get an overview of &quot;climate change&quot;, find relevant research outputs, and identify important concepts.">
 
     <link type="text/css" rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css">
     <link href='https://fonts.googleapis.com/css?family=Open+Sans:300,400,600,700,800' rel='stylesheet' type='text/css'>

examples/templates/triple.html

@@ -7,7 +7,7 @@
     <title>Overview of research on psychology</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="description"
-        content="Get an overview of psychology, find relevant papers, and identify important concepts.">
+        content="Get an overview of psychology, find relevant research outputs, and identify important concepts.">
 
     <script type="text/javascript" src="./triple/data-config-km.js"></script>
     <script type="text/javascript" src="./triple/data-config-sg.js"></script>

local_dev/config_local_headstart.ini

@@ -35,6 +35,8 @@ snapshot_php = "headstart/server/services/snapshot/headstart_snapshot.php"
 snapshot_local_protocol = "http://"
 # Size of thumbnail width
 snapshot_width = "1200px"
+# chrome executable path optional, only needed if puppeteer cannot find chrome automatically
+chrome_executable_path = "/path/to/chrome"
 
 [connection]
 # PostgreSQL database

server/preprocessing/conf/config.ini

@@ -32,7 +32,7 @@ storage_path = "/path/to/storage/"
 snapshot_php = "server/services/snapshot/headstart_snapshot.php"
 # snapshot_local_protocol fallback for non-server environments
 snapshot_local_protocol = "http://"
-# chrome executable path (optional, only needed if puppeteer cannot find chrome automatically)
+# chrome executable path optional, only needed if puppeteer cannot find chrome automatically
 chrome_executable_path = "/path/to/chrome"
 
 

server/preprocessing/other-scripts/openaire.R

@@ -52,11 +52,6 @@ get_papers <- function(query, params) {
                          funder = funder,
                          format = 'xml')
     pubs_metadata <- parse_response(response)
-    # FYI: The deactivation of the fill_dois() function is a hotfix
-    # to enable creation of OpenAIRE project maps
-    # TODO: root cause analysis of failure mode
-    # TODO: refactor/replace/remove the function, decision pending
-    #pubs_metadata <- fill_dois(pubs_metadata)
   },
   error = function(err){
     olog$warn(paste0("vis_id:", .GlobalEnv$VIS_ID, "publications: ", err))
@@ -193,34 +188,6 @@ parse_response <- function(response) {
   }
 }
 
-fill_dois <- function(df) {
-  missing_doi_indices <- which(is.na(df$doi))
-  titles <- df[missing_doi_indices,]$title
-  if (exists("DEBUG") && DEBUG) {
-    olog$debug(paste("Missing DOIs:", length(titles)))
-    olog$debug("Time for filling missing DOIs")
-    olog$debug(system.time(cr_works(query=queries(titles), async=TRUE)))
-  }
-  tryCatch({
-    if (length(titles) > 1) {
-      response <- cr_works(query=queries(titles))
-      candidates <- lapply(response, get_doi_candidates)
-      dois <- mapply(check_distance, titles, candidates, USE.NAMES=FALSE)
-    } else if (length(titles) == 1) {
-      response <- cr_works(flq=c('query.title'=titles))$data
-      candidate_response = response[1,]
-      dois <- check_distance(titles, candidate_response)
-    } else {
-      dois <- ""
-    }
-    df$doi[c(missing_doi_indices)] <- dois
-    }, error=function(err){
-      olog$error(paste("vis_id:", .GlobalEnv$VIS_ID, "DOI enrichment failed:", paste(err)))
-    }
-  )
-  return (df)
-}
-
 get_doi_candidates <- function(response){
   if (nrow(response) >= 1) {
     candidate = response[1,c('doi', 'title')]

server/preprocessing/other-scripts/update_contentproviders.R

@@ -0,0 +1,50 @@
+rm(list = ls())
+
+args <- commandArgs(TRUE)
+wd <- args[1]
+setwd(wd)
+
+renv::activate()
+renv::restore(lockfile = '../renv.lock')
+Sys.setlocale(category = "LC_ALL", locale = "en_US.UTF-8")
+
+library(jsonlite)
+library(rbace)
+library(logging)
+
+source('utils.R')
+if (Sys.getenv("LOGLEVEL") == "DEBUG") {
+  DEBUG <- FALSE
+} else {
+  DEBUG <- TRUE
+}
+
+if (DEBUG == TRUE) {
+  setup_logging('DEBUG')
+} else {
+  setup_logging('INFO')
+}
+
+log <- getLogger('update_contentproviders')
+
+output_path <- if (length(args) >= 2) args[2] else {
+  file.path(wd, "../../workers/common/common/contentproviders.json")
+}
+output_path <- normalizePath(output_path, mustWork = FALSE)
+
+tryCatch({
+  contentproviders <- bs_repositories("")
+  if (is.null(contentproviders) || nrow(contentproviders) == 0) {
+    stop("No content providers retrieved.")
+  }
+  triple <- list(name = "GoTriple", internal_name = "fttriple")
+  contentproviders <- rbind(contentproviders, triple)
+
+  write(toJSON(contentproviders, pretty = TRUE), file = output_path)
+  log$info(paste("contentproviders.json updated:", output_path,
+                 "—", nrow(contentproviders), "entries"))
+}, error = function(err) {
+  log$error(paste("Content provider update failed", "base",
+                  "update_contentproviders", "", err, sep = "||"))
+  quit(status = 1)
+})