Merge pull request #867 from OpenKnowledgeMaps/base-contentproviders-cacheupdate

Base contentproviders cacheupdate

Author: chreman

server/preprocessing/other-scripts/update_contentproviders.R

@@ -0,0 +1,50 @@
+rm(list = ls())
+
+args <- commandArgs(TRUE)
+wd <- args[1]
+setwd(wd)
+
+renv::activate()
+renv::restore(lockfile = '../renv.lock')
+Sys.setlocale(category = "LC_ALL", locale = "en_US.UTF-8")
+
+library(jsonlite)
+library(rbace)
+library(logging)
+
+source('utils.R')
+if (Sys.getenv("LOGLEVEL") == "DEBUG") {
+  DEBUG <- FALSE
+} else {
+  DEBUG <- TRUE
+}
+
+if (DEBUG == TRUE) {
+  setup_logging('DEBUG')
+} else {
+  setup_logging('INFO')
+}
+
+log <- getLogger('update_contentproviders')
+
+output_path <- if (length(args) >= 2) args[2] else {
+  file.path(wd, "../../workers/common/common/contentproviders.json")
+}
+output_path <- normalizePath(output_path, mustWork = FALSE)
+
+tryCatch({
+  contentproviders <- bs_repositories("")
+  if (is.null(contentproviders) || nrow(contentproviders) == 0) {
+    stop("No content providers retrieved.")
+  }
+  triple <- list(name = "GoTriple", internal_name = "fttriple")
+  contentproviders <- rbind(contentproviders, triple)
+
+  write(toJSON(contentproviders, pretty = TRUE), file = output_path)
+  log$info(paste("contentproviders.json updated:", output_path,
+                 "—", nrow(contentproviders), "entries"))
+}, error = function(err) {
+  log$error(paste("Content provider update failed", "base",
+                  "update_contentproviders", "", err, sep = "||"))
+  quit(status = 1)
+})

server/workers/build_docker_images.sh

@@ -3,6 +3,26 @@
 # Defines the script directory
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
+CONTENTPROVIDERS_FILE="$SCRIPT_DIR/common/common/contentproviders.json"
+
+# Update contentproviders.json cache from the running dev-base-1 container
+echo ""
+echo "Updating contentproviders.json cache..."
+echo ""
+docker exec dev-base-1 Rscript /headstart/other-scripts/update_contentproviders.R \
+  /headstart/other-scripts \
+  /common/contentproviders.json
+docker cp dev-base-1:/common/contentproviders.json "$CONTENTPROVIDERS_FILE"
+
+# Commit if the file changed
+cd "$SCRIPT_DIR/../.." && git diff --quiet "$CONTENTPROVIDERS_FILE"
+if [ $? -ne 0 ]; then
+  echo "contentproviders.json changed, committing..."
+  git add "$CONTENTPROVIDERS_FILE"
+  git commit -m "update of contentprovider.json cache"
+fi
+cd "$SCRIPT_DIR"
+
 # Define the list of services
 services=("api" "persistence" "dataprocessing" "base" "pubmed" "openaire" "orcid" "metrics")