Merge pull request #644 from OpenKnowledgeMaps/content-provider

Content provider in context line

Author: Konstiman

docker-compose-dataworker.yml

@@ -12,7 +12,6 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
     restart: always
     volumes:
       - /opt/local/renv/cache:/renv/cache

docker-compose.yml

@@ -11,10 +11,9 @@ services:
       POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
     command: postgres -c config_file=/etc/postgresql.conf -c hba_file=/etc/pg_hba.conf
     volumes:
-      # - ~/data/OKMaps/${COMPOSE_PROJECT_NAME}/postgresql/data:/var/lib/postgresql/data
       - db_data:/var/lib/postgresql/data
-      - ./server/workers/pg_hba.conf:/etc/pg_hba.conf
-      - ./server/workers/postgresql.conf:/etc/postgresql.conf
+      - ./pg_hba.conf:/etc/pg_hba.conf
+      - ./postgresql.conf:/etc/postgresql.conf
     networks: 
       - headstart
 
@@ -35,8 +34,7 @@ services:
     command: ["redis-server", "/etc/redis/redis.conf", "--bind", "${REDIS_HOST}", "--appendonly", "yes", "--port", "${REDIS_PORT}"]
     volumes:
         - 'redis:/var/lib/redis/data'
-        - ./server/workers/redis.conf:/etc/redis/redis.conf
-        - ./server/workers/certs:/etc/certs
+        - ./redis.conf:/etc/redis/redis.conf
     ports:
      - "127.0.0.1:${REDIS_PORT}:6379"
     restart: always
@@ -52,12 +50,13 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
       REDIS_DB: "${REDIS_DB}"
-      REDIS_SSL: "${REDIS_SSL}"
       BEHIND_PROXY: "${BEHIND_PROXY}"
       DEFAULT_DATABASE: "${DEFAULT_DATABASE}"
       DATABASES: "${DATABASES}"
       FLASK_ENV: "${FLASK_ENV}"
     command: ["gunicorn",  "--workers", "10", "--threads", "2", "-b", "0.0.0.0:${API_PORT}", "app:app", "--timeout", "300"]
+    volumes:
+      - ./api_cache:/var/api_cache
     depends_on:
       - redis
     networks: 
@@ -82,15 +81,12 @@ services:
 
   triple:
     image: triple:${SERVICE_VERSION}
-    env_file:
-      - server/workers/triple/triple.env
     environment:
       SERVICE_VERSION: "${SERVICE_VERSION}"
       REDIS_HOST: "${REDIS_HOST}"
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
       LOGLEVEL: "${LOGLEVEL}"
       TRIPLE_USER: "${TRIPLE_USER}"
       TRIPLE_PASS: "${TRIPLE_PASS}"
@@ -107,15 +103,12 @@ services:
 
   gsheets:
     image: gsheets:${SERVICE_VERSION}
-    env_file:
-      - server/workers/gsheets/gsheets.env
     environment:
       SERVICE_VERSION: "${SERVICE_VERSION}"
       REDIS_HOST: "${REDIS_HOST}"
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
       LOGLEVEL: "${LOGLEVEL}"
     restart: always
     depends_on:
@@ -125,16 +118,19 @@ services:
 
   dataprocessing:
     image: dataprocessing:${SERVICE_VERSION}
-    env_file:
-    - server/workers/dataprocessing/dataprocessing.env
     environment:
       SERVICE_VERSION: "${SERVICE_VERSION}"
       REDIS_HOST: "${REDIS_HOST}"
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
       LOGLEVEL: "${LOGLEVEL}"
+      LOGFILE: "${LOGFILE}"
+      RENV_VERSION: 0.14.0-5
+      CRAN_REPOS: https://cran.wu.ac.at
+      LC_ALL: "en_US.UTF-8"
+      LANG: "en_US.UTF-8"
+      RENV_PATHS_CACHE: /renv/cache
     restart: always
     volumes:
       - /opt/local/renv/cache:/renv/cache
@@ -152,7 +148,6 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
       LOGLEVEL: "${LOGLEVEL}"
       LOGFILE: "/var/log/headstart/headstart.log"
       RENV_VERSION: 0.14.0-5
@@ -178,15 +173,13 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
       LOGLEVEL: "${LOGLEVEL}"
-      LOGFILE: "/var/log/headstart/headstart.log"
+      LOGFILE: "${LOGFILE}"
       RENV_VERSION: 0.14.0-5
       CRAN_REPOS: https://cran.wu.ac.at
       LC_ALL: "en_US.UTF-8"
       LANG: "en_US.UTF-8"
       RENV_PATHS_CACHE: /renv/cache
-      PYTHONIOENCODING: "utf-8"
     restart: always
     volumes:
       - /opt/local/renv/cache:/renv/cache
@@ -204,15 +197,13 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
       LOGLEVEL: "${LOGLEVEL}"
-      LOGFILE: "/var/log/headstart/headstart.log"
+      LOGFILE: "${LOGFILE}"
       RENV_VERSION: 0.14.0-5
       CRAN_REPOS: https://cran.wu.ac.at
       LC_ALL: "en_US.UTF-8"
       LANG: "en_US.UTF-8"
       RENV_PATHS_CACHE: /renv/cache
-      PYTHONIOENCODING: "utf-8"
     restart: always
     volumes:
       - /opt/local/renv/cache:/renv/cache
@@ -226,6 +217,8 @@ volumes:
   redis:
   db_data:
     driver: local
+  api_cache:
+    driver: local
 
 networks:
   headstart:

server/preprocessing/other-scripts/run_base_contentproviders.R

@@ -0,0 +1,45 @@
+rm(list = ls())
+
+args <- commandArgs(TRUE)
+wd <- args[1]
+setwd(wd) #Don't forget to set your working directory
+
+renv::activate()
+renv::restore(lockfile = '../renv.lock')
+Sys.setlocale(category="LC_ALL", locale = "en_US.UTF-8")
+
+library(jsonlite)
+library(rbace)
+library(logging)
+
+source('utils.R')
+if (Sys.getenv("LOGLEVEL") == "DEBUG") {
+  DEBUG <- FALSE
+} else {
+  DEBUG <- TRUE
+}
+
+if (DEBUG==TRUE){
+  setup_logging('DEBUG')
+} else {
+  setup_logging('INFO')
+}
+
+
+log <- getLogger('base_repos')
+
+tryCatch({
+  contentproviders <- bs_repositories("")
+}, error=function(err){
+  log$error(paste("Contentprovider failed", "base", "retrieve_contentproviders", "", err, sep="||"))
+  failed <- list()
+  failed$reason <- list(err)
+  failed$status <- 'error'
+})
+
+
+if (exists('contentproviders')) {
+  print(toJSON(contentproviders))
+} else {
+  print(toJSON(failed))
+}

server/services/search.php

@@ -109,6 +109,15 @@ function search($service_integration, $dirty_query
     }
     $unique_id = ($precomputed_id === null)?($unique_id):($precomputed_id);
     $post_params["vis_id"] = $unique_id;
+    if (array_key_exists("repo", $post_params)) {
+      $payload = json_encode(array("repo" => $post_params["repo"]));
+      $res = $apiclient->call_api($endpoint . "/contentproviders", $payload);
+      $res = $res["result"];
+      $res = json_decode($res, true);
+      $repo_name = $res["repo_name"];
+      $post_params["repo_name"] = $repo_name;
+      $param_types[] = "repo_name";
+    }
     $params_json = packParamsJSON($param_types, $post_params);
 
     if($retrieve_cached_map) {

server/workers/api/src/apis/base.py

@@ -39,6 +39,22 @@
                                                       description='raw results from ElasticSearch')})
 
 
+def get_or_create_contentprovider_lookup():
+    try:
+        k = str(uuid.uuid4())
+        d = {"id": k, "params": {},"endpoint": "contentproviders"}
+        base_ns.logger.debug(d)
+        redis_store.rpush("base", json.dumps(d))
+        result = get_key(redis_store, k)
+        df = pd.DataFrame(json.loads(result["contentproviders"]))
+        df.set_index("internal_name", inplace=True)
+        cp_dict = df.name.to_dict()
+        return cp_dict
+    except Exception as e:
+        base_ns.logger.error(e)
+   
+contentprovider_lookup = get_or_create_contentprovider_lookup()
+
 @base_ns.route('/search')
 class Search(Resource):
     @base_ns.doc(responses={200: 'OK',
@@ -53,6 +69,9 @@ def post(self):
         if "optradio" in params:
             del params["optradio"]
         errors = search_param_schema.validate(params, partial=True)
+        if "repo" in params:
+            repo_name = contentprovider_lookup.get(params["repo"])
+            params["repo_name"] = repo_name
         params["limit"] = 120
         params["list_size"] = 100
         base_ns.logger.debug(errors)
@@ -85,10 +104,40 @@ def post(self):
             base_ns.logger.error(e)
             abort(500, "Problem encountered, check logs.")
 
+@base_ns.route('/contentproviders')
+class ContentProvider(Resource):
+    @base_ns.doc(responses={200: 'OK',
+                              400: 'Invalid search parameters'})
+    @base_ns.produces(["application/json"])
+    def post(self):
+        """
+        params: can be empty
+        content_provider: BASE internal name, e.g. "ftunivlausanne"
+
+        returns: json
+        {"contentprovider_short": "ftunivlausanne",
+         "repo_name": "Université de Lausanne (UNIL): Serval - Serveur académique lausannois"}
+        """
+        params = request.get_json()
+        base_ns.logger.debug(params)
+        if not params:
+            result = contentprovider_lookup
+        else:
+            result = {"repo_name": contentprovider_lookup.get(params["repo"])}
+        try:
+            headers = {}
+            headers["Content-Type"] = "application/json"
+            return make_response(result,
+                                 200,
+                                 headers)
+        except Exception as e:
+            base_ns.logger.error(e)
+            abort(500, "Problem encountered, check logs.")
+        
 
 
 @base_ns.route('/service_version')
 class ServiceVersion(Resource):
     def get(self):
         result = {"service_version": os.getenv("SERVICE_VERSION")}
-        return make_response(result, 200, {"Content-Type": "application/json"})
+        return make_response(result, 200, {"Content-Type": "application/json"})

server/workers/api/src/apis/request_validators.py

@@ -21,6 +21,9 @@ class SearchParamSchema(Schema):
     unique_id = fields.Str()
     raw = fields.Boolean()
     sg_method = fields.Str()
+    repo = fields.Str()
+    repo_name = fields.Str()
+    coll = fields.Str()
     vis_id = fields.Str(default=None)
     optradio = fields.Str()
     service = fields.Str()

server/workers/api/src/apis/utils.py

@@ -11,8 +11,7 @@
     "port": os.getenv("REDIS_PORT"),
     "db": os.getenv("REDIS_DB"),
     "password": os.getenv("REDIS_PASSWORD"),
-    "client_name": "api",
-    "ssl": True if os.getenv("REDIS_SSL") == "true" else False
+    "client_name": "api"
 }
 redis_store = redis.StrictRedis(**redis_config)
 

server/workers/api/src/app.py

@@ -1,9 +1,10 @@
 import os
 import sys
-from flask import Flask, redirect, url_for
+from flask import Flask
 from flask_restx import Api
 from flask_cors import CORS
 from werkzeug.middleware.proxy_fix import ProxyFix
+import logging
 
 from apis.triple import triple_ns
 from apis.gsheets import gsheets_ns
@@ -13,19 +14,45 @@
 from apis.create_vis import vis_ns
 from apis.export import export_ns
 
-from utils.monkeypatches import ReverseProxied, __schema__, specs_url, _register_apidoc
-import logging
 
+class ReverseProxied(object):
+    '''Wrap the application in this middleware and configure the
+    front-end server to add these headers, to let you quietly bind
+    this to a URL other than / and to an HTTP scheme that is
+    different than what is used locally.
+
+    location /myprefix {
+        proxy_pass http://192.168.0.1:5001;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Scheme $scheme;
+        proxy_set_header X-Script-Name /myprefix;
+        }
+
+    :param app: the WSGI application
+    '''
+    def __init__(self, app):
+        self.app = app
+
+    def __call__(self, environ, start_response):
+        script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
+        if script_name:
+            environ['SCRIPT_NAME'] = script_name
+            path_info = environ['PATH_INFO']
+            if path_info.startswith(script_name):
+                environ['PATH_INFO'] = path_info[len(script_name):]
+
+        scheme = environ.get('HTTP_X_SCHEME', '')
+        if scheme:
+            environ['wsgi.url_scheme'] = scheme
+        return self.app(environ, start_response)
 
 def api_patches(app):
-    Api._register_apidoc = _register_apidoc
-    Api.__schema__ = __schema__
-    Api.specs_url = specs_url
 
     api_fixed = Api(
         app,
         title="Head Start API",
-        description="Head Start API demo",
+        description="Head Start API",
         version="0.1",
         prefix='/api',
         doc="/docs")