api cleanup; contenprovider wip

Author: chreman

docker-compose.yml

@@ -11,10 +11,9 @@ services:
       POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
     command: postgres -c config_file=/etc/postgresql.conf -c hba_file=/etc/pg_hba.conf
     volumes:
-      # - ~/data/OKMaps/${COMPOSE_PROJECT_NAME}/postgresql/data:/var/lib/postgresql/data
       - db_data:/var/lib/postgresql/data
-      - ./server/workers/pg_hba.conf:/etc/pg_hba.conf
-      - ./server/workers/postgresql.conf:/etc/postgresql.conf
+      - ./pg_hba.conf:/etc/pg_hba.conf
+      - ./postgresql.conf:/etc/postgresql.conf
     networks: 
       - headstart
 
@@ -35,8 +34,7 @@ services:
     command: ["redis-server", "/etc/redis/redis.conf", "--bind", "${REDIS_HOST}", "--appendonly", "yes", "--port", "${REDIS_PORT}"]
     volumes:
         - 'redis:/var/lib/redis/data'
-        - ./server/workers/redis.conf:/etc/redis/redis.conf
-        - ./server/workers/certs:/etc/certs
+        - ./redis.conf:/etc/redis/redis.conf
     ports:
      - "127.0.0.1:${REDIS_PORT}:6379"
     restart: always
@@ -57,6 +55,8 @@ services:
       DATABASES: "${DATABASES}"
       FLASK_ENV: "${FLASK_ENV}"
     command: ["gunicorn",  "--workers", "10", "--threads", "2", "-b", "0.0.0.0:${API_PORT}", "app:app", "--timeout", "300"]
+    volumes:
+      - ./api_cache:/var/api_cache
     depends_on:
       - redis
     networks: 
@@ -81,8 +81,6 @@ services:
 
   triple:
     image: triple:${SERVICE_VERSION}
-    env_file:
-      - server/workers/triple/triple.env
     environment:
       SERVICE_VERSION: "${SERVICE_VERSION}"
       REDIS_HOST: "${REDIS_HOST}"
@@ -105,8 +103,6 @@ services:
 
   gsheets:
     image: gsheets:${SERVICE_VERSION}
-    env_file:
-      - server/workers/gsheets/gsheets.env
     environment:
       SERVICE_VERSION: "${SERVICE_VERSION}"
       REDIS_HOST: "${REDIS_HOST}"
@@ -122,15 +118,19 @@ services:
 
   dataprocessing:
     image: dataprocessing:${SERVICE_VERSION}
-    env_file:
-    - server/workers/dataprocessing/dataprocessing.env
     environment:
       SERVICE_VERSION: "${SERVICE_VERSION}"
       REDIS_HOST: "${REDIS_HOST}"
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
       LOGLEVEL: "${LOGLEVEL}"
+      LOGFILE: "${LOGFILE}"
+      RENV_VERSION: 0.14.0-5
+      CRAN_REPOS: https://cran.wu.ac.at
+      LC_ALL: "en_US.UTF-8"
+      LANG: "en_US.UTF-8"
+      RENV_PATHS_CACHE: /renv/cache
     restart: always
     volumes:
       - /opt/local/renv/cache:/renv/cache
@@ -174,13 +174,12 @@ services:
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
       LOGLEVEL: "${LOGLEVEL}"
-      LOGFILE: "/var/log/headstart/headstart.log"
+      LOGFILE: "${LOGFILE}"
       RENV_VERSION: 0.14.0-5
       CRAN_REPOS: https://cran.wu.ac.at
       LC_ALL: "en_US.UTF-8"
       LANG: "en_US.UTF-8"
       RENV_PATHS_CACHE: /renv/cache
-      PYTHONIOENCODING: "utf-8"
     restart: always
     volumes:
       - /opt/local/renv/cache:/renv/cache
@@ -199,13 +198,12 @@ services:
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
       LOGLEVEL: "${LOGLEVEL}"
-      LOGFILE: "/var/log/headstart/headstart.log"
+      LOGFILE: "${LOGFILE}"
       RENV_VERSION: 0.14.0-5
       CRAN_REPOS: https://cran.wu.ac.at
       LC_ALL: "en_US.UTF-8"
       LANG: "en_US.UTF-8"
       RENV_PATHS_CACHE: /renv/cache
-      PYTHONIOENCODING: "utf-8"
     restart: always
     volumes:
       - /opt/local/renv/cache:/renv/cache
@@ -219,6 +217,8 @@ volumes:
   redis:
   db_data:
     driver: local
+  api_cache:
+    driver: local
 
 networks:
   headstart:

server/workers/api/src/apis/base.py

@@ -85,10 +85,57 @@ def post(self):
             base_ns.logger.error(e)
             abort(500, "Problem encountered, check logs.")
 
+def get_or_create_contentprovider_lookup():
+    try:
+        k = str(uuid.uuid4())
+        d = {"id": k, "params": {},
+                "endpoint": "contentproviders"}
+        base_ns.logger.debug(d)
+        redis_store.rpush("base", json.dumps(d))
+        result = get_key(redis_store, k)
+        df = pd.DataFrame(json.loads(result))
+        df.set_index("internal_name", inplace=True)
+        cp_dict = df.name.to_dict()
+        return cp_dict
+    except Exception as e:
+        base_ns.logger.error(e)
+   
+contentprovider_lookup = get_or_create_contentprovider_lookup()
+base_ns.logger.debug(len(contentprovider_lookup))
+
+@base_ns.route('contentprovider')
+class ContentProvider(Resource):
+    @base_ns.doc(responses={200: 'OK',
+                              400: 'Invalid search parameters'})
+    @base_ns.produces(["application/json"])
+    def post(self):
+        """
+        params: can be empty
+        content_provider: BASE internal name, e.g. "fthsaugsburg"
+
+        returns: json
+        {"contentprovider_short": "fthsaugsburg",
+         "contentprovider_long": "OPUS - Publikationsserver der Hochschule Augsburg"}
+        """
+        params = request.get_json()
+        if not params:
+            result = contentprovider_lookup
+        else:
+            result = contentprovider_lookup.get(params["repo"])
+        try:
+            headers = {}
+            headers["Content-Type"] = "application/json"
+            return make_response(result,
+                                 200,
+                                 headers)
+        except Exception as e:
+            base_ns.logger.error(e)
+            abort(500, "Problem encountered, check logs.")
+        
 
 
 @base_ns.route('/service_version')
 class ServiceVersion(Resource):
     def get(self):
         result = {"service_version": os.getenv("SERVICE_VERSION")}
-        return make_response(result, 200, {"Content-Type": "application/json"})
+        return make_response(result, 200, {"Content-Type": "application/json"})

server/workers/api/src/app.py

@@ -1,9 +1,10 @@
 import os
 import sys
-from flask import Flask, redirect, url_for
+from flask import Flask
 from flask_restx import Api
 from flask_cors import CORS
 from werkzeug.middleware.proxy_fix import ProxyFix
+import logging
 
 from apis.triple import triple_ns
 from apis.gsheets import gsheets_ns
@@ -12,19 +13,45 @@
 from apis.openaire import openaire_ns
 from apis.create_vis import vis_ns
 
-from utils.monkeypatches import ReverseProxied, __schema__, specs_url, _register_apidoc
-import logging
 
+class ReverseProxied(object):
+    '''Wrap the application in this middleware and configure the
+    front-end server to add these headers, to let you quietly bind
+    this to a URL other than / and to an HTTP scheme that is
+    different than what is used locally.
+
+    location /myprefix {
+        proxy_pass http://192.168.0.1:5001;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Scheme $scheme;
+        proxy_set_header X-Script-Name /myprefix;
+        }
+
+    :param app: the WSGI application
+    '''
+    def __init__(self, app):
+        self.app = app
+
+    def __call__(self, environ, start_response):
+        script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
+        if script_name:
+            environ['SCRIPT_NAME'] = script_name
+            path_info = environ['PATH_INFO']
+            if path_info.startswith(script_name):
+                environ['PATH_INFO'] = path_info[len(script_name):]
+
+        scheme = environ.get('HTTP_X_SCHEME', '')
+        if scheme:
+            environ['wsgi.url_scheme'] = scheme
+        return self.app(environ, start_response)
 
 def api_patches(app):
-    Api._register_apidoc = _register_apidoc
-    Api.__schema__ = __schema__
-    Api.specs_url = specs_url
 
     api_fixed = Api(
         app,
         title="Head Start API",
-        description="Head Start API demo",
+        description="Head Start API",
         version="0.1",
         prefix='/api',
         doc="/docs")