Merge remote-tracking branch 'upstream/master' into exclude-date-filters-param

Author: chreman

.gitignore

@@ -41,8 +41,10 @@ server/preprocessing/other-scripts/renv
 .Rhistory
 .Rprofile
 .Rproj*
+/*.Rproj
 .Rproj.user
 
+
 # python files
 *.pyc
 *.pkl

docker-compose-end2endtest.yml

@@ -24,10 +24,9 @@ services:
       - db
       - backend
     restart: "no"
-    entrypoint: [ "pytest", '/app/workers/tests/test_end2end.py', '-s', '-rfA']
+    entrypoint: ["pytest", '/app/workers/tests/test_end2end.py', '-s', '-rfA']
     networks:
-      test:
-        ipv4_address: 172.18.0.2
+      - test
 
   backend:
     container_name: backend
@@ -37,13 +36,47 @@ services:
       dockerfile: ./Dockerfile_backend
     volumes:
       - ./server/:/var/www/html/server
+      - ./server/workers/tests/test_data/test.sqlite:/var/www/localstorage/test.sqlite
     restart: "no"
     networks:
-      test:
-        ipv4_address: 172.18.0.3
+      - test
     ports:
       - "80:80"
 
+  api:
+    build:
+      context: server
+      dockerfile: workers/api/Dockerfile
+    restart: unless-stopped
+    environment:
+      SERVICE_VERSION: "test"
+      BEHIND_PROXY: "false"
+      DEFAULT_DATABASE: "testdb"
+      FLASK_ENV: "development"
+    volumes:
+      - ./server/workers/tests/mock_app.py:/app/mock_app.py
+    command: ["python", "mock_app.py"]
+    networks: 
+      - test
+
+  persistence:
+    container_name: api
+    hostname: "test_api"
+    build:
+      context: server
+      dockerfile: workers/persistence/Dockerfile
+    restart: "no"
+    environment:
+      SERVICE_VERSION: "test"
+      BEHIND_PROXY: "false"
+      DEFAULT_DATABASE: "testdb"
+      FLASK_ENV: "development"
+    volumes:
+      - ./server/workers/tests/mock_app.py:/app/mock_app.py
+    command: ["python", "mock_app.py"]
+    networks:
+      - test
+
   db:
     container_name: test_db
     image: 'postgres:12.2-alpine'
@@ -63,16 +96,10 @@ services:
     ports:
       - "5432:5432"
     networks:
-      test:
-        ipv4_address: 172.18.0.4
+      - test
 
 volumes:
   db_data:
 
 networks:
   test:
-    driver: bridge
-    ipam:
-      config:
-        - subnet: 172.18.0.0/16
-          gateway: 172.18.0.1

examples/project_website/data/custom_clustering.json

@@ -7,7 +7,7 @@ preprocessing_dir = "/var/www/html/dev/server/preprocessing/"
 # Full path to the images directory for the client. Needs to be in the public_html/www directory. Make sure that your webserver has write access to this directory.
 images_path = "/var/www/html/dev/server/paper_preview/"
 # Host of the client visualization
-host = "localhost/"
+host = "dev-searchflow-1/"
 # Relative path to the client visualization. Needs to be in the public_html/www directory.
 vis_path = "headstart"
 # Relative path to the client REST services. Needs to be in the public_html/www directory.
@@ -17,44 +17,25 @@ services_path = "server/services/"
 api_url = "http://proxy-proxy-1/"
 # flavor of API, default: stable
 api_flavor = "dev"
-# The value that is used to determine which db to use: SQLite - 0 or PostgreSQL - 1
-shift_read_percentage = 1
 
 [snapshot]
 # Set to 1 to enable snapshot feature, 0 to disable
-snapshot_enabled = 0
+snapshot_enabled = 1
 # Absolute path to node binary
-node_path = "/path/to/node"
+node_path = "/usr/local/nvm/versions/node/v10.17.0/bin/node"
 # Absolute paths to node_modules folder, where puppeteer is installed
-nodemodules_path = "/path/to/node_modules"
+nodemodules_path = "/usr/local/nvm/versions/node/v10.17.0/lib/node_modules/"
 # Absolute path to getChartSVG.js
-getsvg_path = "/path/to/getChartSVG.js"
+getsvg_path = "/var/www/html/headstart/server/services/getChartSVG.js"
 # Absolute path to the directory, where the snapshots are stored. Webserver must have write access to this directory
-storage_path = "/path/to/storage/"
+storage_path = "/var/www/html/snapshots/"
 # PHP File responsible for rendering the bubble in a way to be snapshotted. Relative path to general host
-snapshot_php = "server/services/snapshot/headstart_snapshot.php"
+snapshot_php = "headstart/server/services/snapshot/headstart_snapshot.php"
 # snapshot_local_protocol fallback for non-server environments
 snapshot_local_protocol = "http://"
-
-[output]
-# Relative paths for offline calculation
-output_dir = "other-scripts/"
-cooc = "cooc.csv";
-metadata = "metadata.csv"
-output_scaling_clustering = "output_scaling_clustering.csv"
-output_naming = "output_naming.csv"
-unique_id = "vis_id2"
-title = "Visualization"
+# Size of thumbnail width
+snapshot_width = "1200px"
 
 [connection]
-# Full path to the sqlite datatabase file. Make sure that your webserver has write access to this file. For development purposes, duplicate headstart.sqlite in server/storage/ and rename it to a filename of your choice. Enter the path to this file here.
-sqlite_db = "/var/www/localstorage/local.sqlite"
 # PostgreSQL database
-database = "dev"
-
-[calculation]
-# Path to the RScript binary
-binary = "/usr/bin/Rscript"
-# Relative path from preprocessing_dir to the R script
-script = "other-scripts/text_similarity.R"
-mode = "bookmarks"
+database = "dev"

local_dev/config_local_headstart.ini

@@ -5,8 +5,19 @@ MAINTAINER Chris Kittel "christopher.kittel@openknowledgemaps.org"
 RUN a2enmod rewrite
 RUN apt-get update && apt-get install -y curl libsqlite3-dev php7.4-sqlite libonig-dev libxml2-dev
 RUN docker-php-ext-install pdo pdo_sqlite mbstring xml fileinfo
+# for puppeteer
+RUN apt-get install -y gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget
+ENV NVM_DIR /usr/local/nvm
+ENV NODE_VERSION 10.17.0
+RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.1/install.sh | bash \
+    && . $NVM_DIR/nvm.sh \
+    && nvm install $NODE_VERSION \
+    && nvm alias default $NODE_VERSION \
+    && nvm use default
 
-RUN sed -i 's#AllowOverride [Nn]one#AllowOverride All#' /etc/apache2/apache2.conf
-RUN mkdir /var/www/localstorage
-COPY test.sqlite /var/www/localstorage/local.sqlite
-RUN chown -R www-data:www-data /var/www/localstorage
+ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules
+ENV PATH      $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+ENV NPM_PATH  $NVM_DIR/versions/node/v$NODE_VERSION/bin/npm
+RUN $NPM_PATH install -g puppeteer@^2.0.0 --unsafe-perm=true --allow-root
+
+RUN sed -i 's#AllowOverride [Nn]one#AllowOverride All#' /etc/apache2/apache2.conf

local_dev/localstorage/local.sqlite

@@ -14,7 +14,6 @@ public function load_configs($ini_array) {
         $this->ini_array = $ini_array;
         $this->settings = $this->ini_array["general"];
         $this->database = $this->ini_array["connection"]["database"];
-        $this->WORKING_DIR = $this->ini_array["general"]["preprocessing_dir"] . $this->ini_array["output"]["output_dir"];
         $api_url = $this->ini_array["general"]["api_url"];
         $api_flavor = isset($this->ini_array["general"]["api_flavor"])
                                 ? ($this->ini_array["general"]["api_flavor"])

local_dev/searchflow-container/Dockerfile

@@ -2,118 +2,55 @@
 
 namespace headstart\persistence;
 
-
 /**
  * This class implements the PersistenceInterface and provides methods to interact with the database.
  */
 class DispatchingPersistence implements Persistence
 {
-    private Persistence $oldPersistence;
     private Persistence $newPersistence;
-    private int|float $shiftReadPercentage;
-
 
-    public function __construct(Persistence $oldPersistence, Persistence $newPersistence, $shiftReadPercentage)
+    public function __construct(Persistence $newPersistence)
     {
-        $this->shiftReadPercentage = $shiftReadPercentage;
-        $this->oldPersistence = $oldPersistence;
         $this->newPersistence = $newPersistence;
     }
 
     public function createVisualization($vis_id, $vis_title, $input_json, $query, $dirty_query, $params_json): void
     {
-        $this->oldPersistence->createVisualization($vis_id, $vis_title, $input_json, $query, $dirty_query, $params_json);
         $this->newPersistence->createVisualization($vis_id, $vis_title, $input_json, $query, $dirty_query, $params_json);
     }
 
     public function getRevision($vis_id, $rev_id): array|bool
     {
-        $randomFloat = getRandomFloat();
-
-        if (($randomFloat * 100) > ($this->shiftReadPercentage * 100)) {
-            $result = $this->oldPersistence->getRevision($vis_id, $rev_id);
-        } else {
-            $result = $this->newPersistence->getRevision($vis_id, $rev_id);
-        }
-
-        return $result;
+        return $this->newPersistence->getRevision($vis_id, $rev_id);
     }
 
     public function writeRevision($vis_id, $data): void
     {
-        $this->oldPersistence->writeRevision($vis_id, $data);
         $this->newPersistence->writeRevision($vis_id, $data);
     }
 
     public function existsVisualization($vis_id): array|bool
     {
-        $randomFloat = getRandomFloat();
-
-        if ($randomFloat * 100 > $this->shiftReadPercentage * 100) {
-            $result = $this->oldPersistence->existsVisualization($vis_id);
-        } else {
-            $result = $this->newPersistence->existsVisualization($vis_id);
-        }
-
-        return $result;
+        return $this->newPersistence->existsVisualization($vis_id);
     }
 
     public function getLastVersion($vis_id, $details, $context): array|bool
     {
-        $randomFloat = getRandomFloat();
-
-        if ($randomFloat * 100 > $this->shiftReadPercentage * 100) {
-            $result = $this->oldPersistence->getLastVersion($vis_id, $details, $context);
-        } else {
-            $result = $this->newPersistence->getLastVersion($vis_id, $details, $context);
-        }
-
-        return $result;
+        return $this->newPersistence->getLastVersion($vis_id, $details, $context);
     }
 
     public function getLatestRevisions(): array|bool
     {
-        $randomFloat = getRandomFloat();
-
-        if ($randomFloat * 100 > $this->shiftReadPercentage * 100) {
-            $result = $this->oldPersistence->getLatestRevisions();
-        } else {
-            $result = $this->newPersistence->getLatestRevisions();
-        }
-
-        return $result;
+        return $this->newPersistence->getLatestRevisions();
     }
 
     public function getContext($vis_id): array|bool
     {
-        $randomFloat = getRandomFloat();
-
-        if ($randomFloat * 100 > $this->shiftReadPercentage * 100) {
-            $result = $this->oldPersistence->getContext($vis_id);
-        } else {
-            $result = $this->newPersistence->getContext($vis_id);
-        }
-
-        return $result;
+        return $this->newPersistence->getContext($vis_id);
     }
 
     public function createID($string_array, $payload): string
     {
-        $randomFloat = getRandomFloat();
-
-        if ($randomFloat * 100 > $this->shiftReadPercentage * 100) {
-            $result = $this->oldPersistence->createID($string_array, $payload);
-        } else {
-            $result = $this->newPersistence->createID($string_array, $payload);
-        }
-
-        return $result;
+        return $this->newPersistence->createID($string_array, $payload);
     }
 }
-
-
-//    function for random float number generation from 0 to 1
-function getRandomFloat(): float
-{
-    return mt_rand() / mt_getrandmax();
-}

server/classes/headstart/library/APIClient.php

@@ -1,11 +1,6 @@
 <?php
 
 namespace headstart\persistence;
-/**
- * Description of Logger
- *
- * @author pkraker
- */
 
 /**
  * Interface Persistence

server/classes/headstart/persistence/DispatchingPersistence.php

@@ -3,9 +3,12 @@
 namespace headstart\persistence;
 use headstart\library\APIClient;
 
+require_once 'Persistence.php';
+
 /**
  * This class implements the PersistenceInterface and provides methods to interact with the database.
  */
+
 class PostgresPersistence implements Persistence
 {
 
@@ -99,4 +102,4 @@ public function createID($string_array, $payload): string
 
         throw new \Exception("Could not create ID, response code was : " . $res["httpcode"]);
     }
-}
+}