Merge pull request #842 from OpenKnowledgeMaps/dev

Local dev database insertion workaround

Author: chreman

.docker.test.env

@@ -33,6 +33,7 @@ server/workers/tests/testutils/
 local_dev/renv/*
 local_dev/dev.env
 local_dev/paper_preview
+.docker.test.env
 
 # php files
 /server/classes/headstart/vendor

.gitignore

@@ -1,13 +1,13 @@
 services:
-
   db:
-    image: 'postgres:12.2-alpine'
+    image: "postgres:12.2-alpine"
     hostname: "${POSTGRES_HOSTNAME}"
     restart: unless-stopped
     environment:
       POSTGRES_USER: "${POSTGRES_USER}"
       POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
-    command: postgres -c config_file=/etc/postgresql.conf -c hba_file=/etc/pg_hba.conf
+    command:
+      postgres -c config_file=/etc/postgresql.conf -c hba_file=/etc/pg_hba.conf
     volumes:
       - db_data:/var/lib/postgresql/data
       - ./local_dev/pg_hba.conf:/etc/pg_hba.conf
@@ -16,18 +16,26 @@ services:
       - headstart
 
   redis:
-    image: 'redis:6.0-alpine'
+    image: "redis:6.0-alpine"
     restart: unless-stopped
     hostname: "${REDIS_HOST}"
     environment:
       REDIS_HOST: "${REDIS_HOST}"
       REDIS_PORT: "${REDIS_PORT}"
-    command: ["redis-server", "/etc/redis/redis.conf", "--bind", "${REDIS_HOST}", "--port", "${REDIS_PORT}"]
+    command:
+      [
+        "redis-server",
+        "/etc/redis/redis.conf",
+        "--bind",
+        "${REDIS_HOST}",
+        "--port",
+        "${REDIS_PORT}",
+      ]
     volumes:
-        - 'redis:/var/lib/redis/data'
-        - ./local_dev/redis.conf:/etc/redis/redis.conf
+      - "redis:/var/lib/redis/data"
+      - ./local_dev/redis.conf:/etc/redis/redis.conf
     ports:
-     - "127.0.0.1:${REDIS_PORT}:${REDIS_PORT}"
+      - "127.0.0.1:${REDIS_PORT}:${REDIS_PORT}"
     networks:
       - headstart
 
@@ -80,6 +88,7 @@ services:
       - ./server/workers/persistence/src:/api
     depends_on:
       - redis
+      - db
     networks:
       - headstart
 
@@ -264,7 +273,6 @@ services:
     networks:
       - headstart
 
-
 volumes:
   redis:
   db_data:

docker-compose.yml

@@ -0,0 +1 @@
+{"author":null,"documents":"[]"}

local_dev/tools/db-insertion/BASE/81abae6d3af6e47761e2a761a47c2c11.json

@@ -0,0 +1,15 @@
+{
+  "endpoint": "http://127.0.0.1:8081/dev/persistence/createVisualization/dev",
+  "visualizations": [
+    {
+      "vis_id": "81abae6d3af6e47761e2a761a47c2c11",
+      "vis_title": "base",
+      "vis_clean_query": "digital education",
+      "vis_query": "digital education",
+      "vis_params": "{\"from\":\"1665-01-01\",\"to\":\"2025-12-03\",\"document_types\":[\"121\"],\"sorting\":\"most-relevant\",\"min_descsize\":\"300\",\"lang_id\":[\"all-lang\"]}",
+      "data_file": "81abae6d3af6e47761e2a761a47c2c11.json"
+    }
+  ]
+}
+
+

local_dev/tools/db-insertion/BASE/config.json

@@ -0,0 +1,15 @@
+{
+  "endpoint": "http://127.0.0.1:8081/dev/persistence/createVisualization/dev",
+  "visualizations": [
+    {
+      "vis_id": "9d4dc6b920d1e2cc08a741f7c56821db",
+      "vis_title": "openaire",
+      "vis_clean_query": "DP0878177",
+      "vis_query": "DP0878177",
+      "vis_params": "{\"project_id\":\"DP0878177\",\"funder\":\"ARC\",\"acronym\":\"\",\"title\":\"Understanding the impact of global environmental change on Australian forests and woodlands using rainforest boundaries and Callitris growth as bio-indicators\",\"start_date\":\"2008-01-01\",\"end_date\":\"2011-12-31\",\"special_clause\":\"false\",\"oa_mandate\":\"false\",\"organisations\":[],\"openaire_link\":\"http:\\/\\/purl.org\\/au-research\\/grants\\/arc\\/DP0878177\",\"obj_id\":\"arc_________::fe52f7d04f4139c2c80b4144c294f12d\",\"call_id\":\"\",\"funding_tree\":[null,null,\"Discovery Projects\"]}",
+      "data_file": "9d4dc6b920d1e2cc08a741f7c56821db.json"
+    }
+  ]
+}
+
+

local_dev/tools/db-insertion/OpenAIRE/9d4dc6b920d1e2cc08a741f7c56821db.json

@@ -0,0 +1,21 @@
+# About that folder
+
+This folder contains code that allows adding visualization information to the database.
+
+> [!TIP]
+> This can be useful when, for example, due to errors on the part of other services (not ours), we are unable to create a visualization, but such visualization is necessary for testing or development. To avoid stopping work and blocking the task, visualisation can be added to the database using this code.
+
+## How does it all work?
+
+To work with this functionality, it is important to understand two main entities: `insert.py` and configuration files for each service.
+
+The `insert.py` file is universal code that can write visualization information to the database for any service. To do this, you only need to specify the desired service in the `insert.py` file (or, in other words, connect the necessary configuration).
+
+Services and their configurations are stored in folders such as `./BASE/` or `./OpenAIRE/`. These folders contain two files: `<visualization_id>.json` and `config.json`: 
+- The `<visualization_id>.json` file stores visualization data that will be used to build the visualization on the client;
+- The `config.json` file stores the rest of the information that is important for creating a database record and will also be partially used on the client.
+
+To connect the required service, simply make a change in line 7 in the `insert.py` file - this is the line where the path to a configuration (variable `CONFIG_PATH`) is created. You need to replace the service name with the required one.
+
+> [!IMPORTANT]
+> The name of the service must match the name of the folder where its configurations are stored. Pay attention to the case of the characters!

local_dev/tools/db-insertion/OpenAIRE/config.json

@@ -0,0 +1,47 @@
+import json
+import requests
+
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parent
+CONFIG_PATH = BASE_DIR / "OpenAIRE" / "config.json"
+
+VISUALIZATION_DATA = []
+ENDPOINT_URL = "http://127.0.0.1:8081/dev/persistence/createVisualization/dev"
+
+
+def insert_from_config(config_path: Path) -> None:
+    """Insert visualizations described in the given config JSON file."""
+    with open(config_path, "r") as f:
+        config = json.load(f)
+
+    endpoint = config.get("endpoint", ENDPOINT_URL)
+    visualizations = config.get("visualizations", VISUALIZATION_DATA)
+
+    for vis in visualizations:
+        data_file = config_path.parent / vis["data_file"]
+        with open(data_file, "r") as df:
+            data = df.read()
+
+        payload = {
+            "vis_id": vis["vis_id"],
+            "vis_title": vis["vis_title"],
+            "vis_clean_query": vis["vis_clean_query"],
+            "vis_query": vis["vis_query"],
+            "vis_params": vis["vis_params"],
+            "data": data,
+        }
+
+        res = requests.post(endpoint, json=payload)
+        print(f"Inserted {vis['vis_id']}: {res.status_code}")
+        print(res.text)
+
+
+def main() -> None:
+    insert_from_config(CONFIG_PATH)
+
+
+if __name__ == "__main__":
+    main()
+
+

local_dev/tools/db-insertion/README.md

@@ -69,6 +69,8 @@ listen_addresses = 'localhost,headstart_db_1,headstart-db-1'
 
 Run tests
 
+Create a `.docker.test.env` file first, then run the following command:
+
 ```
 docker compose -f docker-compose-integration-tests.yml --env-file .docker.test.env run integration_tests
 ```