diff --git a/CONTEXT.md b/CONTEXT.md index e7d726e..2309652 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -21,7 +21,7 @@ Both approaches are planned and partially implemented. The SDK path is fully fun |------|-------------|---------| | Knowledge Engine | KE | The distributed network/platform that enables knowledge exchange between applications | | Knowledge Directory | KD | Central service that matches and routes knowledge interactions between Smart Connectors | -| Smart Connector | SC | A Java runtime process (usually in a container) that acts as a broker between a KB and the KE network; exposes a REST API | +| Smart Connector | SC | Central service where participants register and that participants query for other participants in the network | | Knowledge Base | KB | A user's application that holds or requests knowledge; connects to the KE network via a Smart Connector | | Knowledge Interaction | KI | A declared intent to exchange knowledge — either to provide it (ANSWER/REACT) or request it (ASK/POST) | | Binding Set | BS | A list of dictionaries mapping SPARQL variable names to RDF N3-encoded values | @@ -93,9 +93,6 @@ examples/ settings.yaml # Example YAML config for all four KI types shared.py # Example logging helper compose.yaml # Docker Compose for running two SC instances for examples/testing - legacy/ # ← IGNORE: pre-overhaul examples, do not modify - -src/legacy/ # ← IGNORE: pre-overhaul implementation, do not modify tests/ test_ask_and_post.py @@ -397,7 +394,7 @@ uv run pytest - **Python**: ≥ 3.13 - **Package manager**: `uv` (see `uv.lock`) -- **Linter/formatter**: `ruff` (configured in `pyproject.toml`, legacy dirs excluded) +- **Linter/formatter**: `ruff` (configured in `pyproject.toml`) - **Build system**: `setuptools` ```bash @@ -411,12 +408,7 @@ uv run ruff format . # format ## Legacy Code -**Do not modify or reference code in these directories:** - -- `src/legacy/` — the pre-overhaul mapper implementation (config-file-driven SQL/SPARQL mapper) -- `examples/legacy/` — examples for the legacy implementation - -These are excluded from linting (`ruff`) and are kept for historical reference only. The `mapper-legacy` git tag marks the last legacy release. +The pre-overhaul, config-file-driven mapper implementation has been removed from the repository. The `mapper-legacy` git tag marks the last legacy release and preserves the full history. --- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3467a7a..7de24e1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,7 +96,3 @@ uv publish - `tests/` — unit and integration tests - `examples/` — runnable examples demonstrating features - `docs/` — documentation assets (architecture diagrams) - -## Legacy Code - -The directories `src/knowledge_mapper/legacy/` and `examples/legacy/` contain pre-overhaul code kept for historical reference. **Do not modify these.** diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c136e8c..0000000 --- a/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM python:3.11.1 AS builder - -WORKDIR /usr/src/app/ - -RUN pip install --upgrade pip - -# Install requirements in the Python environment. -COPY ./dev-requirements.txt . -RUN pip install -r dev-requirements.txt - -COPY ./knowledge_mapper ./knowledge_mapper -COPY ./setup.py . -COPY ./MANIFEST.in . -COPY ./README.md . - -RUN python setup.py sdist bdist_wheel - -FROM python:3.11.1-alpine - -WORKDIR /usr/src/app/ - -COPY --from=builder /usr/src/app/dist/*.whl . -RUN ls -RUN pip install ./*.whl - -COPY ./examples/sparql-mapper/config.jsonc ./conf/config.jsonc - -ENTRYPOINT [ "python", "-m", "knowledge_mapper", "./conf/config.jsonc" ] diff --git a/examples/legacy/README.md b/examples/legacy/README.md deleted file mode 100644 index ed7ba3a..0000000 --- a/examples/legacy/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Knowledge Mapper examples - -This is a docker-compose project that demonstrates 3 ways in which you can configure a Knowledge Mapper: - -- __`sql-mapper`__: This service maps data from the database running in the `mariadb` service. The `mariadb` service has a table with trees, containing their heights and names. The Knowledge Mapper's [configuration](./sql-mapper/config.jsonc) specifies how the data can be queried (with SQL) from the table and translated into terms of an example ontology. -- __`sparql-mapper`__: This service maps data from the triple store in the `fuseki` service. - The `fuseki` service contains RDF data about trees. The Knowledge Mapper's [configuration](./sparql-mapper/config.jsonc) specifies how to connect to the triple store (with SPARQL) and what data is in there. -- __`custom-mapper`__: This service shows that it is also possible to plug in your own code if you need more flexibility in your Knowledge Mapper. You might want to use this if you want to disclose data from a web API. In this case, it simply returns knowledge about a maple tree. - -Apart from these three knowledge mappers, the project also contains the following data sources (mentioned above): - -- `sql-db`: An SQL database that contains data about trees. See [this file](./sql-db/seed_data/0-schema.sql) for the schema and [this file](./sql-db/seed_data/1-data.sql) for the seed data. -- `fuseki`: A triple store that is approachable with SPARQL. The data content is in [this file](./fuseki/data/data.ttl) - -Then there is a service `tree-printer`, which registers itself as a knowledge base and asks for knowledge about the trees. When receiving this, it prints it. - -Last but not least is the __`tke-runtime`__ service, which takes care of all interoperability. - -## Running the example - -To run the example, do the following: - -```bash -# Make sure older containers (if any) are removed, so we start fresh. -docker-compose down - -# Start all services. -docker-compose up -d - -# Look at the logs of tree-printer so we can see the trees. -docker-compose logs -f tree-printer - -# Shut everything down again. -docker-compose down -``` - -You will see that the `tree-printer` receives knowledge about trees coming from the 3 different Knowledge Mappers. -This shows that Knowledge Mappers can be used with relative ease to disclose different data sources into a unified model. - -Note: to receive all 5 expected trees you need to give the `tree-printer` access to the data from `dynamic-auth`. You can use `adminer` to add the knowledge base id `https://example.org/tree-printer` and knowledge interaction id `https://example.org/a-custom-knowledge-base-with-dyn-auth/interaction/answer-trees-with-heights-and-names` to the database by going to the following URL: `http://localhost:8080/?server=auth-db&username=user&db=knowledge_mapper_db&edit=policies`. diff --git a/examples/legacy/auth-db/seed_data/0-schema.sql b/examples/legacy/auth-db/seed_data/0-schema.sql deleted file mode 100644 index 33733b2..0000000 --- a/examples/legacy/auth-db/seed_data/0-schema.sql +++ /dev/null @@ -1,25 +0,0 @@ -USE knowledge_mapper_db; - -CREATE TABLE knowledge_interactions ( - id VARCHAR(512) NOT NULL, - name VARCHAR(512), - PRIMARY KEY(id) -); - -CREATE TABLE policies ( - id INT NOT NULL AUTO_INCREMENT, - knowledge_interaction_id VARCHAR(512), - knowledge_base_id VARCHAR(512), - PRIMARY KEY(id), - FOREIGN KEY (knowledge_interaction_id) REFERENCES knowledge_interactions(id) -); - -CREATE TABLE access_log ( - id INT NOT NULL AUTO_INCREMENT, - knowledge_interaction_id VARCHAR(512) NOT NULL, - knowledge_base_id VARCHAR(512) NOT NULL, - access_datetime DATETIME NOT NULL, - success BOOLEAN NOT NULL, - PRIMARY KEY (id), - FOREIGN KEY (knowledge_interaction_id) REFERENCES knowledge_interactions(id) -); diff --git a/examples/legacy/custom-mapper/Dockerfile b/examples/legacy/custom-mapper/Dockerfile deleted file mode 100644 index 2e39571..0000000 --- a/examples/legacy/custom-mapper/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.11.1-alpine - -WORKDIR /usr/src/app/ - -RUN pip install --upgrade pip -COPY ./requirements.txt . -RUN pip install -r requirements.txt - -# NOTE: In a real mapper you shouldn't copy the folder, but can simply add it to `requirements.txt` -COPY ./knowledge_mapper ./knowledge_mapper - -# Place the configuration in the Docker image -COPY ./examples/custom-mapper/config.jsonc . - -COPY ./examples/custom-mapper/example.py ./your_package/ - -ENTRYPOINT [ "python", "-m", "knowledge_mapper", "./config.jsonc" ] diff --git a/examples/legacy/custom-mapper/config.jsonc b/examples/legacy/custom-mapper/config.jsonc deleted file mode 100644 index eda2e3f..0000000 --- a/examples/legacy/custom-mapper/config.jsonc +++ /dev/null @@ -1,32 +0,0 @@ -{ - // The endpoint where a knowledge engine is available. - "knowledge_engine_endpoint": "http://tke-runtime:8280/rest", - "knowledge_base": { - // An URL representing the identity of this knowledge base - "id": "https://example.org/a-custom-knowledge-base", - // A name for this knowledge base - "name": "Some knowledge base", - // A description for this knowledge base - "description": "This is just an example." - }, - - // This property tells the Knowledge Mapper that we want to use a custom - // Python class as the data source. - "plugin": { - "class": "your_package.example.ExampleDataSource", - "args": ["this is an argument that ends up in ExampleDataSource's __init__"] - }, - - // Several knowledge interaction definitions can be placed here. - "knowledge_interactions": [ - { - // The type of this knowledge interaction. If we have knowledge - // available that is requestable, the type should be "answer". - "type": "answer", - // The graph pattern that expresses the 'shape' of our knowledge. - "pattern": "?tree ?height . ?tree ?name .", - // An optional name of this knowledge interaction. - "name": "answer-trees-with-heights-and-names" - } - ] -} \ No newline at end of file diff --git a/examples/legacy/custom-mapper/example.py b/examples/legacy/custom-mapper/example.py deleted file mode 100644 index 3c18eeb..0000000 --- a/examples/legacy/custom-mapper/example.py +++ /dev/null @@ -1,18 +0,0 @@ -from src.data_source import DataSource - - -class ExampleDataSource(DataSource): - def __init__(self, argument): - print(argument) - - def test(self): - print("testing example") - - def handle(self, ki, binding_set, requesting_kb): - result_bindings = [] - binding = dict() - binding["tree"] = "" - binding["height"] = '"44"^^' - binding["name"] = '"Maple"' - result_bindings.append(binding) - return result_bindings diff --git a/examples/legacy/docker-compose.yml b/examples/legacy/docker-compose.yml deleted file mode 100644 index c0f1bd1..0000000 --- a/examples/legacy/docker-compose.yml +++ /dev/null @@ -1,96 +0,0 @@ -services: - # This service is the TNO Knowledge Engine runtime where all interoperability is interoperated. - tke-runtime: - image: ci.tno.nl/tke/knowledge-engine/smart-connector-rest-dist:1.1.3 - ports: - - 8280:8280 - - # Data source 1: A triple store with a SPARQL endpoint. - triple-store: - image: oxigraph/oxigraph:v0.3.14 - platform: linux/amd64 - ports: - - 7878:7878 - volumes: - - ./triple-store/data.ttl:/seed-data/data.ttl - - ./triple-store/start.sh:/start.sh - entrypoint: /start.sh - - # Knowledge mapper for the SPARQL endpoint - sparql-mapper: - build: - context: .. - dockerfile: ./examples/sparql-mapper/Dockerfile - environment: - SPARQL_USERNAME: user - SPARQL_PASSWORD: password - - # Data source 2: An SQL database - sql-db: - image: mariadb:10.5 - ports: - - 3306:3306 - volumes: - - ./sql-db/seed_data:/docker-entrypoint-initdb.d/:ro - environment: - MARIADB_DATABASE: treedb - MARIADB_USER: user - MARIADB_PASSWORD: password - MARIADB_RANDOM_ROOT_PASSWORD: 1 - - # Knowledge mapper for the SQL database - sql-mapper: - build: - context: .. - dockerfile: ./examples/sql-mapper/Dockerfile - - # This is a custom knowledge mapper without an underlying data source that - # uses a Python function to generate its data. You can imagine that it is - # possible to disclose sources like REST APIs this way, but it requires some - # coding. - custom-mapper: - build: - context: .. - dockerfile: ./examples/custom-mapper/Dockerfile - - # This is a simple Python application that uses the TkeClient class to - # register itself and request knowledge that it then prints. - tree-printer: - build: - context: .. - dockerfile: ./examples/tree-printer/Dockerfile - - # This is a database that is used to demonstrate the dynamic authorization - # capabilities of the knowledge mapper. Currently, you still have to manually - # insert the authorization policies into the database. - auth-db: - image: mariadb:10.5 - volumes: - - ./auth-db/seed_data:/docker-entrypoint-initdb.d/:ro - - auth-db-data:/var/lib/mysql - environment: - MARIADB_DATABASE: knowledge_mapper_db - MARIADB_USER: user - MARIADB_PASSWORD: password - MARIADB_RANDOM_ROOT_PASSWORD: 1 - - # UI for the auth database to be able to easily change it. Add a rule here: - # http://localhost:8080/?server=auth-db&username=user&db=knowledge_mapper_db&edit=policies - adminer: - image: adminer:4.8.1 - ports: - - 8080:8080 - environment: - ADMINER_DESIGN: pepa-linha - ADMINER_DEFAULT_SERVER: auth-db - - dynamic-auth-mapper: - build: - context: .. - dockerfile: ./examples/dynamic-auth-mapper/Dockerfile - environment: - SPARQL_USERNAME: user - SPARQL_PASSWORD: password - -volumes: - auth-db-data: diff --git a/examples/legacy/dynamic-auth-mapper/Dockerfile b/examples/legacy/dynamic-auth-mapper/Dockerfile deleted file mode 100644 index ad49b0b..0000000 --- a/examples/legacy/dynamic-auth-mapper/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.11.1-alpine - -WORKDIR /usr/src/app/ - -RUN pip install --upgrade pip -COPY ./requirements.txt . -RUN pip install -r requirements.txt - -# NOTE: In a real mapper you shouldn't copy the folder, but can simply add it to `requirements.txt` -COPY ./knowledge_mapper ./knowledge_mapper - -# Place the configuration in the Docker image -COPY ./examples/dynamic-auth-mapper/config.jsonc . - -COPY ./examples/dynamic-auth-mapper/example.py ./your_package/ - -ENTRYPOINT [ "python", "-m", "knowledge_mapper", "./config.jsonc" ] diff --git a/examples/legacy/dynamic-auth-mapper/config.jsonc b/examples/legacy/dynamic-auth-mapper/config.jsonc deleted file mode 100644 index 75df530..0000000 --- a/examples/legacy/dynamic-auth-mapper/config.jsonc +++ /dev/null @@ -1,45 +0,0 @@ -{ - // The endpoint where a knowledge engine is available. - "knowledge_engine_endpoint": "http://tke-runtime:8280/rest", - "knowledge_base": { - // An URL representing the identity of this knowledge base - "id": "https://example.org/a-custom-knowledge-base-with-dyn-auth", - // A name for this knowledge base - "name": "Some knowledge base", - // A description for this knowledge base - "description": "This is just an example." - }, - - // This property tells the Knowledge Mapper that we want to use a custom - // Python class as the data source. - "plugin": { - "class": "your_package.example.ExampleDataSource", - "args": ["this is an argument that ends up in ExampleDataSource's __init__"] - }, - - // This property enables dynamic authorization so that you can permit/block - // specific knowledge bases dynamically (in a database). - "authorization": { - "type": "sql", - "sql": { - "host": "auth-db", - "port": 3306, - "database": "knowledge_mapper_db", - "user": "user", - "password": "password" - } - }, - - // Several knowledge interaction definitions can be placed here. - "knowledge_interactions": [ - { - // The type of this knowledge interaction. If we have knowledge - // available that is requestable, the type should be "answer". - "type": "answer", - // The graph pattern that expresses the 'shape' of our knowledge. - "pattern": "?tree ?height . ?tree ?name .", - // An optional name of this knowledge interaction. - "name": "answer-trees-with-heights-and-names" - } - ] -} \ No newline at end of file diff --git a/examples/legacy/dynamic-auth-mapper/example.py b/examples/legacy/dynamic-auth-mapper/example.py deleted file mode 100644 index 8079a05..0000000 --- a/examples/legacy/dynamic-auth-mapper/example.py +++ /dev/null @@ -1,17 +0,0 @@ -from src.data_source import DataSource - -class ExampleDataSource(DataSource): - def __init__(self, argument): - print(argument) - - def test(self): - print('testing example') - - def handle(self, ki, binding_set, requesting_kb): - result_bindings = [] - binding = dict() - binding['tree'] = '' - binding['height'] = '"48"^^' - binding['name'] = '"Dynamically secured Maple"' - result_bindings.append(binding) - return result_bindings diff --git a/examples/legacy/sparql-mapper/Dockerfile b/examples/legacy/sparql-mapper/Dockerfile deleted file mode 100644 index 61375e4..0000000 --- a/examples/legacy/sparql-mapper/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3.11.1-alpine - -WORKDIR /usr/src/app/ - -RUN pip install --upgrade pip -COPY ./requirements.txt . -RUN pip install -r requirements.txt - -# NOTE: In a real mapper you shouldn't copy the folder, but can simply add it to `requirements.txt` -COPY ./knowledge_mapper ./knowledge_mapper - -# Place the configuration in the Docker image -COPY ./examples/sparql-mapper/config.jsonc . - -ENTRYPOINT [ "python", "-m", "knowledge_mapper", "./config.jsonc" ] diff --git a/examples/legacy/sparql-mapper/config.jsonc b/examples/legacy/sparql-mapper/config.jsonc deleted file mode 100644 index 52a22bb..0000000 --- a/examples/legacy/sparql-mapper/config.jsonc +++ /dev/null @@ -1,52 +0,0 @@ -{ - // The endpoint where a knowledge engine is available. - "knowledge_engine_endpoint": "http://tke-runtime:8280/rest", - "knowledge_base": { - // An URL representing the identity of this knowledge base - "id": "https://example.org/a-sparql-knowledge-base", - // A name for this knowledge base - "name": "Some SPARQL knowledge base", - // A description for this knowledge base - "description": "This is just an example." - }, - - // This property tells the Knowledge Mapper to use a SPARQL endpoint as - // its data source. - "sparql": { - "endpoint": "http://triple-store:7878", - "username_environment_var": "SPARQL_USERNAME", - "password_environment_var": "SPARQL_PASSWORD" - }, - - // This property enables authorization so that you can permit/block specific - // knowledge bases - "authorization_enabled": true, - - // Several knowledge interaction definitions can be placed here. - "knowledge_interactions": [ - { - // The type of this knowledge interaction. If we have knowledge - // available that is requestable, the type should be "answer". - "type": "answer", - // The graph pattern that expresses the 'shape' of our knowledge. - "pattern": "?tree ?height . ?tree ?name .", - // A list of id's of knowledge bases that are permitted to use this interaction. - "permitted": ["https://example.org/another-knowledge-base", "https://example.org/tree-printer"], - // An optional name of this knowledge interaction. - "name": "answer-trees-with-heights-and-names" - }, - { - // The type of this knowledge interaction. If we want to respons to when new knowledge - // arrives, the type should be "react". - "type": "react", - // The graph pattern that expresses the 'shape' of the knowledge that we receive. - "argument_pattern": "?tree ?height . ?tree ?name .", - // The graph pattern that expresses the 'shape' of the knowledge that we send as a response. - "result_pattern": null, - // A list of id's of knowledge bases that are permitted to use this interaction. - "permitted": ["https://example.org/another-knowledge-base", "https://example.org/tree-printer"], - // An optional name of this knowledge interaction. - "name": "react-trees-with-heights-and-names" - } - ] -} \ No newline at end of file diff --git a/examples/legacy/sql-db/seed_data/0-schema.sql b/examples/legacy/sql-db/seed_data/0-schema.sql deleted file mode 100644 index e4f88bf..0000000 --- a/examples/legacy/sql-db/seed_data/0-schema.sql +++ /dev/null @@ -1,8 +0,0 @@ -USE treedb; - -CREATE TABLE trees ( - id VARCHAR(255) NOT NULL, - height INT UNSIGNED NOT NULL, - name VARCHAR(255) NOT NULL, - PRIMARY KEY(id) -); diff --git a/examples/legacy/sql-db/seed_data/1-data.sql b/examples/legacy/sql-db/seed_data/1-data.sql deleted file mode 100644 index 462d589..0000000 --- a/examples/legacy/sql-db/seed_data/1-data.sql +++ /dev/null @@ -1,6 +0,0 @@ -USE treedb; - -INSERT INTO trees (id, height, name) VALUES - ('giantRedwood', 94, 'Giant Redwood'), - ('silverBirch', 15, 'Silver Birch') -; diff --git a/examples/legacy/sql-mapper/Dockerfile b/examples/legacy/sql-mapper/Dockerfile deleted file mode 100644 index 86ff2f4..0000000 --- a/examples/legacy/sql-mapper/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3.11.1-alpine - -WORKDIR /usr/src/app/ - -RUN pip install --upgrade pip -COPY ./requirements.txt . -RUN pip install -r requirements.txt - -# NOTE: In a real mapper you shouldn't copy the folder, but can simply add it to `requirements.txt` -COPY ./knowledge_mapper ./knowledge_mapper - -# Place the configuration in the Docker image -COPY ./examples/sql-mapper/config.jsonc . - -ENTRYPOINT [ "python", "-m", "knowledge_mapper", "./config.jsonc" ] diff --git a/examples/legacy/sql-mapper/config.jsonc b/examples/legacy/sql-mapper/config.jsonc deleted file mode 100644 index 18618f8..0000000 --- a/examples/legacy/sql-mapper/config.jsonc +++ /dev/null @@ -1,69 +0,0 @@ -{ - // The endpoint where a knowledge engine is available. - "knowledge_engine_endpoint": "http://tke-runtime:8280/rest", - "knowledge_base": { - // An URL representing the identity of this knowledge base - "id": "https://example.org/a-sql-knowledge-base", - // A name for this knowledge base - "name": "Some SQL knowledge base", - // A description for this knowledge base - "description": "This is just an example." - }, - - // These connection details for the SQL database tell the Knowledge Mapper - // to use the SQL database as its data source. - "sql_host": "sql-db", - "sql_port": 3306, - "sql_database": "treedb", - "sql_user": "user", - "sql_password": "password", - - // Several knowledge interaction definitions can be placed here. - "knowledge_interactions": [ - // Knowledge interaction that discloses all trees - { - // The type of this knowledge interaction. If we have knowledge - // available that is requestable, the type should be "answer". - "type": "answer", - // The graph pattern that expresses the 'shape' of our knowledge. - "pattern": "?tree ?height . ?tree ?name .", - // SQL query to query data to be used to fill bindings for the graph pattern. - // Note that the column names in the result set "tree" and "height" must - // correspond with the variable names in the graph pattern. - "sql_query": "SELECT id AS tree, height, name FROM trees", - // This tells the Knowledge Mapper to map values in the "tree" column - // to URIs prefixed with "http://example.org/trees/". - "column_prefixes": { - "tree": "http://example.org/trees/" - }, - // An optional name of this knowledge interaction. - "name": "answer-trees-with-heights-and-names" - }, - // Knowledge interaction that WRITES trees to the SQL database when they are - // POSTed to the interoperability layer - { - // The type of this knowledge interaction. If we want to respons to when new knowledge - // arrives, the type should be "react". - "type": "react", - // The graph pattern that expresses the 'shape' of the knowledge that we receive. - "argument_pattern": "?tree ?height . ?tree ?name .", - // The graph pattern that expresses the 'shape' of the knowledge that we send as a response. - "result_pattern": null, - // A list of SQL queries to be fired when new knowledge arrives. - // In this case, we simply insert the bindings into the table. - "sql_query": [ - { - "statement": "INSERT INTO trees (id, height, name) VALUES (%s, %s, %s)", - "bindings": ["tree", "height", "name"] - } - ], - // This tells the Knowledge Mapper to map values in the "tree" column - // to URIs prefixed with "http://example.org/trees/". - "column_prefixes": { - "tree": "http://example.org/trees/" - }, - // An optional name of this knowledge interaction. - "name": "react-trees-with-heights-and-names" - } - ] -} \ No newline at end of file diff --git a/examples/legacy/tree-printer/Dockerfile b/examples/legacy/tree-printer/Dockerfile deleted file mode 100644 index 34caea2..0000000 --- a/examples/legacy/tree-printer/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3.11.1-alpine - -WORKDIR /usr/src/app/ - -RUN pip install --upgrade pip -COPY ./requirements.txt . -RUN pip install -r requirements.txt - -# NOTE: In a real mapper you shouldn't copy the folder, but can simply add it to `requirements.txt` -COPY ./knowledge_mapper ./knowledge_mapper - -COPY ./examples/tree-printer/app.py . - -ENTRYPOINT [ "python", "app.py" ] diff --git a/examples/legacy/tree-printer/app.py b/examples/legacy/tree-printer/app.py deleted file mode 100644 index 54ae5c1..0000000 --- a/examples/legacy/tree-printer/app.py +++ /dev/null @@ -1,35 +0,0 @@ -from time import sleep -from src.knowledge_mapper.knowledge_interaction import AskKnowledgeInteraction, AskKnowledgeInteractionRegistrationRequest -from src.tke_client import TkeClient -from src.knowledge_base import KnowledgeBaseRegistrationRequest - -def print_tree(tree): - print(f'\t - Tree {tree["name"]} has height {tree["height"]}', flush=True) - -if __name__ == '__main__': - client = TkeClient('http://tke-runtime:8280/rest') - client.connect() - - kb = client.register(KnowledgeBaseRegistrationRequest(id='https://example.org/tree-printer', name='Tree Printer', description="This knowledge base prints trees.")) - - ki: AskKnowledgeInteraction = kb.register_knowledge_interaction(AskKnowledgeInteractionRegistrationRequest(pattern='?tree ?height . ?tree ?name .')) - - while True: - print('Asking for all trees...', flush=True) - trees = ki.ask([])['bindingSet'] - - print(f'There are {len(trees)} trees:') - for tree in trees: - print_tree(tree) - - if len(trees) < 5: - print(f'Haven\'t found all trees yet. Retrying in 5 seconds...', flush=True) - if (len(trees) == 4): - print(f'(do you still need to give access at http://localhost:8080/?server=auth-db&username=user&db=knowledge_mapper_db&select=policies ?)', flush=True) - sleep(5) - continue - else: - print(f'Found all {len(trees)} trees! Bye.') - break - - kb.unregister() diff --git a/examples/legacy/triple-store/data.ttl b/examples/legacy/triple-store/data.ttl deleted file mode 100644 index 8929e3a..0000000 --- a/examples/legacy/triple-store/data.ttl +++ /dev/null @@ -1,4 +0,0 @@ -@prefix : . - -:easternWhitePine :hasHeight 31 . -:easternWhitePine :hasName "Eastern White Pine" . diff --git a/examples/legacy/triple-store/start.sh b/examples/legacy/triple-store/start.sh deleted file mode 100755 index faa34b3..0000000 --- a/examples/legacy/triple-store/start.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -oxigraph_server load -f /seed-data/data.ttl --location /data - -exec oxigraph_server serve --bind 0.0.0.0:7878 --location /data diff --git a/pyproject.toml b/pyproject.toml index 7ae1c8b..c1c1932 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,8 +39,6 @@ filterwarnings = [ exclude = [ ".pytest_cache", ".venv", - "./src/knowledge_mapper/legacy", - "./examples/legacy", ] line-length = 88 indent-width = 4 diff --git a/setup.py b/setup.py deleted file mode 100644 index 5e589a1..0000000 --- a/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import find_packages, setup - -setup(packages=find_packages()) diff --git a/src/knowledge_mapper/legacy/__main__.py b/src/knowledge_mapper/legacy/__main__.py deleted file mode 100644 index 2004747..0000000 --- a/src/knowledge_mapper/legacy/__main__.py +++ /dev/null @@ -1,187 +0,0 @@ -import os -import argparse -import logging as log -import json5 -import sys -import importlib -import time -import signal -import requests.exceptions -from src.knowledge_mapper.kb.knowledge_base import KnowledgeBaseUnregistered - -from src.knowledge_mapper import KnowledgeMapper -from src.auth.sql_auth import SqlAuth -from src.auth.static_auth import StaticAuth -from src.data_source import DataSource -from src.sparql_source import SparqlSource -from src.sql_source import SqlSource - -log.basicConfig(level=log.INFO) - -# This function is called when a SIGTERM signal is received. This makes it so -# that the knowledge mapper can be gracefully killed by Docker. -def handle_sigterm(*args): - raise KeyboardInterrupt() - - -signal.signal(signal.SIGTERM, handle_sigterm) - -DATA_SOURCE_MAX_CONNECTION_ATTEMPTS = 10 -DATA_SOURCE_WAIT_BEFORE_RETRY = 3 - -KE_DISAPPEARED_COOLDOWN = 5 - - -def test_data_source(data_source: DataSource): - success = False - attempts = 0 - while not success: - try: - data_source.test() - success = True - except Exception as e: - attempts += 1 - if attempts < DATA_SOURCE_MAX_CONNECTION_ATTEMPTS: - log.warning( - f"Request to data source failed. Retrying in {DATA_SOURCE_WAIT_BEFORE_RETRY} s." - ) - time.sleep(DATA_SOURCE_WAIT_BEFORE_RETRY) - else: - log.error(f"Request to data source failed.") - raise e - - -def main(): - from ...legacy import __version__ - - log.info(f"Running Knowledge Mapper {__version__}") - parser = argparse.ArgumentParser( - description="Expose an endpoint to a knowledge network." - ) - parser.add_argument("config") - parser.add_argument("--wizard", action=argparse.BooleanOptionalAction) - args = parser.parse_args() - if args.wizard: - from ...legacy import wizard_mapper - - wizard_mapper.start() - exit() - with open(args.config) as config_file: - config = json5.load(config_file) - - if "sparql" in config: - endpoint = config["sparql"]["endpoint"] - username = None - password = None - if ( - "username_environment_var" in config["sparql"] - and "password_environment_var" in config["sparql"] - ): - username = config["sparql"]["username_environment_var"] - password = config["sparql"]["password_environment_var"] - data_source = SparqlSource(endpoint, username, password) - elif "sql_host" in config: - data_source = SqlSource( - config["sql_host"], - config["sql_port"], - config["sql_database"], - config["sql_user"], - config["sql_password"], - ) - elif "plugin" in config: - plugin_cfg = config["plugin"] - module_name, class_name = plugin_cfg["class"].rsplit(".", 1) - plugin_module = importlib.import_module(module_name) - plugin_class = getattr(plugin_module, class_name) - if "args" in plugin_cfg: - data_source = plugin_class(*plugin_cfg["args"]) - else: - data_source = plugin_class() - else: - log.error("Invalid config.") - sys.exit(1) - - test_data_source(data_source) - - if "authorization_enabled" in config: - if "authorization" in config: - log.error( - 'Cannot use both `authorization_enabled` and `authorization`, as `authorization_enabled=true` is a shorthand for `authorization={type="static"}`.' - ) - sys.exit(1) - - if config["authorization_enabled"] == True: - auth_config = {"type": "static"} - elif config["authorization_enabled"] == False: - auth_config = None - else: - log.error('"authorization_enabled" must be either "true" or "false"') - sys.exit(1) - elif "authorization" in config: - auth_config = config["authorization"] - else: - auth_config = None - - if auth_config is not None: - if auth_config["type"] == "sql": - authorization = SqlAuth(auth_config) - elif auth_config["type"] == "static": - authorization = StaticAuth(auth_config) - else: - log.error('Unknown authorization type "%s"', auth_config["type"]) - sys.exit(1) - else: - authorization = None - - if "KE_ENDPOINT" in os.environ: - ke_endpoint = os.environ.get("KE_ENDPOINT") - if "knowledge_engine_endpoint" in config: - log.warning( - 'Using "KE_ENDPOINT" environment variable, but also found "knowledge_engine_endpoint" property in configuration file.' - ) - else: - ke_endpoint = config["knowledge_engine_endpoint"] - - km = KnowledgeMapper( - data_source, - authorization, - ke_endpoint, - config["knowledge_base"]["id"], - config["knowledge_base"]["name"], - config["knowledge_base"]["description"], - ) - for ki in config["knowledge_interactions"]: - if "vars" in ki: - log.warning( - 'config file provided "vars" property in a knowledge interaction, but they can also be inferred' - ) - km.add_knowledge_interaction(ki) - - try: - while True: - try: - km.start() - break - except KnowledgeBaseUnregistered: - log.warning( - "Knowledge Base has been registered for some reason. Will try to reregister." - ) - km.reregister() - for ki in config["knowledge_interactions"]: - km.add_knowledge_interaction(ki) - except requests.exceptions.ConnectionError: - log.warning( - f"Knowledge Engine runtime disappeared. Will re-enter handle loop in {KE_DISAPPEARED_COOLDOWN} seconds..." - ) - time.sleep(KE_DISAPPEARED_COOLDOWN) - except KeyboardInterrupt: - log.info("Shutting down gracefully...") - finally: - km.clean_up() - - log.info("Goodbye.") - exit(0) - - -if __name__ == "__main__": - main() diff --git a/src/knowledge_mapper/legacy/auth/__init__.py b/src/knowledge_mapper/legacy/auth/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/knowledge_mapper/legacy/auth/base_auth.py b/src/knowledge_mapper/legacy/auth/base_auth.py deleted file mode 100644 index 87f47ca..0000000 --- a/src/knowledge_mapper/legacy/auth/base_auth.py +++ /dev/null @@ -1,7 +0,0 @@ -class BaseAuth: - def has_permission(self, kb_id, ki): - raise NotImplementedError("Please implement this abstract method.") - - def add_knowledge_interaction(self, ki): - """Called when a Knowledge Interaction is added to this Knowledge Mapper""" - pass diff --git a/src/knowledge_mapper/legacy/auth/sql_auth.py b/src/knowledge_mapper/legacy/auth/sql_auth.py deleted file mode 100644 index 761ba2c..0000000 --- a/src/knowledge_mapper/legacy/auth/sql_auth.py +++ /dev/null @@ -1,70 +0,0 @@ -import logging as log -import mysql.connector -import time -from datetime import datetime - -from src.auth.base_auth import BaseAuth - -MAX_CONNECTION_ATTEMPTS = 10 -WAIT_BEFORE_RETRY = 2 - -class SqlAuth(BaseAuth): - def __init__(self, conf: dict): - self.host = conf['sql']['host'] - self.port = conf['sql']['port'] - self.database = conf['sql']['database'] - self.user = conf['sql']['user'] - self.password = conf['sql']['password'] - self.connect() - - def connect(self, max_attempts=MAX_CONNECTION_ATTEMPTS, wait_between_attempts=WAIT_BEFORE_RETRY): - success = False - attempts = 0 - while not success: - try: - log.info(f'Attempting SQL connection to {self.database} in {self.host}:{self.port} for SQL auth...') - self.conn = mysql.connector.connect( - user=self.user, - password=self.password, - host=self.host, - port=self.port, - database=self.database - ) - self.ping() - success = True - except Exception as e: - attempts += 1 - if attempts < max_attempts: - log.warning(f'SQL connection attempt to {self.database} in {self.host}:{self.port} failed. Retrying in {wait_between_attempts} s.') - time.sleep(wait_between_attempts) - else: - log.error(f'Failed to establish SQL connection.') - raise e - log.info('Successfully made SQL connection for SQL auth.') - - def ping(self): - self.conn.ping() - - def has_permission(self, kb_id, ki): - cursor = self.conn.cursor(named_tuple=True) - cursor.execute('SELECT COUNT(*) > 0 AS permission FROM policies WHERE (knowledge_interaction_id=%s OR knowledge_interaction_id IS NULL) AND (knowledge_base_id=%s OR knowledge_base_id IS NULL);', (ki['id'], kb_id)) - row = cursor.fetchone() - if row is None: - raise Exception('Expected a row in the result set.') - cursor.close() - self.conn.commit() - if row.permission: - log.debug(f'KB {kb_id} has access to KI {ki["id"]}') - else: - log.debug(f'KB {kb_id} does not have access to KI {ki["id"]}') - - cursor = self.conn.cursor() - cursor.execute('INSERT INTO access_log (knowledge_interaction_id, knowledge_base_id, access_datetime, success) VALUES (%s, %s, %s, %s);', (ki['id'], kb_id, datetime.now(), row.permission)) - self.conn.commit() - - return row.permission - - def add_knowledge_interaction(self, ki): - cursor = self.conn.cursor() - cursor.execute('INSERT INTO knowledge_interactions (id, name) VALUES (%s, %s) ON DUPLICATE KEY UPDATE name=%s;', (ki['id'], ki['name'], ki['name'])) - self.conn.commit() diff --git a/src/knowledge_mapper/legacy/auth/static_auth.py b/src/knowledge_mapper/legacy/auth/static_auth.py deleted file mode 100644 index 8252f8f..0000000 --- a/src/knowledge_mapper/legacy/auth/static_auth.py +++ /dev/null @@ -1,22 +0,0 @@ -import logging as log - -from src.auth.base_auth import BaseAuth - -class StaticAuth(BaseAuth): - def __init__(self, _conf): - pass - - def has_permission(self, kb_id, ki): - permission = False - if 'permitted' in ki: - if ki['permitted'] != "*": - # check whether the requesting kb is in the permitted list - if kb_id in ki['permitted']: - permission = True - else: - log.info('Knowledge base %s is not permitted to do this request!', kb_id) - else: # permission is set to *, so every one is permitted - permission = True - else: - permission = False - return permission diff --git a/src/knowledge_mapper/legacy/data_source.py b/src/knowledge_mapper/legacy/data_source.py deleted file mode 100644 index 9d83ec9..0000000 --- a/src/knowledge_mapper/legacy/data_source.py +++ /dev/null @@ -1,10 +0,0 @@ -from src.knowledge_mapper.kb.knowledge_base import KnowledgeBase - -class DataSource: - def test(self): - raise NotImplementedError("Please implement this abstract method.") - def handle(self, ki, binding_set, requesting_kb): - raise NotImplementedError("Please implement this abstract method.") - - def set_knowledge_base(self, kb: KnowledgeBase): - self.kb = kb diff --git a/src/knowledge_mapper/legacy/knowledge_base.py b/src/knowledge_mapper/legacy/knowledge_base.py deleted file mode 100644 index fa547f8..0000000 --- a/src/knowledge_mapper/legacy/knowledge_base.py +++ /dev/null @@ -1,258 +0,0 @@ -from __future__ import annotations -from dataclasses import dataclass -import logging as log -import requests - -import src.knowledge_mapper.knowledge_interaction as knowledge_interaction - -from src.tke_exceptions import UnexpectedHttpResponseError - - -class KnowledgeBaseUnregistered(Exception): - pass - - -class KnowledgeEngineTerminated(Exception): - pass - - -@dataclass(kw_only=True) -class KnowledgeBaseRegistrationRequest: - """Class with necessary data itemsfor registering a new knowledge base.""" - - id: str - name: str - description: str - - -class KnowledgeBase: - def __init__(self, req: KnowledgeBaseRegistrationRequest, ke_url: str): - self.ke_url = ke_url - self.id = req.id - self.name = req.name - self.description = req.description - self.kis = dict() - self.kis_by_name = dict() - - def from_json(kb_json: dict, ke_url: str) -> KnowledgeBase: - kb = KnowledgeBase( - KnowledgeBaseRegistrationRequest( - id=kb_json["knowledgeBaseId"], - name=kb_json["knowledgeBaseName"], - description=kb_json["knowledgeBaseDescription"], - ), - ke_url=ke_url, - ) - - kb.sync_knowledge_interactions() - - return kb - - def unregister(self): - if self.ke_url is None: - raise Exception( - "Cannot unregister this KB because no knowledge engine URL is known for this object." - ) - - response = requests.delete( - f"{self.ke_url}/sc", headers={"Knowledge-Base-Id": self.id} - ) - - if not response.ok: - raise UnexpectedHttpResponseError(response) - - def register_knowledge_interaction( - self, - ki: knowledge_interaction.KnowledgeInteractionRegistrationRequest, - name=None, - ) -> knowledge_interaction.KnowledgeInteraction: - body = {"knowledgeInteractionType": ki.type, "prefixes": ki.prefixes} - - if name is not None: - body["knowledgeInteractionName"] = name - - if isinstance( - ki, - knowledge_interaction.AskKnowledgeInteractionRegistrationRequest - | knowledge_interaction.AnswerKnowledgeInteractionRegistrationRequest, - ): - body["graphPattern"] = ki.pattern - body["knowledgeGapsEnabled"] = ki.knowledge_gaps_enabled - elif isinstance( - ki, - knowledge_interaction.PostKnowledgeInteractionRegistrationRequest - | knowledge_interaction.ReactKnowledgeInteractionRegistrationRequest, - ): - body["argumentGraphPattern"] = ki.argument_pattern - body["resultGraphPattern"] = ki.result_pattern - else: - raise Exception("`ki` must be a concrete knowledge interaction object") - - response = requests.post( - f"{self.ke_url}/sc/ki", headers={"Knowledge-Base-Id": self.id}, json=body - ) - - if not response.ok: - raise UnexpectedHttpResponseError(response) - - ki_id = response.json()["knowledgeInteractionId"] - - log.info(f"Successfully registered knowledge interaction {ki_id}.") - - registered_ki = knowledge_interaction.KnowledgeInteraction.from_req( - ki, ki_id, self - ) - self.kis[ki_id] = registered_ki - - if name is not None: - self.kis_by_name[name] = registered_ki - - return registered_ki - - def sync_knowledge_interactions(self): - response = requests.get( - f"{self.ke_url}/sc/ki", headers={"Knowledge-Base-Id": self.id} - ) - - if not response.ok: - raise UnexpectedHttpResponseError(response) - - raw_kis = response.json() - kis_not_seen_yet = self.kis.copy() - kis_by_name_not_seen_yet = self.kis_by_name.copy() - for raw_ki in raw_kis: - if raw_ki["knowledgeInteractionType"] == "AskKnowledgeInteraction": - req = knowledge_interaction.AskKnowledgeInteractionRegistrationRequest( - pattern=raw_ki["graphPattern"] - ) - elif raw_ki["knowledgeInteractionType"] == "AnswerKnowledgeInteraction": - req = ( - knowledge_interaction.AnswerKnowledgeInteractionRegistrationRequest( - pattern=raw_ki["graphPattern"], handler=None - ) - ) - elif raw_ki["knowledgeInteractionType"] == "PostKnowledgeInteraction": - req = knowledge_interaction.PostKnowledgeInteractionRegistrationRequest( - argument_pattern=raw_ki["argumentGraphPattern"], - result_pattern=raw_ki["resultGraphPattern"] - if "resultGraphPattern" in raw_ki - else None, - ) - elif raw_ki["knowledgeInteractionType"] == "ReactKnowledgeInteraction": - req = ( - knowledge_interaction.ReactKnowledgeInteractionRegistrationRequest( - argument_pattern=raw_ki["argumentGraphPattern"], - result_pattern=raw_ki["resultGraphPattern"] - if "resultGraphPattern" in raw_ki - else None, - handler=None, - ) - ) - else: - raise Exception( - f"Invalid KI type: {raw_ki['knowledgeInteractionType']}" - ) - - if "knowledgeInteractionName" in raw_ki: - name = raw_ki["knowledgeInteractionName"] - else: - name = None - - ki_id = raw_ki["knowledgeInteractionId"] - - ki = knowledge_interaction.KnowledgeInteraction.from_req( - req, ki_id, self, name - ) - - self.kis[ki_id] = ki - if ki_id in kis_not_seen_yet: - del kis_not_seen_yet[ki_id] - if name is not None: - self.kis_by_name[name] = ki - if name in kis_by_name_not_seen_yet: - del kis_by_name_not_seen_yet[name] - - for no_longer_existing_ki_id in kis_not_seen_yet.keys(): - del self.kis[no_longer_existing_ki_id] - - for no_longer_existing_ki_name in kis_by_name_not_seen_yet.keys(): - del self.kis_by_name[no_longer_existing_ki_name] - - def get_ki(self, name=None, id=None) -> knowledge_interaction.KnowledgeInteraction: - if name is not None and id is None: - if name in self.kis_by_name: - return self.kis_by_name[name] - if id is not None and name is None: - if id in self.kis: - return self.kis[id] - return None - - def start_handle_loop(self, loops=None): - did_loops = 0 - while loops is not None and did_loops < loops or loops is None: - status, maybe_ke_request = self.long_poll() - did_loops += 1 - if status == "repoll": - continue - elif status == "reregister": - self.reregister() - elif status == "exit": - log.warn("KE returned status code 410, meaning I have to exit") - break - elif status == "handle": - ke_request = maybe_ke_request - - # Extract relevant data items from the KE request - ki_id = ke_request["knowledgeInteractionId"] - handle_request_id = ke_request["handleRequestId"] - bindings = ke_request["bindingSet"] - requesting_kb_id = ke_request["requestingKnowledgeBaseId"] - - ki = self.kis[ki_id] - if isinstance(ki, knowledge_interaction.AnswerKnowledgeInteraction): - result_bindings = ki.answer(bindings, requesting_kb_id) - elif isinstance(ki, knowledge_interaction.ReactKnowledgeInteraction): - result_bindings = ki.react(bindings, requesting_kb_id) - - self.post_handle_response(ki, handle_request_id, result_bindings) - - def long_poll(self): - log.info("Waiting for response to long poll...") - response = requests.get( - f"{self.ke_url}/sc/handle", headers={"Knowledge-Base-Id": self.id} - ) - if response.status_code == 202: - log.info("Received 202.") - return "repoll", None - elif response.status_code == 500: - log.error(response.text) - log.error("TKE had an internal server error. Reinitiating long poll.") - return "repoll", None - elif response.status_code == 410: - return "exit", None - elif response.status_code == 200: - return "handle", response.json() - elif response.status_code == 404: - log.warn("Our Knowledge Base has been unregistered!") - raise KnowledgeBaseUnregistered() - else: - log.warn(f"long_poll received unexpected status {response.status_code}") - log.warn(response.text) - log.warn("repolling anyway..") - return "repoll", None - - def post_handle_response(self, ki, handle_id, bindings): - response = requests.post( - f"{self.ke_url}/sc/handle", - json={ - "handleRequestId": handle_id, - "bindingSet": bindings, - }, - headers={ - "Knowledge-Base-Id": self.id, - "Knowledge-Interaction-Id": ki.id, - }, - ) - - if not response.ok: - raise UnexpectedHttpResponseError(response) diff --git a/src/knowledge_mapper/legacy/knowledge_interaction.py b/src/knowledge_mapper/legacy/knowledge_interaction.py deleted file mode 100644 index d4b3d79..0000000 --- a/src/knowledge_mapper/legacy/knowledge_interaction.py +++ /dev/null @@ -1,131 +0,0 @@ -from __future__ import annotations -from dataclasses import dataclass, field -from typing import Callable - -import requests - -from src.tke_exceptions import UnexpectedHttpResponseError - -ASK = 'AskKnowledgeInteraction' -ANSWER = 'AnswerKnowledgeInteraction' -POST = 'PostKnowledgeInteraction' -REACT = 'ReactKnowledgeInteraction' - -@dataclass(kw_only=True) -class KnowledgeInteractionRegistrationRequest: - prefixes: dict = field(default_factory=dict) - type: str = None # This is defined in the concrete child classes. - knowledge_gaps_enabled: bool = None - -@dataclass(kw_only=True) -class AskKnowledgeInteractionRegistrationRequest(KnowledgeInteractionRegistrationRequest): - pattern: str - type: str = ASK - -@dataclass(kw_only=True) -class AnswerKnowledgeInteractionRegistrationRequest(KnowledgeInteractionRegistrationRequest): - pattern: str - type: str = ANSWER - handler: Callable[[list[dict], str], list[dict]] - -@dataclass(kw_only=True) -class PostKnowledgeInteractionRegistrationRequest(KnowledgeInteractionRegistrationRequest): - argument_pattern: str - result_pattern: str - type: str = POST - -@dataclass(kw_only=True) -class ReactKnowledgeInteractionRegistrationRequest(KnowledgeInteractionRegistrationRequest): - argument_pattern: str - result_pattern: str - type: str = REACT - handler: Callable[[list[dict], str], list[dict]] - -class KnowledgeInteraction: - def __init__(self, id: str, type: str, kb, kge=False, name=None): - self.id = id - self.type = type - self.knowledge_gaps_enabled = kge - self.kb = kb - self.name = name - - def from_req(req: KnowledgeInteractionRegistrationRequest, id: str, kb, name: str=None) -> KnowledgeInteraction: - if isinstance(req, AskKnowledgeInteractionRegistrationRequest): - return AskKnowledgeInteraction(req, id, kb, name) - elif isinstance(req, AnswerKnowledgeInteractionRegistrationRequest): - return AnswerKnowledgeInteraction(req, id, kb, name) - elif isinstance(req, PostKnowledgeInteractionRegistrationRequest): - return PostKnowledgeInteraction(req, id, kb, name) - elif isinstance(req, ReactKnowledgeInteractionRegistrationRequest): - return ReactKnowledgeInteraction(req, id, kb, name) - else: - raise Exception('`req` must be a concrete knowledge interaction object') - - def __eq__(self, other: object) -> bool: - return self.id == other.id and self.type == other.type and self.name == other.name - - -class AskKnowledgeInteraction(KnowledgeInteraction): - def __init__(self, req: AskKnowledgeInteractionRegistrationRequest, id: str, kb, name=None): - super().__init__(id, req.type, kb, req.knowledge_gaps_enabled) - self.pattern = req.pattern - self.prefixes = req.prefixes - - def ask(self, bindings: dict) -> dict: - response = requests.post( - f'{self.kb.ke_url}/sc/ask', - json=bindings, - headers={ - 'Knowledge-Base-Id': self.kb.id, - 'Knowledge-Interaction-Id': self.id, - } - ) - if not response.ok: - raise UnexpectedHttpResponseError(response) - - return response.json() - - -class AnswerKnowledgeInteraction(KnowledgeInteraction): - def __init__(self, req: AnswerKnowledgeInteractionRegistrationRequest, id: str, kb, name=None): - super().__init__(id, req.type, kb) - self.pattern = req.pattern - self.prefixes = req.prefixes - self.handler = req.handler - - def answer(self, bindings: list[dict], requesting_kb_id: str) -> list[dict]: - return self.handler(bindings, requesting_kb_id) - - -class PostKnowledgeInteraction(KnowledgeInteraction): - def __init__(self, req: PostKnowledgeInteractionRegistrationRequest, id: str, kb, name=None): - super().__init__(id, req.type, kb) - self.argument_pattern = req.argument_pattern - self.result_pattern = req.result_pattern - self.prefixes = req.prefixes - - def post(self, bindings: list[dict]) -> list[dict]: - response = requests.post( - f'{self.kb.ke_url}/sc/post', - json=bindings, - headers={ - 'Knowledge-Base-Id': self.kb.id, - 'Knowledge-Interaction-Id': self.id, - } - ) - if not response.ok: - raise UnexpectedHttpResponseError(response) - - return response.json() - - -class ReactKnowledgeInteraction(KnowledgeInteraction): - def __init__(self, req: ReactKnowledgeInteractionRegistrationRequest, id: str, kb, name=None): - super().__init__(id, req.type, kb) - self.argument_pattern = req.argument_pattern - self.result_pattern = req.result_pattern - self.prefixes = req.prefixes - self.handler = req.handler - - def react(self, bindings: list[dict], requesting_kb_id: str) -> list[dict]: - return self.handler(bindings, requesting_kb_id) diff --git a/src/knowledge_mapper/legacy/knowledge_mapper.py b/src/knowledge_mapper/legacy/knowledge_mapper.py deleted file mode 100644 index 8aac732..0000000 --- a/src/knowledge_mapper/legacy/knowledge_mapper.py +++ /dev/null @@ -1,134 +0,0 @@ -from functools import partial -import logging as log -from src.auth.base_auth import BaseAuth -from src.utils import extract_variables -from src.knowledge_mapper.kb.knowledge_base import ( - KnowledgeBaseRegistrationRequest, -) -from src.knowledge_mapper.knowledge_interaction import ( - AnswerKnowledgeInteractionRegistrationRequest, - AskKnowledgeInteractionRegistrationRequest, - PostKnowledgeInteractionRegistrationRequest, - ReactKnowledgeInteractionRegistrationRequest, -) - -from .data_source import DataSource -from .tke_client import TkeClient - -WAIT_BEFORE_RETRY = 1 - - -class KnowledgeMapper: - def __init__( - self, - data_source: DataSource, - authorization: BaseAuth, - ke_url: str, - kb_id: str, - kb_name: str, - kb_desc: str, - ): - self.data_source = data_source - self.ke_url = ke_url - self.kb_id = kb_id - self.kb_name = kb_name - self.kb_desc = kb_desc - self.kis = dict() - - self.tke_client = TkeClient(ke_url) - self.tke_client.connect() - - self.kb = self.tke_client.register( - KnowledgeBaseRegistrationRequest( - id=kb_id, name=kb_name, description=kb_desc - ) - ) - self.data_source.set_knowledge_base(self.kb) - - self.authorization = authorization - - def start(self): - self.kb.start_handle_loop() - - def clean_up(self): - self.kb.unregister() - - def reregister(self): - self.kb = self.tke_client.register( - KnowledgeBaseRegistrationRequest( - id=self.kb_id, name=self.kb_name, description=self.kb_desc - ) - ) - - def handle(self, ki, bindings: list[dict], requesting_kb: str): - # For this implementation we assume that the knowledge mapper is responsible for authorisation - # Check whether the requesting knowledge base is permitted to request the knowledge interaction - if self.authorization is not None: - permission = self.authorization.has_permission(requesting_kb, ki) - else: # no authorization is defined so, have the data source handle the request. - permission = True - - # if permitted, then handle the request - if permission: - try: - result = self.data_source.handle(ki, bindings, requesting_kb) - except Exception: - log.exception( - "an exception occurred while the data source was handling a knowledge request" - ) - result = [] - else: - result = [] - return result - - def add_knowledge_interaction(self, ki): - if "prefixes" in ki: - prefixes = ki["prefixes"] - else: - prefixes = dict() - - if ki["type"] == "ask": - req = AskKnowledgeInteractionRegistrationRequest(pattern=ki["pattern"]) - if "vars" not in ki: - ki["vars"] = extract_variables(ki["pattern"], prefixes=prefixes) - elif ki["type"] == "answer": - req = AnswerKnowledgeInteractionRegistrationRequest( - pattern=ki["pattern"], handler=partial(self.handle, ki) - ) - if "vars" not in ki: - ki["vars"] = extract_variables(ki["pattern"], prefixes=prefixes) - elif ki["type"] == "post": - req = PostKnowledgeInteractionRegistrationRequest( - argument_pattern=ki["argument_pattern"], - result_pattern=ki["result_pattern"], - ) - if "vars" not in ki: - ki["vars"] = extract_variables( - ki["argument_pattern"], prefixes=prefixes - ) - elif ki["type"] == "react": - req = ReactKnowledgeInteractionRegistrationRequest( - argument_pattern=ki["argument_pattern"], - result_pattern=ki["result_pattern"], - handler=partial(self.handle, ki), - ) - if "vars" not in ki: - ki["vars"] = extract_variables( - ki["argument_pattern"], prefixes=prefixes - ) - else: - raise Exception(f"Invalid KI type: {ki['type']}") - - req.prefixes = prefixes - - if "name" not in ki: - ki["name"] = None - name = ki["name"] - - registered_ki = self.kb.register_knowledge_interaction(req, name=name) - ki["id"] = registered_ki.id - - # The authorization object needs to be made aware of the new knowledge - # interaction - if self.authorization is not None: - self.authorization.add_knowledge_interaction(ki) diff --git a/src/knowledge_mapper/legacy/sparql_source.py b/src/knowledge_mapper/legacy/sparql_source.py deleted file mode 100644 index 8db155e..0000000 --- a/src/knowledge_mapper/legacy/sparql_source.py +++ /dev/null @@ -1,188 +0,0 @@ -import requests -from requests.auth import HTTPBasicAuth -import os -import logging as log -from urllib.parse import quote - -from .data_source import DataSource - - -class SparqlSource(DataSource): - def __init__(self, endpoint: str, env_username, env_password): - self.endpoint = endpoint - self.auth = False - # env_username and env_password MUST be the names of the environment variables - # that contain the username and password to get access to the endpoint - # if they are present, the self.auth flag will go up - if (env_username != None and env_username in os.environ) and ( - env_password != None and env_password in os.environ - ): - self.auth = True - self.username = os.environ[env_username] - self.password = os.environ[env_password] - - def test(self): - log.info("Testing SPARQL endpoint.") - self.do_sparql_select("SELECT * WHERE { ?s ?p ?o . } LIMIT 0") - log.info("Succes!") - - def handle(self, ki, binding_set, requesting_kb): - if ki["type"] == "answer": - return self.handle_answer(ki, binding_set, requesting_kb) - elif ki["type"] == "react": - return self.handle_react(ki, binding_set, requesting_kb) - - def handle_answer(self, ki, binding_set, requesting_kb): - # Generate the SPARQL query based on the incoming bindings and the knowledge interaction's graph pattern. - query = generate_sparql_select(ki, binding_set) - # Fire the SPARQL query. - result = self.do_sparql_select(query) - # Restructure the bindings into TKE bindings and return it - return restructure_bindings(result) - - def handle_react(self, ki, binding_set, requesting_kb): - # Generate the SPARQL query based on the incoming bindings and the knowledge interaction's graph pattern. - query = generate_sparql_insert(ki, binding_set) - # Fire the SPARQL query. - self.do_sparql_insert(query) - # Restructure the bindings into TKE bindings and return it - return [] - - def do_sparql_select(self, query): - args = { - "headers": { - "Accept": "application/sparql-results+json", - "Content-Type": "application/sparql-query", - } - } - - if self.auth: - args["auth"] = HTTPBasicAuth(self.username, self.password) - - response = requests.post(f"{self.endpoint}/query", data=query, **args) - - if response.status_code == 401: - raise UnauthorizedError( - "Provide BasicAuth with system environment variables." - ) - elif not response.ok: - raise RuntimeError( - "Invalid response from SPARQL endpoint. (status: {}, body: {})".format( - response.status_code, response.text - ) - ) - - return response.json() - - def do_sparql_insert(self, query): - args = { - "headers": { - "Accept": "application/json", - "Content-Type": "application/sparql-update", - } - } - - if self.auth: - args["auth"] = HTTPBasicAuth(self.username, self.password) - - response = requests.post(f"{self.endpoint}", data=query, **args) - - if response.status_code == 401: - raise UnauthorizedError( - "Provide BasicAuth with system environment variables with the names used in the config file." - ) - elif not response.ok: - raise RuntimeError( - "Invalid response from SPARQL endpoint. (status: {}, body: {})".format( - response.status_code, response.text - ) - ) - - return - - -def generate_sparql_select(ki, incoming_bindings): - # For all partial bindings that are actually partial, we set the - # variables that ARE in the graph pattern, but NOT in the binding to - # UNDEF, so that they match anything. - for var in ki["vars"]: - for incoming_binding in incoming_bindings: - if var not in incoming_binding: - incoming_binding[var] = "UNDEF" - - if "prefixes" in ki: - prefixes = ki["prefixes"] - else: - prefixes = dict() - - return """ - {prefixes_clause} - SELECT {{variables}} WHERE {{{{ - {triple_pattern} - {values_clause} - }}}} - """.format( - prefixes_clause="\n\t".join( - f"PREFIX {prefix}: <{ki['prefixes'][prefix]}>" for prefix in prefixes.keys() - ), - triple_pattern=ki["pattern"], - values_clause=""" - VALUES ({{variables}}) {{{{ - {bindings} - }}}} - """.format( - bindings="\n".join( - [ - f'({" ".join([binding[var] for var in ki["vars"]])})' - for binding in incoming_bindings - ] - ) - ) - if incoming_bindings - else "", - ).format( - variables=" ".join([f"?{var}" for var in ki["vars"]]), - ) - - -def generate_sparql_insert(ki, bindings): - variables = ki["vars"] - if "prefixes" in ki: - prefixes = ki["prefixes"] - else: - prefixes = dict() - pref = "\n\t".join( - f"PREFIX {prefix}: <{ki['prefixes'][prefix]}>" for prefix in prefixes.keys() - ) - - return f""" - {pref} - INSERT {{ - {ki['argument_pattern']} - }} WHERE {{ VALUES ({' '.join([f'?{variable}' for variable in variables])}) {{ - {os.linesep.join(f'({" ".join([str(binding[variable]) for variable in variables])})' for binding in bindings)} - }} }} - """ - - -def restructure_bindings(sparql_results): - restructured_binding_set = [] - for binding in sparql_results["results"]["bindings"]: - restructured_binding = dict() - for key, value in binding.items(): - if value["type"] == "uri": - restructured_value = f'<{value["value"]}>' - elif binding[key]["type"] == "literal": - if "datatype" in value: - restructured_value = f'"{value["value"]}"^^<{value["datatype"]}>' - else: - restructured_value = f'"{value["value"]}"' - restructured_binding[key] = restructured_value - - restructured_binding_set.append(restructured_binding) - - return restructured_binding_set - - -class UnauthorizedError(RuntimeError): - pass diff --git a/src/knowledge_mapper/legacy/sql_source.py b/src/knowledge_mapper/legacy/sql_source.py deleted file mode 100644 index 6cd9e4a..0000000 --- a/src/knowledge_mapper/legacy/sql_source.py +++ /dev/null @@ -1,131 +0,0 @@ -import logging as log -import mysql.connector - -from .data_source import DataSource - -class SqlSource(DataSource): - def __init__(self, host: str, port: int, database: str, user: str, password: str): - self.host = host - self.port = port - self.database = database - self.user = user - self.password = password - - def test(self): - log.info('Testing SQL connection.') - self.conn = mysql.connector.connect( - user=self.user, - password=self.password, - host=self.host, - port=self.port, - database=self.database) - self.conn.ping() - log.info('Succes!') - - - def handle(self, ki, binding_set, requesting_kb): - if ki['type'] == 'answer': - return self.handle_answer(ki, binding_set, requesting_kb) - elif ki['type'] == 'react': - return self.handle_react(ki, binding_set, requesting_kb) - - - def handle_answer(self, ki, binding_set, requesting_kb): - sql_bindings = () - if binding_set: - binding_constraints = '0 ' - - # Add constraint clause with a disjunct for every binding, and in - # each disjunct a conjunction with a conjunct for every binding - # entry. - for binding in binding_set: - binding_constraints += 'OR 1 ' - for key, value in binding.items(): - binding_constraints += f'AND {key} = %s ' - prefix = "" - if key in ki['column_prefixes']: - prefix = ki['column_prefixes'][key] - # If prefixed, remove the <>'s and the prefix, otherwise, - # use `value` as is. - if prefix: - unprefixed = value[len(prefix) + 1:-1] - else: - unprefixed = value - sql_bindings += (unprefixed,) - - # HAVING is used because WHERE is evaluated before aggregations, and - # so aliases defined in the query are unavailable. - query = f"{ki['sql_query']} HAVING {binding_constraints}" - else: - query = ki['sql_query'] - - # Create a cursor and execute the query. - cursor = self.conn.cursor(named_tuple=True) - cursor.execute(query, sql_bindings) - - result_binding_set = [] - for row in cursor: - binding = dict() - for variable in ki['vars']: - # Get the value of the current variable from the current row. - value = getattr(row, variable) - - # Check the type of the value and set the datatype accordingly. - # TODO: Support more data types? - if variable in ki['column_prefixes']: - prefix = ki['column_prefixes'][variable] - typed_value = f"<{prefix}{value}>" - elif isinstance(value, int): - typed_value = f"\"{value}\"^^" - else: - # Fall back to a string literal. - typed_value = f"\"{value}\"" - - binding[variable] = typed_value - - result_binding_set.append(binding) - - cursor.close() - - # The connection caches query results if not committed and this resulted in - # subsequent queries not containing the latest changes to the tables. - self.conn.commit() - - - return result_binding_set - - - def handle_react(self, ki, binding_set, requesting_kb): - if binding_set: - for binding in binding_set: - for statement in ki['sql_query']: - variables = statement['bindings'] - sql_binding = () - for variable in variables: - value = binding[variable] - # If prefixed, remove the <>'s and the prefix, otherwise, - # use `value` as is. - if variable in ki['column_prefixes']: - prefix = ki['column_prefixes'][variable] - sql_binding += (value[len(prefix) + 1:-1],) - elif value[0] == '"' and value[-1] == '"': - # This is a string literal. We want to use the - # string INSIDE the quotes for the SQL binding. - sql_binding += (value[1:-1],) - else: - # TODO: Handle other datatypes correctly if they're - # given with "value"^^:datatype syntax. - sql_binding += (value,) - - # Create a cursor and execute the query. - try: - cursor = self.conn.cursor() - cursor.execute(statement['statement'], sql_binding) - except mysql.connector.Error as e: - log.warn(e) - self.conn.rollback() - continue - # Only commit after a complete query (with multiple statements) - # is executed. - self.conn.commit() - return [] diff --git a/src/knowledge_mapper/legacy/tests/__init__.py b/src/knowledge_mapper/legacy/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/knowledge_mapper/legacy/tests/legacy_test_tke_client.py b/src/knowledge_mapper/legacy/tests/legacy_test_tke_client.py deleted file mode 100644 index 930c9b0..0000000 --- a/src/knowledge_mapper/legacy/tests/legacy_test_tke_client.py +++ /dev/null @@ -1,137 +0,0 @@ -import src.knowledge_mapper.kb.knowledge_base as tke_kb -import src.knowledge_mapper.knowledge_interaction as tke_ki -import src.tke_client as tke -import pytest -import asyncio -import requests -from os import environ - -from uuid import uuid4 - -if 'KE_RUNTIME_URL' in environ: - ke_runtime_url = environ['KE_RUNTIME_URL'] -else: - ke_runtime_url = 'http://localhost:8280/rest' - -print(f'Using KE runtime at {ke_runtime_url} for tests.') - -def generate_kb_id_and_name(prefix='https://example.org/'): - random_characters = uuid4().hex - return (f'{prefix}{random_characters}', random_characters) - - -@pytest.mark.asyncio -async def test_ask_answer(): - - kb1_id, kb1_name = generate_kb_id_and_name() - kb2_id, kb2_name = generate_kb_id_and_name() - - answer_ki_registered = asyncio.Event() - - # Next up are two coroutines that have to run asynchronously, because they - # interact with eachother via the KE. - - async def kb1_task(): - client_1 = tke.TkeClient(ke_runtime_url) - client_1.connect() - kb1 = client_1.register(tke_kb.KnowledgeBaseRegistrationRequest(id=kb1_id, name=kb1_name, description="KB 1")) - - ki_name = 'things-that-like-other-things' - ask_ki: tke_ki.AskKnowledgeInteraction = kb1.register_knowledge_interaction(tke_ki.AskKnowledgeInteractionRegistrationRequest(prefixes={'ex': 'http://example.org/'}, pattern='?a ex:likes ?b'), ki_name) - - # Make sure that the KIs name is in the KI's ID. - assert(kb1.id + '/interaction/' + ki_name in requests.get( - ke_runtime_url + '/sc/ki', - headers = { - 'Knowledge-Base-Id': kb1.id - } - ).text) - - # Wait for the other KI to tell us that it has been registered. - await answer_ki_registered.wait() - - # Trigger the knowledge interaction on the default executor. - result = await asyncio.get_event_loop().run_in_executor( - None, - lambda: ask_ki.ask([]) - ) - - # Assert some things - bindings = result['bindingSet'] - assert len(bindings) == 1 - assert bindings[0]['a'] == '' - assert bindings[0]['b'] == '"coffee"' - - kb1.unregister() - - async def kb2_task(): - client_2 = tke.TkeClient(ke_runtime_url) - client_2.connect() - kb2 = client_2.register(tke_kb.KnowledgeBaseRegistrationRequest(id=kb2_id, name=kb2_name, description="KB 2")) - - def handler(bindings: dict, requesting_kb_id: str) -> dict: - assert len(bindings) == 0 - assert requesting_kb_id == kb1_id - return [{'c': '', 'd': '"coffee"'}] - - kb2.register_knowledge_interaction( - tke_ki.AnswerKnowledgeInteractionRegistrationRequest( - prefixes={'ex': 'http://example.org/'}, - pattern='?c ex:likes ?d', - handler=handler - ), - ) - - # Signal that the ANSWER KI has been registered to the other task in - # this test. - answer_ki_registered.set() - - # Make this knowledge base's client long poll for 1 incoming knowledge - # request. - await asyncio.get_event_loop()\ - .run_in_executor(None, lambda: kb2.start_handle_loop(1)) - - kb2.unregister() - - # Wait for both tasks to complete. (They are scheduled not sequentially, but - # asynchronously.) - await asyncio.gather(kb1_task(), kb2_task()) - -def test_reregister(): - kb_id, kb_name = generate_kb_id_and_name() - client = tke.TkeClient(ke_runtime_url) - client.connect() - - kb1 = client.register(tke_kb.KnowledgeBaseRegistrationRequest(id=kb_id, name=kb_name, description="KB that will be reregistered")) - assert kb1 is not None - - # Try to register with the same id, and the reregister flag turned OFF - kb2 = client.register(tke_kb.KnowledgeBaseRegistrationRequest(id=kb_id, name=kb_name, description="KB that will be reregistered"), reregister=False) - assert kb2 is None - - # Try to register with the same id, and the reregister flag turned ON - kb3 = client.register(tke_kb.KnowledgeBaseRegistrationRequest(id=kb_id, name=kb_name, description="KB that will be reregistered"), reregister=True) - assert kb3 is not None - - # We only have to clean up the final one since it uses the same ID - kb3.unregister() - -def test_reconnect_retains_kis(): - try: - kb_id, kb_name = generate_kb_id_and_name() - client1 = tke.TkeClient(ke_runtime_url) - client1.connect() - - kb1 = client1.register(tke_kb.KnowledgeBaseRegistrationRequest(id=kb_id, name=kb_name, description="KB that will be reregistered")) - kb1.register_knowledge_interaction(tke_ki.AskKnowledgeInteractionRegistrationRequest(prefixes={'ex': 'http://example.org/'}, pattern='?a ex:likes ?b'), 'an-ask-ki') - assert kb1 is not None - - client2 = tke.TkeClient(ke_runtime_url) - client2.connect() - kb1_that_reconnects = client2.get_knowledge_base(kb_id) - assert kb1_that_reconnects.id == kb_id - - assert kb1_that_reconnects.kis == kb1.kis - assert kb1_that_reconnects.kis_by_name == kb1.kis_by_name - finally: - kb1_that_reconnects.unregister() diff --git a/src/knowledge_mapper/legacy/tke_client.py b/src/knowledge_mapper/legacy/tke_client.py deleted file mode 100644 index 5c5e797..0000000 --- a/src/knowledge_mapper/legacy/tke_client.py +++ /dev/null @@ -1,102 +0,0 @@ -import os -import requests -import logging as log -import time - -import src.knowledge_mapper.kb.knowledge_base as knowledge_base - -from src.tke_exceptions import UnexpectedHttpResponseError - -MAX_CONNECTION_ATTEMPTS = 10 -WAIT_BEFORE_RETRY = 1 - -if "ENABLE_REASONER" in os.environ: - REASONER_ENABLED = os.environ.get("ENABLE_REASONER").lower() in ["true", "1"] -else: - REASONER_ENABLED = False - - -class TkeClient: - def __init__(self, ke_url: str): - self.ke_url = ke_url - - def connect( - self, - max_attempts=MAX_CONNECTION_ATTEMPTS, - wait_between_attempts=WAIT_BEFORE_RETRY, - ): - attempts = 0 - success = False - while not success: - try: - attempts += 1 - self.get_knowledge_bases() - success = True - except requests.exceptions.ConnectionError: - log.warning(f"Connecting to {self.ke_url} failed.") - - if not success and (max_attempts is None or attempts < max_attempts): - log.warning( - f"Request to {self.ke_url} failed after attempt {attempts}. Retrying in {wait_between_attempts} s." - ) - time.sleep(wait_between_attempts) - elif not success: - raise Exception( - f"Request to {self.ke_url} failed. Gave up after {attempts} attempts." - ) - log.info(f"Successfully connected to {self.ke_url}.") - - def get_knowledge_bases(self) -> list[knowledge_base.KnowledgeBase]: - response = requests.get(f"{self.ke_url}/sc") - - if not response.ok: - raise UnexpectedHttpResponseError(response) - - return [ - knowledge_base.KnowledgeBase.from_json(kb_data, self.ke_url) - for kb_data in response.json() - ] - - def get_knowledge_base(self, kb_id: str) -> knowledge_base.KnowledgeBase | None: - response = requests.get( - f"{self.ke_url}/sc", headers={"Knowledge-Base-Id": kb_id} - ) - - if response.status_code == 404: - return None - elif not response.ok: - raise UnexpectedHttpResponseError(response) - - return knowledge_base.KnowledgeBase.from_json(response.json()[0], self.ke_url) - - def register( - self, req: knowledge_base.KnowledgeBaseRegistrationRequest, reregister=True - ) -> knowledge_base.KnowledgeBase | None: - already_existing = self.get_knowledge_base(req.id) - if already_existing is not None: - if reregister: - already_existing.unregister() - else: - return None - - body = { - "knowledgeBaseId": req.id, - "knowledgeBaseName": req.name, - "knowledgeBaseDescription": req.description, - } - - if REASONER_ENABLED: - body["reasonerEnabled"] = True - - response = requests.post( - f"{self.ke_url}/sc", - json=body, - ) - if not response.ok: - raise UnexpectedHttpResponseError(response) - - return knowledge_base.KnowledgeBase(req, self.ke_url) - - -class CleanUpFailedError(RuntimeError): - pass diff --git a/src/knowledge_mapper/legacy/tke_exceptions.py b/src/knowledge_mapper/legacy/tke_exceptions.py deleted file mode 100644 index 1ba0bb6..0000000 --- a/src/knowledge_mapper/legacy/tke_exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -from requests.models import Response - -class UnexpectedHttpResponseError(Exception): - def __init__(self, response: Response): - super().__init__(f'Unexpected response from {response.url} with status {response.status_code} and body "{response.text}".') diff --git a/src/knowledge_mapper/legacy/utils.py b/src/knowledge_mapper/legacy/utils.py deleted file mode 100644 index e858d03..0000000 --- a/src/knowledge_mapper/legacy/utils.py +++ /dev/null @@ -1,32 +0,0 @@ -from rdflib import Graph, URIRef, Variable - - -def match_bindings(query: list[dict], source: list[dict]) -> list: - matches = [] - for s in source: - for q in query: - q_matches = True - for k, v in q.items(): - if not (k in s and s[k] == v): - q_matches = False - break - if q_matches: - matches.append(s.copy()) - break - return matches - - -def extract_variables(graph_pattern, prefixes=dict()): - """Given a graph pattern, returns a set with the variables (strings) that - are used in the graph pattern""" - g = Graph() - # Wrap the graph pattern in a SELECT query - query = f"SELECT * WHERE {{ {graph_pattern} }}" - # Parse the query into an RDFLib query object - q = g.query(query, initNs=prefixes) - # Extract the variables from the query - variables = set() - for var in q.vars: - if isinstance(var, Variable): - variables.add(var.n3()[1:]) - return variables diff --git a/src/knowledge_mapper/legacy/wizard_mapper.py b/src/knowledge_mapper/legacy/wizard_mapper.py deleted file mode 100644 index d8dae78..0000000 --- a/src/knowledge_mapper/legacy/wizard_mapper.py +++ /dev/null @@ -1,160 +0,0 @@ -import logging -import time -import os -import requests - -from src.knowledge_mapper.kb.knowledge_base import ( - KnowledgeEngineTerminated, -) -from src.utils import match_bindings - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -class KnowledgeBaseNotFoundInApi(Exception): - pass - - -class ApiNotReachable(Exception): - pass - - -def start(): - ke_endpoint = os.environ.get("KE_ENDPOINT") - time.sleep(5) - km_api = os.environ.get("KM_API") - if km_api is None: - logger.error("environment variable KM_API is required when using --wizard") - exit(1) - else: - logger.info(f"using KM_API {km_api}") - - have_kb = False - kb = None - while True: - while kb is None: - try: - resp = requests.get(f"{km_api}/knowledge-bases") - except: - logger.warning(f"KM API cannot be reached. Retrying in 2 seconds.") - time.sleep(2) - continue - assert resp.ok - kbs = resp.json()["data"] - assert len(kbs) <= 2 - if kbs: - kb = kbs[0] - else: - logger.info(f"waiting for user to register knowledge base") - time.sleep(2) - logger.info(f"found knowledge base with ID {kb['id']}") - - kb_disappeared_from_api = False - while not kb_disappeared_from_api: - k_req = wait_for_knowledge_request(kb["id_url"]) - logger.debug(f"received knowledge request: {k_req}") - ki_id = k_req["knowledgeInteractionId"] - handle_id = k_req["handleRequestId"] - binding_set = k_req["bindingSet"] - requesting_kb_id = k_req["requestingKnowledgeBaseId"] - - try: - result_binding_set = map_knowledge_request( - kb["id"], ki_id, binding_set, requesting_kb_id - ) - except KnowledgeBaseNotFoundInApi: - result_binding_set = [] - kb_disappeared_from_api = True - except: - logger.exception("something went wrong while mapping knowledge request") - result_binding_set = [] - - post_response(kb["id_url"], ki_id, handle_id, result_binding_set) - - if kb_disappeared_from_api: - response = requests.delete( - f"{ke_endpoint}/sc", headers={"Knowledge-Base-Id": kb["id_url"]} - ) - assert response.ok - kb = None - - -def wait_for_knowledge_request(kb_id): - ke_endpoint = os.environ.get("KE_ENDPOINT") - - while True: - response = requests.get( - f"{ke_endpoint}/sc/handle", headers={"Knowledge-Base-Id": kb_id} - ) - if response.status_code == 200: - return response.json() - elif response.status_code == 202: - logger.debug("repolling...") - continue - elif response.status_code == 404: - logger.warning("Our Knowledge Base has been unregistered!") - raise KnowledgeBaseNotFoundInApi() - elif response.status_code == 410: - logger.warning("The Knowledge Engine REST server terminated!") - raise KnowledgeEngineTerminated() - elif response.status_code == 500: - logger.error(response.text) - logger.error("TKE had an internal server error. Reinitiating long poll.") - continue - else: - logger.warning( - f"long_poll received unexpected status {response.status_code}" - ) - logger.warning(response.text) - logger.warning("repolling anyway..") - continue - - -def post_response(kb_id, ki_id, handle_request_id, binding_set): - ke_endpoint = os.environ.get("KE_ENDPOINT") - response = requests.post( - f"{ke_endpoint}/sc/handle", - headers={"Knowledge-Base-Id": kb_id, "Knowledge-Interaction-Id": ki_id}, - json={"handleRequestId": handle_request_id, "bindingSet": binding_set}, - ) - if not response.ok: - logger.error("Failed to post a response to knowledge request!") - - -def map_knowledge_request(my_kb_id, ki_id, binding_set, requesting_kb_id): - mapping_rule = get_mapping_rule(my_kb_id, ki_id) - return map(mapping_rule, binding_set) - - -def get_mapping_rule(my_kb_id, ki_id): - km_api = os.environ.get("KM_API") - try: - response = requests.get( - f"{km_api}/knowledge-bases/{my_kb_id}/data-sources/", - params={"knowledgeInteractionId": ki_id, "includeMapping": True}, - ) - except: - raise ApiNotReachable() - if not response.ok: - if response.status_code == 404: - raise KnowledgeBaseNotFoundInApi() - logger.error(response.status_code) - logger.error(response.text) - raise Exception("Request failed!") - body = response.json() - if len(body["data"]) != 1: - raise Exception( - f"Expected 1 data source, but found {len(body['data'])}: {body['data']}" - ) - data_source = body["data"][0] - return data_source["mapping_rule"] - - -def map(mapping_rule, binding_set): - if mapping_rule["type"] != "StaticTable": - raise Exception(f"Unsupported mapping rule type {mapping_rule['type']}!") - source_bindings = mapping_rule["data"] - if len(binding_set) == 0: - binding_set = [dict()] - return match_bindings(binding_set, source_bindings)