feat Nextcloudclient

README.md

@@ -64,6 +64,44 @@ docker run --rm -v $(pwd):/data dbpedia/databus-python-client download https://d
 A docker image is available at [dbpedia/databus-python-client](https://hub.docker.com/r/dbpedia/databus-python-client). See [download section](#usage-of-docker-image) for details.
 
 
+## Deploy to Databus
+Please add databus API_KEY to .env file
+Use metadata.json file to list all files which should be added to the databus
+
+The script registers all files on the databus.
+### Example Call
+```bash
+python -m databusclient.deploy \
+  --no-upload \
+  --metadata ./metadata.json \
+  --version-id https://databus.org/user/dataset/version/1.0 \
+  --title "Test Dataset" \
+  --abstract "This is a short abstract of the test dataset." \
+  --description "This dataset was uploaded for testing the Nextcloud → Databus deployment pipeline." \
+  --license https://dalicc.net/licenselibrary/Apache-2.0
+
+```
+
+## Upload to Nextcloud and Deploy to Databus
+Please add databus API_KEY to .env file
+
+The script uploads all given files and all files in the given folders to the given remote.
+Then registers them on the databus.
+### Example Call
+```bash
+python -m databusclient.deploy \
+--webdav-url https://cloud.scadsai.uni-leipzig.de/remote.php/webdav \
+--remote scads-nextcloud \
+--path test \
+--version-id https://databus.dbpedia.org/gg46ixav/test_group/test_artifact/2023-07-03 \
+--title "Test Dataset" \
+--abstract "This is a short abstract of the test dataset." \
+--description "This dataset was uploaded for testing the Nextcloud → Databus deployment pipeline." \
+--license https://dalicc.net/licenselibrary/Apache-2.0 \
+/home/CSVTest/newtestoutputfolder \
+/home/CSVTest/output.csv.bz2
+
+```
 ## CLI Usage
 
 **Installation**

databusclient/client.py

@@ -343,7 +343,7 @@ def append_to_dataset_graph_if_existent(add_key: str, add_value: str):
     graphs.append(dataset_graph)
 
     dataset = {
-        "@context": "https://downloads.dbpedia.org/databus/context.jsonld",
+        "@context": "https://databus.dbpedia.org/res/context.jsonld",
         "@graph": graphs,
     }
     return dataset

databusclient/deploy.py

@@ -0,0 +1,110 @@
+import os
+import sys
+import argparse
+import json
+
+from databusclient import create_distribution, create_dataset, deploy
+from dotenv import load_dotenv
+
+from nextcloudclient.upload import upload_to_nextcloud
+
+def deploy_to_databus(
+    metadata,
+    version_id,
+    title,
+    abstract,
+    description,
+    license_url
+):
+    load_dotenv()
+    api_key = os.getenv("API_KEY")
+    if not api_key:
+        raise ValueError("API_KEY not found in .env")
+
+    distributions = []
+    counter = 0
+    for filename, checksum, size, url in metadata:
+        # Expect a SHA-256 hex digest (64 chars). Reject others.
+        if not isinstance(checksum, str) or len(checksum) != 64:
+            raise ValueError(f"Invalid checksum for {filename}: expected SHA-256 hex (64 chars), got '{checksum}'")
+        parts = filename.split(".")
+        if len(parts) == 1:
+            file_format = "none"
+            compression = "none"
+        elif len(parts) == 2:
+            file_format = parts[-1]
+            compression = "none"
+        else:
+            file_format = parts[-2]
+            compression = parts[-1]
+
+        distributions.append(
+            create_distribution(
+                url=url,
+                cvs={"count": f"{counter}"},
+                file_format=file_format,
+                compression=compression,
+                sha256_length_tuple=(checksum, size)
+            )
+        )
+        counter += 1
+
+    dataset = create_dataset(
+        version_id=version_id,
+        title=title,
+        abstract=abstract,
+        description=description,
+        license_url=license_url,
+        distributions=distributions
+    )
+
+    deploy(dataset, api_key)
+    metadata_string = ",\n".join([entry[-1] for entry in metadata])
+
+    print(f"Successfully deployed\n{metadata_string}\nto databus {version_id}")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Upload files to Nextcloud and deploy to DBpedia Databus.")
+    parser.add_argument("files", nargs="*", help="Path(s) to file(s) or folder(s) to upload")
+    parser.add_argument("--webdav-url", help="WebDAV URL (e.g., https://cloud.example.com/remote.php/webdav)")
+    parser.add_argument("--remote", help="rclone remote name (e.g., 'nextcloud')")
+    parser.add_argument("--path", help="Remote path on Nextcloud (e.g., 'datasets/mydataset')")
+    parser.add_argument("--no-upload", action="store_true", help="Skip file upload and use existing metadata")
+    parser.add_argument("--metadata", help="Path to metadata JSON file (required if --no-upload is used)")
+
+    parser.add_argument("--version-id", required=True, help="Databus version URI")
+    parser.add_argument("--title", required=True, help="Title of the dataset")
+    parser.add_argument("--abstract", required=True, help="Short abstract of the dataset")
+    parser.add_argument("--description", required=True, help="Detailed description of the dataset")
+    parser.add_argument("--license", required=True, help="License URL (e.g., https://dalicc.net/licenselibrary/Apache-2.0)")
+
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    if args.no_upload:
+        if not args.metadata:
+            print("Error: --metadata is required when using --no-upload")
+            sys.exit(1)
+        if not os.path.isfile(args.metadata):
+            print(f"Error: Metadata file not found: {args.metadata}")
+            sys.exit(1)
+        with open(args.metadata, 'r') as f:
+            metadata = json.load(f)
+    else:
+        if not (args.webdav_url and args.remote and args.path):
+            print("Error: --webdav-url, --remote, and --path are required unless --no-upload is used")
+            sys.exit(1)
+        metadata = upload_to_nextcloud(args.files, args.remote, args.path, args.webdav_url)
+
+    deploy_to_databus(
+        metadata,
+        version_id=args.version_id,
+        title=args.title,
+        abstract=args.abstract,
+        description=args.description,
+        license_url=args.license
+    )

databusclient/metadata.json

@@ -0,0 +1,14 @@
+[
+  [
+    "example.ttl",
+    "6e340b9cffb37a989ca544e6bb780a2c7e5d7dcb",
+    12345,
+    "https://cloud.example.com/remote.php/webdav/datasets/mydataset/example.ttl"
+  ],
+  [
+    "example.csv.gz",
+    "3f786850e387550fdab836ed7e6dc881de23001b",
+    54321,
+    "https://cloud.example.com/remote.php/webdav/datasets/mydataset/example.csv.gz"
+  ]
+]

nextcloudclient/upload.py

@@ -0,0 +1,77 @@
+import hashlib
+import os
+import subprocess
+import posixpath
+from urllib.parse import urljoin, quote
+
+
+def compute_sha256_and_length(filepath):
+    sha256 = hashlib.sha256()
+    total_length = 0
+    with open(filepath, 'rb') as f:
+        while True:
+            chunk = f.read(4096)
+            if not chunk:
+                break
+            sha256.update(chunk)
+            total_length += len(chunk)
+    return sha256.hexdigest(), total_length
+
+def get_all_files(path):
+    if os.path.isfile(path):
+        return [path]
+    files = []
+    for root, _, filenames in os.walk(path):
+        for name in filenames:
+            files.append(os.path.join(root, name))
+    return files
+
+def upload_to_nextcloud(source_paths: list[str], remote_name: str, remote_path: str, webdav_url: str):
+    result = []
+    for path in source_paths:
+        if not os.path.exists(path):
+            print(f"Path not found: {path}")
+            continue
+
+        abs_path = os.path.abspath(path)
+        basename = os.path.basename(abs_path)
+        files = get_all_files(abs_path)
+
+        tmp_results = []
+
+        for file in files:
+            checksum,size  = compute_sha256_and_length(file)
+
+            if os.path.isdir(path):
+                rel_file = os.path.relpath(file, abs_path)
+                # Normalize to POSIX for WebDAV/URLs
+                rel_file = rel_file.replace(os.sep, "/")
+                remote_webdav_path = posixpath.join(remote_path, basename, rel_file)
+            else:
+                remote_webdav_path = posixpath.join(remote_path, os.path.basename(file))
+
+            # Preserve scheme/host and percent-encode path segments
+            url = urljoin(webdav_url.rstrip("/") + "/", quote(remote_webdav_path.lstrip("/"), safe="/"))
+
+            filename = os.path.basename(file)
+            tmp_results.append((filename, checksum, size, url))
+
+        dest_subpath = posixpath.join(remote_path.lstrip("/"), basename)
+        if os.path.isdir(path):
+            destination = f"{remote_name}:{dest_subpath}"
+            command = ["rclone", "copy", abs_path, destination, "--progress"]
+        else:
+            destination = f"{remote_name}:{dest_subpath}"
+            command = ["rclone", "copyto", abs_path, destination, "--progress"]
+
+        print(f"Upload: {path} → {destination}")
+        try:
+            subprocess.run(command, check=True)
+            result.extend(tmp_results)
+            print("✅ Uploaded successfully.\n")
+        except subprocess.CalledProcessError as e:
+            print(f"❌ Error uploading {path}: {e}\n")
+        except FileNotFoundError:
+            print("❌ rclone not found on PATH. Install rclone and retry.")
+
+    return result

pyproject.toml

@@ -12,6 +12,7 @@ click = "^8.0.4"
 requests = "^2.28.1"
 tqdm = "^4.42.1"
 SPARQLWrapper = "^2.0.0"
+python-dotenv = "^1.1.1"
 rdflib = "^7.2.1"
 
 [tool.poetry.group.dev.dependencies]