diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 46b708483..0f3043871 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -224,6 +224,7 @@ jobs: - acs-admin - acs-cluster-manager - acs-cmdesc + - acs-data-access - acs-edge-sync - acs-files - acs-git diff --git a/Makefile b/Makefile index d7ce30691..671ff34c0 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,7 @@ subdirs+= acs-auth subdirs+= acs-cluster-manager subdirs+= acs-cmdesc subdirs+= acs-configdb +subdirs+= acs-data-access subdirs+= acs-directory subdirs+= acs-edge subdirs+= acs-edge-sync @@ -67,4 +68,5 @@ subdirs+= historian-sparkplug subdirs+= historian-uns subdirs+= uns-ingester-sparkplug + include ${mk}/acs.subdirs.mk diff --git a/acs-data-access/.dockerignore b/acs-data-access/.dockerignore new file mode 100644 index 000000000..40b878db5 --- /dev/null +++ b/acs-data-access/.dockerignore @@ -0,0 +1 @@ +node_modules/ \ No newline at end of file diff --git a/acs-data-access/.gitignore b/acs-data-access/.gitignore new file mode 100644 index 000000000..c7b916f01 --- /dev/null +++ b/acs-data-access/.gitignore @@ -0,0 +1,3 @@ +.env +node_modules/ +tmp/ \ No newline at end of file diff --git a/acs-data-access/Dockerfile b/acs-data-access/Dockerfile new file mode 100644 index 000000000..940f69306 --- /dev/null +++ b/acs-data-access/Dockerfile @@ -0,0 +1,32 @@ +ARG base_version +ARG base_prefix=ghcr.io/amrc-factoryplus/acs-base + +FROM ${base_prefix}-js-build:${base_version} AS build +ARG acs_npm=NO + +# Install the node application on the build container where we can +# compile the native modules. +RUN install -d -o node -g node /home/node/app \ + && mkdir /home/node/lib +COPY --from=lib . /home/node/lib/ + +WORKDIR /home/node/app +USER node +COPY package*.json ./ +RUN <<'SHELL' + touch /home/node/.npmrc + npm install --save=false --install-links +SHELL +COPY --chown=node . . +RUN <<'SHELL' + echo "export const GIT_VERSION=\"$revision\";" > ./lib/git-version.js +SHELL + +FROM ${base_prefix}-js-run:${base_version} + +# Copy across from the build container. +WORKDIR /home/node/app +COPY --from=build --chown=root:root /home/node/app ./ + +USER node +CMD node bin/api.js diff --git a/acs-data-access/Makefile b/acs-data-access/Makefile new file mode 100644 index 000000000..2cab50f13 --- /dev/null +++ b/acs-data-access/Makefile @@ -0,0 +1,7 @@ +top=.. +include ${top}/mk/acs.init.mk + +repo?=acs-data-access +k8s.deployment?=data-access + +include ${mk}/acs.js.mk diff --git a/acs-data-access/bin/api.js b/acs-data-access/bin/api.js new file mode 100644 index 000000000..c2cb8befc --- /dev/null +++ b/acs-data-access/bin/api.js @@ -0,0 +1,95 @@ +#!/usr/bin/env node + +/* +* ACS Data Access Service +* Entry point: api.js +*/ + +import {InfluxDB, flux} from '@influxdata/influxdb-client'; + +import { RxClient, UUIDs } from '@amrc-factoryplus/rx-client'; +import { WebAPI } from '@amrc-factoryplus/service-api'; +import {DataFlow} from '../lib/dataflow.js'; +import { APIv1 } from '../lib/api-v1.js'; +import {DataAccessNotify} from '../lib/notify.js'; +import { InfluxReader } from '../lib/influx-reader.js'; + +const { env } = process; + +const fplus = await new RxClient({ + env, +}).init(); + +const Version = "2.0.0"; + +const debug = fplus.debug; +debug.log("app", "Starting acs-data-service revision %s"); + +const influxClient = new InfluxDB({ + url: env.INFLUXDB_URL, + token: env.INFLUXDB_TOKEN +}); + + +// the dataflow object is full of sequences that does most of the work of this service. Sequence means an RX observable. +const data = new DataFlow({ + debug, + cdb: fplus.ConfigDB, + auth: fplus.Auth, +}); + +const influxReader = new InfluxReader({ + debug, + influx_client: influxClient, + influx_org: env.INFLUXDB_ORG, + influx_bucket: env.INFLUXDB_BUCKET, +}) + +const apiv1 = new APIv1({ + data, + debug, + auth: fplus.Auth, + cdb: fplus.ConfigDB, + influxReader +}); + +const api = await new WebAPI({ + debug: fplus.debug, + ping: { + version: Version, + service: UUIDs.Service.DataAccess, + software: { + vendor: 'AMRC', + application: 'acs-data-access', + }, + }, + realm: env.REALM, + hostname: env.HOSTNAME, + keytab: env.SERVER_KEYTAB, + http_port: env.PORT, + max_age: env.CACHE_MAX_AGE, + routes: app => { + app.use("/v1", apiv1.routes); + } + +}).init(); + + +const notify = new DataAccessNotify({ + api, data, debug, + auth: fplus.Auth +}); + +debug.log("app", "Running Data Access DataFlow"); +data.run(); + +debug.log("app", "Running Data Access Notify"); +notify.run(); + +debug.log("app", "Running Data Access WebAPI") +api.run(); + + + + + diff --git a/acs-data-access/lib/api-v1.js b/acs-data-access/lib/api-v1.js new file mode 100644 index 000000000..f577ee3b1 --- /dev/null +++ b/acs-data-access/lib/api-v1.js @@ -0,0 +1,648 @@ +/* +* ACS Data Access Service +* APIv1 +*/ + +import express from "express"; +import { Map as IMap, Seq as ISeq, merge } from "immutable"; +import * as rx from "rxjs"; + +import { APIError } from "@amrc-factoryplus/service-api"; +import { ServiceError } from "@amrc-factoryplus/service-client"; +import { retryBackoff } from "@amrc-factoryplus/rx-util"; +import { DataAccess as Constants } from "./constants.js"; +import { valid_uuid, valid_datetime } from "./validate.js"; +import { csv_escape, maxDate, minDate } from './utils.js'; + +function fail(log, status, message) { + log(message); + throw new APIError(status); + } + +export class APIv1 { + constructor(opts) { + this.data = opts.data; + this.auth = opts.auth; + this.cdb = opts.cdb; + this.log = opts.debug.bound("apiv1"); + this.influxReader = opts.influxReader; + this.routes = this.setup_routes(); + } + + setup_routes() { + let api = express.Router(); + + api.route("/metadata") + .get(this.metadata_list.bind(this)); + + api.route("/metadata/:uuid") + .get(this.metadata_uuid.bind(this)); + + api.route("/data/:uuid") + .post(this.dataset_data.bind(this)); + + api.route("/structure") + .get(this.structure_list.bind(this)) + .post(this.structure_create.bind(this)); + + api.route("/structure/:uuid") + .get(this.structure_uuid.bind(this)) + .put(this.structure_update.bind(this)); + + api.route("/delete/:uuid") + .get(this.delete_dataset.bind(this)); + + return api; + } + + async delete_dataset(req, res){ + const dataset_uuid = req.params.uuid; + if(!dataset_uuid) return fail(this.log, 422, `No req.params.uuid`); + if(!valid_uuid(dataset_uuid)) return fail(this.log, 422, `Invalid uuid ${dataset_uuid}`); + + const ok = await this.auth.check_acl( + req.auth, + Constants.Perm.DeleteDataset, + dataset_uuid, + true, + ); + + if (!ok) return fail(this.log, 403, `You don't have DELETE permissions for ${dataset_uuid}`); + + this.log(`Delete dataset called by ${req.auth} for ${dataset_uuid}`); + + // remove all subclass relationships + + const subclasses = await this.cdb.class_subclasses(dataset_uuid); + if(subclasses){ + for(let s of subclasses){ + await this.cdb.class_remove_subclass(dataset_uuid, s); + } + } + + await this.cdb.delete_object(dataset_uuid); + + return res.status(200).json(dataset_uuid); + } + + /** GET. Returns a list of Dataset UUIDs that the client has READ_DATASET access to. + * The dataset can be optionally restricted by from and to dates (this is not implemented yet). + * Dates must be in the format ISO date-time string in UTC 2025-11-13T09:33:18.000Z + * @param from {date} (optional, inclusive) query param + * @param to {date} (optional, inclusive) query param + * Don't return invalid datasets + */ + async metadata_list(req, res) { + const uuids = await rx.firstValueFrom(this.data.allowed_valid_dataset_uuids(req.auth, Constants.Perm.ReadDataset)); + return res.status(200).json(uuids); + } + + /** GET. Accepts Dataset UUID and returns metadata about a Published dataset. + * @param uuid {request param} + * @param + * @returns metadata - JSON object with properties: + * uuid {UUID} - Dataset UUID + * name {string} - Dataset name from General Information + * from {date} - Dataset starting bound - derived from bounds of dataset Sessions (if any) + * to {date} - Dataset finishing bound - derived from bounds of dataset Sessions (if any) + * function {array} - Array of Functional classes - UUIDs of classes dataset belongs to (only members of Functional dataset type) + * metadata {object} - Map of metadata configs keyed by Application UUID - contains all config entries for dataset which use applications in the Dataset metadata class + * parts {array} - Subset datasets - contains UUIDs of all subclasses of dataset the client has READ access to. + * Returns 404 if dataset is invalid. + */ + async metadata_uuid(req, res){ + const dataset_uuid = req.params.uuid; + if (!valid_uuid(dataset_uuid)) fail(this.log, 422, `${dataset_uuid} is invalid uuid.`); + + const ok = await this.auth.check_acl( + req.auth, + Constants.Perm.ReadDataset, + dataset_uuid, + true, + ); + + if (!ok) return fail(this.log, 403, `You don't have READ permissions for ${dataset_uuid}`); + + + const [ + datasets, + infos, + all_f_types, + all_metadata, + parts + ] = await Promise.all([ + rx.firstValueFrom(this.data.allowed_valid_datasets(req.auth, Constants.Perm.ReadDataset)), + rx.firstValueFrom(this.data.general_infos), + rx.firstValueFrom(this.data.functional_types), + rx.firstValueFrom(this.data.metadata), + rx.firstValueFrom( + this.data.allowed_dataset_parts(dataset_uuid, req.auth, Constants.Perm.ReadDataset) + ) + ]); + + const dataset = datasets.get(dataset_uuid); + if(!dataset) return fail(this.log, 404, `Dataset ${dataset_uuid} is not found or invalid.`); + + const info = infos.get(dataset_uuid); + + const meta = { + uuid: dataset_uuid, + name: info?.name ? info.name : "UNKNOWN", + from: dataset.from ? dataset.from : undefined, + to: dataset.to ? dataset.to : undefined, + function: all_f_types.get(dataset_uuid), + metadata: all_metadata.get(dataset_uuid), + parts + } + return res.status(200).json(meta); + } + + + // Check if principal has necessary permission to referenced source + async _check_second_level_permission(principal, structure, config){ + + if( structure == Constants.App.SparkplugSrc){ + + const target = config.source; + if(!target) return fail(this.log, 422, `Dataset definition does not contain source.`); + + if(!valid_uuid(target)) return fail(this.log, 422, `Source uuid ${target} is invalid.`); + + const ok = await this.auth.check_acl( + principal, + Constants.Perm.UseSparkplug, + target, + true + ); + + return ok; + } + + if(structure == Constants.App.SessionLimits){ + + const target = config.source; + if(!target) return fail(this.log, 422, `Dataset definition does not contain source.`); + + if(!valid_uuid(target)) return fail(this.log, 422, `Source uuid ${target} is invalid.`); + + const ok = await this.auth.check_acl( + principal, + Constants.Perm.UseForSession, + target, + true + ); + + return ok; + + }else if(structure == Constants.App.UnionComponents){ + if(config.length == 0) return true; + + if(!Array.isArray(config)) return fail(this.log, 422, `Dataset def of structure ${structure} must be an Array.`); + + for(let target of config){ + + const ok = await this.auth.check_acl( + principal, + Constants.Perm.IncludeInUnion, + target, + true + ); + + if(!ok) return false; + } + + return true; + + }else{ + // Unhandled Structure type + return fail(this.log, 422, `Structure ${structure} is unknown`); + } + } + + + async _resolve_dataset( + dataset_uuid, + inherited_range = {from: null, to: null}, + visited = new Set(), + ){ + // prevent circular references + if(visited.has(dataset_uuid)) return fail(this.log, 404, `Circular dataset reference detected ${dataset_uuid}`); + + visited.add(dataset_uuid); + + const datasets = await rx.firstValueFrom(this.data.datasets); + const dataset = datasets.get(dataset_uuid); + + if(!dataset) return fail(this.log, 404, `Dataset not found ${dataset_uuid}`); + + const {structure, config} = dataset; + if(!structure) return fail(this.log, 404, `Structure not found for dataset ${dataset_uuid}`); + + if(structure === Constants.Special.InvalidDataset) return fail(this.log, 404, `Invalid dataset ${dataset_uuid}`); + + if(!config) return fail(this.log, 404, `Config not found for ${dataset_uuid}`); + + + // ====================================================== + // SparkplugSRC + // ====================================================== + + if(structure == Constants.App.SparkplugSrc){ + return [ + { + dataset_uuid, + device_uuid: config.source, + from: inherited_range.from, + to: inherited_range.to, + }, + ]; + } + + // ====================================================== + // SessionLimits + // ====================================================== + + if(structure == Constants.App.SessionLimits){ + const session_range = { + from: config.from ?? null, + to: config.to ?? null, + }; + + const merged_range = this._intersectRanges( + inherited_range, + session_range, + ); + + return this._resolve_dataset( + config.source, + merged_range, + visited, + ); + } + + // ====================================================== + // UnionComponents + // ====================================================== + if(structure == Constants.App.UnionComponents){ + if(!Array.isArray(config)) return fail(this.log, 404, `Union dataset config must be an array.`); + + const results = []; + + for (const src_ds_uuid of config){ + const src_results = await this._resolve_dataset( + src_ds_uuid, + inherited_range, + new Set(visited), + ); + results.push(...src_results); + } + return results; + } + + return fail(this.log, 404, `Unknown dataset structure ${structure}`); + } + + _intersectRanges(parent, child){ + const from = maxDate(parent.from, child.from); + const to = minDate(parent.to, child.to); + + if(from && to && new Date(from) > new Date(to)){ + return fail(this.log, 404, `Invalid intersected range. From: ${from} > to: ${to}`); + } + + return {from, to}; + } + + + /** POST. Queries all possible Influx suffixes, combines the measuremenets and returns actual data from a dataset. + * Empty POST body requests all dataset measurements. + * @param {*} req + * @param {*} res + * @returns CSV with columns: + * device - Device this data point comes from + * metric - metric name (don't include Influx :x suffix) + * timestamp - ISO string + * value - actual data value + * unit - Engineering unit (if available) + * Don't return valid response if dataset is invalid + */ + async dataset_data(req, res) { + + const dataset_uuid = req.params.uuid; + + if (!valid_uuid(dataset_uuid)) { + return fail(this.log, 422, `Invalid dataset uuid`); + } + + const ok = await this.auth.check_acl( + req.auth, + Constants.Perm.ReadDataset, + dataset_uuid, + true, + ); + + if (!ok) { + return fail(this.log, 403, `Unauthorised to read ${dataset_uuid}`); + } + + const meta = req.body?.measurement + ? { measurement: req.body?.measurement } + : undefined; + + try { + // Resolve dataset tree + const resolved_sources = await this._resolve_dataset(dataset_uuid); + + res.setHeader("Content-Type", "text/csv"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${dataset_uuid}.csv"` + ); + + const zipStream = this.influxReader.exportDevices(resolved_sources, meta); + + + res.setHeader( + "Content-Type", + "application/zip" + ); + + res.setHeader( + "Content-Disposition", + `attachment; filename=${dataset_uuid}.zip` + ); + + zipStream.pipe(res); + + }catch (err) { + this.log(err); + return fail(this.log, 500); + } + } + + + /** GET. + * + * @param {*} req + * @param {*} res + * @returns list of dataset UUIDs the client has permission to EDIT + * the response includes invalid datasets + */ + async structure_list(req, res){ + const uuids = await rx.firstValueFrom(this.data.allowed_all_dataset_uuids(req.auth, Constants.Perm.EditDataset)); + return res.status(200).json(uuids); + } + + /** GET. Fetches structural definition of dataset + * Requires EDIT permission on the dataset + * READONLY clients can't see this structure + * If dataset is structurally INVALID -> Special UUID (structurally invalid dataset) and absent config + * @param {*} req + * @param {*} res + * @returns object: + * uuid {UUID} - Dataset UUID + * class {UUID} - Structural class: Sparkplug device, Union Dataset, Session + * config {any} - Structural definition: "not visible", Union components, Session limits + * the response includes invalid datasets but their structure field references Invalid Dataset uuid + */ + async structure_uuid(req, res) { + const dataset_uuid = req.params.uuid; + if (!valid_uuid(dataset_uuid)) return fail(this.log, 422, `Dataset uuid ${dataset_uuid} is invalid.`); + + const ok = await this.auth.check_acl( + req.auth, + Constants.Perm.EditDataset, + dataset_uuid, + true, + ); + + if (!ok) return fail(this.log, 403, `You don't have permission to Edit dataset ${uuid}.`); + + const dataset = await rx.firstValueFrom( + this.data.allowed_all_datasets(req.auth, Constants.Perm.EditDataset).pipe( + rx.map(datasets => datasets.get(dataset_uuid)) + ) + ); + + if (!dataset) return fail(this.log, 404, `Dataset not found for ${dataset_uuid}.`); + const {from, to, ...def} = dataset; + return res.status(200).json(def); + } + + + async _update_dataset_config(principal, structure, config, objectUuid){ + const ok2 = await this._check_second_level_permission(principal, structure, config); + + if(!ok2) return fail(this.log, 403, `You don't have permission for source(s) in config.`); + + // Create new Dataset Object + if(!objectUuid){ + objectUuid = await this.cdb.create_object(Constants.Class.Dataset); + this.log("Created new dataset object in ConfigDB", objectUuid); + } + + // Create config entry for the dataset object + await this.cdb.put_config(structure, objectUuid, config); + this.log(`Added config for ${objectUuid}`); + + this._create_subclass_relationship(structure, objectUuid, config); + + this.log(`Created subclass relationship for ${objectUuid}`); + return objectUuid; + } + + async _create_subclass_relationship(structure, dataset_uuid, config) { + const subclasses = + structure == Constants.App.SessionLimits ? [[config.source, dataset_uuid]] + : structure == Constants.App.UnionComponents ? config.map(source => [dataset_uuid, source]) + : []; + for (const subclass of subclasses) { + await rx.lastValueFrom( + rx.defer(() => + this.cdb.class_add_subclass(...subclass) + ).pipe(retryBackoff(500, e => this.log(e))) + ); + } + } + + async _unlink_subclasses(dataset_uuid, structure, config){ + if (structure === Constants.App.SparkplugSrc) return; + else if(structure === Constants.App.SessionLimits){ + // remove the dataset from subclasses of its source dataset + await this.cdb.class_remove_subclass(config.source, dataset_uuid); + this.log(`Removed ${dataset_uuid} from ${config.source} subclasses.`) + + }else if(structure === Constants.App.UnionComponents){ + if(config.length <= 0) return; + + for (const source_uuid of config){ + await this.cdb.class_remove_subclass(dataset_uuid, source_uuid); + this.log(`Removed ${dataset_uuid} from ${source_uuid} subclasses.`); + } + }else{ + return fail(this.log, 404, `Unknown structure type ${structure}`); + } + } + + + /** POST. Creates a new dataset. + * + * @param {*} req.body must be object (structure, config) without uuid. + * @param {*} res + * @returns new dataset's UUID - JSON string + */ + + _validate_config(config, structure){ + if(structure === Constants.App.SparkplugSrc){ + const source = config.source; + if(!source) + return fail(this.log, 422, `req.body.config for SparkplugSrc structure type ${structure} must contain "source" field.`); + + if(!valid_uuid(source)) + return fail(this.log, 422, `req.body.config.source uuid is invalid.`); + + } + else if(structure === Constants.App.SessionLimits){ + const {to, from, source} = config; + + if(!to || !from || !source) + return fail(this.log, 422, `req.body.config for SessionLimits type must contain "source", "to" and "from" fields.`); + + if(!valid_uuid(source)) + return fail(this.log, 422, `req.body.config.source uuid is invalid.`); + + if(!valid_datetime(to)) + return fail(this.log, 422, `req.body.config.to datetime format is invalid.`); + + if(!valid_datetime(from)) + return fail(this.log, 422, `req.body.config.from datetime format is invalid.`); + } + else if(structure === Constants.App.UnionComponents){ + if(!Array.isArray(config)) + return fail(this.log, 422, `req.body.config must be array []`); + + for(let src of config){ + if (!valid_uuid(src)) + return fail(this.log, 422, `${src} is invalid UUID`); + } + } + else{ + return fail(this.log, 404, `Unknown structure type ${structure}`); + } + } + + async structure_create(req, res){ + const {structure, config} = req.body; + + if(!structure) return fail(this.log, 422, `Structure not provided in request body.`); + if(!valid_uuid(structure)) return fail(this.log, 422, `Structure uuid ${structure} is invalid.`); + + if(!config) return fail(this.log, 422, `Config not provided in request body.`); + this._validate_config(config, structure); + + const ok = await this.auth.check_acl( + req.auth, + Constants.Perm.CreateDataset, + structure, + true + ); + + if (!ok) return fail(this.log, 403, `You don't have Create permission for structure ${structure}`); + + const objectUuid = await this._update_dataset_config( + req.auth, + structure, + config, + null + ); + + return res.status(200).json(objectUuid); + } + + + /** PUT. Updates dataset definition. Principal should have CreateDataset permission + * + * @param {*} req.body must be object (structure, config) and UUID (optional) + * @param {*} res + */ + /* + For VALID dataset: + Don't let changing the structure type only config. So, if request.body.structure != dataset.structure then return 409. + Remove all subclass relationships between dataset and its current config sources + Add subclass relationships between datasets and its new config sources + + For INVALID dataset: + Remove config entries for this dataset from all structure apps and ignore 404 + Don't remove any existing subclass relationships -> Completely ignore existing subclass relationships for Invalid ones. + Add subclass relationship between dataset and its new config source + */ + + async structure_update(req, res){ + const dataset_uuid = req.params.uuid; + if(!dataset_uuid) return fail(this.log, 422, `Dataset uuid not provided`); + if(!valid_uuid(dataset_uuid)) return fail(this.log, 422, `Invalid uuid ${dataset_uuid}`); + + const structure = req.body.structure; + if(!structure) return fail(this.log, 422, `Structure not provided`); + if(!valid_uuid(structure)) return fail(this.log, 422, `Structure uuid is invalid`); + + const new_config = req.body.config; + if(!new_config) return fail(this.log, 422, `Config not provided`); + this._validate_config(new_config, structure); + + const ok = await this.auth.check_acl( + req.auth, + Constants.Perm.EditDataset, + dataset_uuid, + true + ); + + if (!ok) return fail(this.log, 403, `You don't have Edit permission for dataset ${dataset_uuid}`); + + const datasets = await rx.firstValueFrom(this.data.allowed_all_datasets(req.auth, Constants.Perm.EditDataset)); + const dataset = datasets.get(dataset_uuid); + if(!dataset) return fail(this.log, 404, `Dataset ${dataset_uuid} not found.`); + + const current_structure = dataset.structure; + const current_config = dataset.config; + const is_valid = current_structure !== Constants.Special.InvalidDataset; + + // for VALID dataset + if(is_valid){ + if(current_structure != structure) return fail(this.log, 409, `Changing structure type is not allowed (current: ${current_structure}, new: ${structure})`); + + // remove all subclass relationships with current config sources + await this._unlink_subclasses( + dataset_uuid, + current_structure, + current_config + ); + } + // For INVALID dataset + else{ + // delete configs for all other structures + const all_structure_apps = Object.values(Constants.App); + + for(const s of all_structure_apps){ + try{ + await this.cdb.delete_config(s, dataset_uuid); + }catch(e){ + ServiceError.check(404); + } + } + } + + const objectUuid = await this._update_dataset_config( + req.auth, + structure, + new_config, + dataset_uuid, + ); + + if(!objectUuid){ + return fail(this.log, 500, `Failed to update dataset ${dataset_uuid}`); + } + + return res.status(200).json(objectUuid); + } +} diff --git a/acs-data-access/lib/constants.js b/acs-data-access/lib/constants.js new file mode 100644 index 000000000..7172c5147 --- /dev/null +++ b/acs-data-access/lib/constants.js @@ -0,0 +1,53 @@ +export const DataAccess = { + App: { + DatasetDefinition: "eae2d4ae-164d-4dc6-b646-7e0320057bd9", + DatasetMetadata: "e3b9fd2c-9de1-470b-9675-739e2a55b77f", + SparkplugSrc: "f5d550c4-2831-11f1-b0b0-83fda3035799", + UnionComponents: "1c4ca454-de38-44d9-92fb-aa5218bfa257", + SessionLimits: "8754c000-3778-4ae6-b2b8-bbcd959bb775", + MESIdentifiers: "af178f0c-3b1e-44f2-9724-5cf06e8fd056", + }, + + Class: { + Measurement: "cce0ac4e-b5ba-4837-b45d-c74df55aa2d7", + Dataset: "c31d3cbd-01cd-4833-8014-c4512aef1e5c", + Published: "414d2d10-6be8-4c27-8e9f-c716ef5432b9", + Partial: "6c583d11-9a88-4bc1-b77c-81b01e9c9827", + MESDataset: "586205bf-81c6-4091-9d2c-f3c0465ebdc4", + Equipment: "4c93ddc1-e610-4efe-91e3-a355f9ba1a09", + WorkOrder: "b416e44c-c57e-4486-9431-64c425f1b2c6", + Product: "4a089748-b26b-4f12-8f1a-164bfba97809", + Operation: "bd0354eb-b8f7-4bd9-8407-0588e545603c", + MES: "2c691583-89fe-4421-bf2c-64e34e663711" + }, + + Group: { + DatasetGroup: "17e37253-8626-4031-b217-28c6a03e91c1", + DatasetRoleGroup: "56c52f70-0649-4962-8526-9ec9d1c85ca4", + StructuralDatasetType: "70ff7bea-bb2d-48c2-88fd-4f7a79b1aa3c", + FunctionalDatasetGroup: "86e5b048-e956-4820-939e-3abf3eda4e03", + }, + + Service: { + DataAccess: "06cee697-29d3-4972-9479-bc392e24946e" + }, + + Special: { + InvalidDataset: "696396a0-2831-11f1-9b12-33d63b8c5115", + }, + + Requirement: { + ServiceRole: "dd18050c-a9ab-4287-8af2-e983f20e89c8" + }, + + Perm: { + All: "1e3cb5aa-a3f7-4bb6-9ef1-c3bc08427f60", + CreateDataset: "2d666b41-7a0d-4845-ad59-3113f25b469a", + IncludeInUnion: "94d51085-af83-4796-8059-fcd578e3f572", + ReadDataset: "ec48462e-37eb-4f56-8efa-83d813e85559", + UseForSession: "c089b9a9-06cd-4211-94fc-9ad52a759987", + EditDataset: "af06b9e5-456a-43e4-b636-5b17de28fc7f", + DeleteDataset: "6f301df8-0ad1-496f-8391-8de92c43ad8e", + UseSparkplug: "788b049c-2831-11f1-99fd-2b0bf86d6f77", + } +} \ No newline at end of file diff --git a/acs-data-access/lib/dataflow.js b/acs-data-access/lib/dataflow.js new file mode 100644 index 000000000..29c037713 --- /dev/null +++ b/acs-data-access/lib/dataflow.js @@ -0,0 +1,701 @@ +/* +* ACS Data Access Service +* Data-Flow / sequence management +*/ + +import { List as IList, Map as IMap } from "immutable"; +import rx from "rxjs"; + +import * as rxu from "@amrc-factoryplus/rx-util"; + +import { DataAccess as Constants } from './constants.js'; +import { UUIDs } from '@amrc-factoryplus/rx-client'; +import { minDate, maxDate } from './utils.js'; + +export class DataFlow { + constructor(opts) { + this.log = opts.debug.bound("data"); + this.cdb = opts.cdb; + this.auth = opts.auth; + + this.datasets = this._build_datasets(); + + this.valid_datasets = this._build_valid_datasets(); + + this.invalid_datasets = this._build_invalid_datasets(); + + this.general_infos = this._build_general_info(); + this.metadata = this._build_metadata(); + this.functional_types = this._build_functional_types(); + this.parts = this._build_parts(); + } + + run() { + // this.datasets.subscribe(ss => + // this.log("Dataset Definitions UPDATE %o", ss.toJS()) + // ); + } + + /* + * --------------------------------------------------- + * Rx Operators + * --------------------------------------------------- + */ + + filter_allowed_datasets(principal, permission) { + const acls = this.auth.watch_acl_with_perm( + principal, + permission + ); + + return source => source.pipe( + rx.combineLatestWith(acls), + + rx.map(([datasets, targets]) => + datasets.filter( + (_, datasetId) => targets.has(datasetId) + ) + ) + ); + } + + select_dataset_uuids() { + return source => source.pipe( + rx.map(datasets => datasets.keySeq()) + ); + } + + /* + * ----------------------------------------------- + * Public composed streams + * --------------------------------------------------- + */ + + allowed_valid_datasets(principal, permission) { + return this.valid_datasets.pipe( + this.filter_allowed_datasets( + principal, + permission + ) + ); + } + + allowed_valid_dataset_uuids(principal, permission) { + return this.valid_datasets.pipe( + this.filter_allowed_datasets( + principal, + permission + ), + + this.select_dataset_uuids() + ); + } + + allowed_invalid_datasets(principal, permission) { + return this.invalid_datasets.pipe( + this.filter_allowed_datasets( + principal, + permission + ) + ); + } + + allowed_invalid_dataset_uuids(principal, permission) { + return this.invalid_datasets.pipe( + this.filter_allowed_datasets( + principal, + permission + ), + + this.select_dataset_uuids() + ); + } + + allowed_all_datasets(principal, permission) { + return this.datasets.pipe( + this.filter_allowed_datasets( + principal, + permission + ) + ); + } + + allowed_all_dataset_uuids(principal, permission) { + return this.datasets.pipe( + this.filter_allowed_datasets( + principal, + permission + ), + + this.select_dataset_uuids() + ); + } + + /* + * ------------------------------------------------ + * Parts + * ------------------------------------------------ + */ + + allowed_dataset_parts( + dataset_uuid, + principal, + permission + ) { + return rx.combineLatest([ + this.parts, + this.auth.watch_acl_with_perm( + principal, + permission + ) + ]).pipe( + rx.map(([partsMap, allowedParts]) => { + const parts = + partsMap.get(dataset_uuid); + + if (!parts) + return []; + + return parts.filter( + partId => + allowedParts.has(partId) + ); + }), + + rxu.shareLatest() + ); + } + + /* + * -------------------------------------------------- + * Watch Dataset's members' subclasses + * ---------------------------------------------------- + */ + + _build_parts() { + return rxu.rx( + rx.combineLatest([ + this.cdb.watch_members_subclasses( + Constants.Class.Dataset + ), + + this.cdb.watch_members( + Constants.Class.Dataset + ) + ]), + + rx.map(([subclassesMap, datasetMembers]) => + subclassesMap.map( + subclass_uuids => + subclass_uuids.filter( + x => datasetMembers.has(x) + ) + ) + ), + + rxu.shareLatest() + ); + } + + /* + * ---------------------------------------------------- + * Functional types + * ---------------------------------------------------- + */ + + _build_functional_types() { + return rxu.rx( + this.cdb.watch_member_members( + Constants.Group.FunctionalDatasetGroup + ), + + rx.map(data => { + let result = IMap(); + + data.forEach((memberIds, groupId) => { + memberIds.forEach(memberId => { + result = result.update( + memberId, + list => + list + ? list.push(groupId) + : IList([groupId]) + ); + }); + }); + + return result; + }), + + rxu.shareLatest() + ); + } + + /* + * ----------------------------------------------------- + * Generic app config builder + * ---------------------------------------------------- + */ + + _build_apps_configs(app_klass) { + return rxu.rx( + this.cdb.watch_members(app_klass), + + rx.switchMap((membersSet) => { + const appUuids = + membersSet.toArray(); + + if (appUuids.length === 0) { + return rx.of(IMap()); + } + + const dataStreams = + appUuids.map(appId => + this.cdb.search_app(appId).pipe( + rxu.shareLatest(), + + rx.map(config => [ + appId, + config + ]) + ) + ); + + return rx.combineLatest( + dataStreams + ).pipe( + rx.map(appEntries => { + let result = IMap(); + + for (const [appId, config] + of appEntries) { + + for (const [datasetId, payload] + of config) { + + const existing = + result.get( + datasetId, + IMap() + ); + + result = result.set( + datasetId, + existing.set( + appId, + payload + ) + ); + } + } + + return result; + }) + ); + }), + + rxu.shareLatest() + ); + } + + /* + * ------------------------------------------------- + * Metadata + * --------------------------------------------------- + */ + + _build_metadata() { + return this._build_apps_configs( + Constants.App.DatasetMetadata + ); + } + + /* + * -------------------------------------------------- + * General info + * ------------------------------------------------ + */ + + _build_general_info() { + return rxu.rx( + this.cdb.search_app( + UUIDs.App.Info + ), + + rxu.shareLatest() + ); + } + + /* + * ------------------------------------------------- + * Dataset builder + * ------------------------------------------------- + */ + + _build_datasets() { + return rxu.rx( + this._build_apps_configs( + Constants.App.DatasetDefinition + ), + + rx.map((groupedConfigs) => { + + /* + * ------------------------------------------------------------ + * 1. Normalise duplicate definitions + * ------------------------------------------------------------ + * + * Convert: + * + * datasetId => IMap({ + * appId => payload + * }) + * + * into: + * + * datasetId => { + * structure, + * config + * } + * + * Invalid datasets are represented directly by: + * + * structure === Constants.Special.InvalidDataset + */ + + let map = groupedConfigs.map( + (definitions) => { + + const entries = + definitions + .entrySeq() + .toArray(); + + /* + * Multiple structural definitions + * => invalid dataset + */ + + if (entries.length !== 1) { + return { + structure: + Constants.Special.InvalidDataset, + + config: null, + from: null, + to: null + }; + } + + const [structure, config] = + entries[0]; + + return { + structure, + config + }; + } + ); + + /* + * ------------------------------------------------------------ + * 2. Recursive resolution + * ------------------------------------------------------------ + * + * resolve(datasetId) + * + * Returns: + * + * { from, to } + * + * or: + * + * null + * + * where null means invalid. + */ + + const cache = new Map(); + + const resolve = + (datasetId, visited = new Set()) => { + + /* + * Cached result + */ + + if (cache.has(datasetId)) + return cache.get(datasetId); + + /* + * Cycle detection + */ + + if (visited.has(datasetId)) { + cache.set(datasetId, null); + return null; + } + + visited.add(datasetId); + + const def = + map.get(datasetId); + + /* + * Missing dataset + */ + + if (!def) { + cache.set(datasetId, null); + return null; + } + + /* + * Already invalid + */ + + if ( + def.structure === + Constants.Special.InvalidDataset + ) { + cache.set(datasetId, null); + return null; + } + + const { + structure, + config + } = def; + + /* + * -------------------------------------------------- + * SESSION LIMITS + * -------------------------------------------------- + */ + + if ( + structure === + Constants.App.SessionLimits + ) { + + const child = + resolve( + config.source, + new Set(visited) + ); + + /* + * Invalid child + */ + + if (!child) { + cache.set(datasetId, null); + return null; + } + + const from = + config.from + ? new Date(config.from) + : null; + + const to = + config.to + ? new Date(config.to) + : null; + + const result = { + from: + maxDate( + from, + child.from + ), + + to: + minDate( + to, + child.to + ) + }; + + cache.set(datasetId, result); + + return result; + } + + /* + * -------------------------------------------------- + * SPARKPLUG SOURCE + * -------------------------------------------------- + */ + + if ( + structure === + Constants.App.SparkplugSrc + ) { + + const result = { + from: null, + to: null + }; + + cache.set(datasetId, result); + + return result; + } + + /* + * -------------------------------------------------- + * UNION COMPONENTS + * -------------------------------------------------- + */ + + if ( + structure === + Constants.App.UnionComponents + ) { + + const children = + config.map(id => + resolve( + id, + new Set(visited) + ) + ); + + /* + * Any invalid child + */ + + if (children.some(c => !c)) { + cache.set(datasetId, null); + return null; + } + + const fromTimes = + children + .map(v => + v.from?.getTime() + ) + .filter(t => + !isNaN(t) + ); + + const toTimes = + children + .map(v => + v.to?.getTime() + ) + .filter(t => + !isNaN(t) + ); + + const result = { + from: + fromTimes.length + ? new Date( + Math.min( + ...fromTimes + ) + ) + : null, + + to: + toTimes.length + ? new Date( + Math.max( + ...toTimes + ) + ) + : null + }; + + cache.set(datasetId, result); + + return result; + } + + /* + * -------------------------------------------------- + * Unknown structure + * -------------------------------------------------- + */ + + cache.set(datasetId, null); + + return null; + }; + + /* + * ------------------------------------------------------------ + * 3. Resolve all datasets + * ------------------------------------------------------------ + */ + + return map.map( + (def, datasetId) => { + + /* + * Already invalid due to duplicate defs + */ + + if ( + def.structure === + Constants.Special.InvalidDataset + ) { + return def; + } + + const resolved = + resolve(datasetId); + + /* + * Invalid dependency graph + */ + + if (!resolved) { + return { + structure: + Constants.Special.InvalidDataset, + + config: null, + from: null, + to: null + }; + } + + /* + * Valid dataset + */ + + return { + ...def, + ...resolved + }; + } + ); + }), + + rxu.shareLatest() + ); + } + + _build_valid_datasets(){ + return this.datasets.pipe( + rx.map(map => map.filter( + def => def.structure !== Constants.Special.InvalidDataset + )), + rxu.shareLatest() + ); + } + + _build_invalid_datasets(){ + return this.datasets.pipe( + rx.map(map => map.filter( + def => def.structure === Constants.Special.InvalidDataset + )), + rxu.shareLatest() + ); + } +} + + diff --git a/acs-data-access/lib/influx-reader.js b/acs-data-access/lib/influx-reader.js new file mode 100644 index 000000000..66000c5a1 --- /dev/null +++ b/acs-data-access/lib/influx-reader.js @@ -0,0 +1,171 @@ +import { ZipArchive } from "archiver"; +import { PassThrough } from "stream"; +import { once } from "events"; +import pLimit from "p-limit"; + +export class InfluxReader { + constructor(opts) { + this.log = opts.debug.bound("influxReader"); + + this.influx_bucket = opts.influx_bucket; + this.influx_org = opts.influx_org; + + this.influx_query_api = + opts.influx_client.getQueryApi( + this.influx_org + ); + + this.limit = pLimit(4); + } + + exportDevices(deviceSources, meta = {}) { + const archive = new ZipArchive({ + zlib: { + level: 0, + }, + }); + + archive.on("warning", err => { + this.log("archive warning", err); + }); + + archive.on("error", err => { + this.log("archive error", err); + archive.destroy(err); + }); + + this.#appendDevices( + archive, + deviceSources, + meta + ).catch(err => { + archive.destroy(err); + }); + + return archive; + } + + async #appendDevices(archive, deviceSources, meta) { + for (const source of deviceSources) { + await this.limit(() => + new Promise((resolve, reject) => { + const csvStream = new PassThrough({ + highWaterMark: 1024 * 1024, + }); + + archive.append(csvStream, { + name: `${source.device_uuid}.csv`, + }); + + this.#streamDevice(source, csvStream, meta) + .then(resolve) + .catch(reject); + }) + ); + } + + await new Promise((resolve, reject) => { + archive.once("error", reject); + archive.once("close", resolve); + archive.finalize(); + }); + } + + async #streamDevice( + source, + writable, + meta + ) { + const query = + this.#buildFluxQuery( + source, + meta + ); + + this.log( + "streaming", + source.device_uuid + ); + + const response = + this.influx_query_api.response(query); + + try { + for await ( + const chunk of response.iterateLines() + ) { + if ( + !writable.write( + chunk + "\n" + ) + ) { + await once( + writable, + "drain" + ); + } + } + + writable.end(); + + await once( + writable, + "finish" + ); + } + catch (err) { + writable.destroy(err); + throw err; + } + } + + + #buildFluxQuery( + source, + meta = {} + ) { + const start = + source.from ?? + "1970-01-01T00:00:00Z"; + + const stop = + source.to ?? + "2100-01-01T00:00:00Z"; + + const measurementFilter = + meta.measurement + ? ` + |> filter( + fn: (r) => + r._measurement == + "${meta.measurement}" + ) + ` + : ""; + + return ` + from(bucket: "${this.influx_bucket}") + + |> range( + start: time(v: "${start}"), + stop: time(v: "${stop}") + ) + + ${measurementFilter} + + |> filter( + fn: (r) => + r.topLevelInstance == + "${source.device_uuid}" + ) + + |> keep(columns: [ + "_time", + "_value", + "_measurement", + "device", + "unit" + ]) + `; + } +} \ No newline at end of file diff --git a/acs-data-access/lib/notify.js b/acs-data-access/lib/notify.js new file mode 100644 index 000000000..adab0090c --- /dev/null +++ b/acs-data-access/lib/notify.js @@ -0,0 +1,421 @@ +/* +* ACS Data Access Service +* DataAccessNotify +*/ + +import deep_equal from "deep-equal"; +import * as rx from "rxjs"; +import { Map as IMap } from "immutable"; + +import * as rxu from "@amrc-factoryplus/rx-util"; +import { Notify } from "@amrc-factoryplus/service-api"; + +import { DataAccess as Constants } from "./constants.js"; +import { valid_uuid } from "./validate.js"; +import { response } from "express"; + +export class DataAccessNotify { + constructor(opts) { + this.data = opts.data; + this.auth = opts.auth; + this.log = opts.debug.bound("notify"); + + this.notify = this.build_notify(opts.api); + } + + run() { + this.log("Running Data Access Notify Server"); + this.notify.run(); + } + + build_notify(api) { + const notify = new Notify({ + api, + log: this.log, + }); + + notify.watch( + "v2/metadata/", + this.metadata_list.bind(this) + ); + + notify.watch( + "v2/metadata/:uuid", + this.metadata_uuid.bind(this) + ); + + notify.search( + "v2/metadata/", + this.metadata_search.bind(this) + ); + + notify.watch( + "v2/structure/", + this.structure_list.bind(this) + ); + + notify.watch( + "v2/structure/:uuid", + this.structure_uuid.bind(this) + ); + + notify.search( + "v2/structure/", + this.structure_search.bind(this) + ); + + return notify; + } + + + /* + * ========================================================= + * METADATA + * ========================================================= + */ + // WATCH + metadata_list(sess) { + return this.data.allowed_valid_dataset_uuids( + sess.principal, + Constants.Perm.ReadDataset + ).pipe( + rx.map(data => ({ + status: 200, + response: { + body: data + } + })) + ); + } + + _metadata_resource_state(sess, uuid) { + if (!valid_uuid(uuid)) return; + + const datasets$ = + this.data.allowed_valid_datasets( + sess.principal, + Constants.Perm.ReadDataset + ).pipe( + rx.startWith(IMap()) + ); + + const infos$ = + this.data.general_infos.pipe( + rx.startWith(IMap()) + ); + + const ftypes$ = + this.data.functional_types.pipe( + rx.startWith(IMap()) + ); + + const metadata$ = + this.data.metadata.pipe( + rx.startWith(IMap()) + ); + + const parts$ = + this.data.allowed_dataset_parts( + uuid, + sess.principal, + Constants.Perm.ReadDataset + ).pipe( + rx.startWith(IMap()) + ); + + return rx.combineLatest([ + datasets$, + infos$, + ftypes$, + metadata$, + parts$, + ]).pipe( + + rx.map(([datasets, infos, ftypes, metadata, parts]) => { + + const dataset = datasets.get(uuid); + + if (!dataset) return; + + const info = infos.get(uuid); + + return { + uuid, + name: info?.name ?? "UNKNOWN", + from: dataset.from ?? undefined, + to: dataset.to ?? undefined, + function: ftypes?.get(uuid), + metadata: metadata?.get(uuid), + parts: parts ?? undefined, + } + }), + + rx.distinctUntilChanged(deep_equal), + rxu.shareLatest() + ); + } + + // WATCH + metadata_uuid(sess, uuid) { + return this._metadata_resource_state(sess, uuid) + .pipe( + rx.map(data => ({ + status: 200, + response: { + body: data + } + })) + ); + } + + /* + * Searchable metadata map + */ + + + + _build_search_source(uuids) { + const shared = uuids.pipe( + rxu.shareLatest() + ); + + return { + full: async () => { + const state = await rx.firstValueFrom( + shared.pipe( + rx.take(1) + ) + ); + + const children = Object.fromEntries( + state.entrySeq().map(([uuid, response]) => [ + uuid, + { + status: response.status, + body: response.body, + ...(response.headers + ? { headers: response.headers } + : {}), + } + ]) + ); + + return { + children, + response: { status: 204 }, + }; + }, + + updates: shared.pipe( + rx.startWith(IMap()), + rx.pairwise(), + + rx.mergeMap(([prev, curr]) => { + const updates = []; + + /** + * Created / Updated + */ + curr.forEach((response, key) => { + const old = prev.get(key); + + const plain = { + status: response.status, + body: response.body, + ...(response.headers + ? { headers: response.headers } + : {}), + }; + + /** + * Newly visible + */ + if (!old) { + updates.push({ + status: 200, + child: key, + response: { + ...plain, + status: plain.status === 200 + ? 201 + : plain.status, + }, + }); + + return; + } + + /** + * Changed + */ + if (!deep_equal(old, response)) { + updates.push({ + status: 200, + child: key, + response: plain, + }); + } + }); + + /** + * Removed + */ + prev.forEach((_, key) => { + if (!curr.has(key)) { + updates.push({ + status: 200, + child: key, + response: { + status: 404, + }, + }); + } + }); + + return rx.from(updates); + }) + ), + + /** + * Must be an operator function. + */ + acl: rx.pipe(), + }; + } + + metadata_search(sess) { + const dataset_uuids = this.data + .allowed_valid_dataset_uuids( + sess.principal, + Constants.Perm.ReadDataset + ) + .pipe( + rx.switchMap(uuids => { + const list = uuids.toArray?.() ?? []; + + if (list.length === 0) + return rx.of(IMap()); + + return rx.combineLatest( + list.map(uuid => + this._metadata_resource_state(sess, uuid).pipe( + rx.filter(x => x && typeof x === "object"), + + /** + * Convert to plain serialisable object. + */ + rx.map(res => [uuid, { + status: 200, + body: res + }]) + ) + ) + ).pipe( + rx.map(entries => IMap(entries)) + ); + }), + + rxu.shareLatest() + ); + + return this._build_search_source(dataset_uuids); + } + + + + + + + + /* + * ========================================================= + * STRUCTURE + * ========================================================= + */ + + structure_list(sess) { + return this.data.allowed_all_dataset_uuids( + sess.principal, + Constants.Perm.EditDataset + ).pipe( + rx.map(data => ({ + status: 200, + response: { + body: data + } + })) + ); + } + + + _structure_resource_state(sess, uuid) { + if (!valid_uuid(uuid)) return; + + return this.data.allowed_valid_datasets( + sess.principal, + Constants.Perm.EditDataset + ).pipe( + rx.map(datasets => datasets.get(uuid)), + + rx.filter(Boolean), + + rx.map(dataset_def => { + const { from, to, ...body } = dataset_def; + return body; + }), + + rx.distinctUntilChanged(deep_equal), + + rxu.shareLatest(), + ); + } + + structure_uuid(sess, uuid) { + return this._structure_resource_state(sess, uuid) + .pipe( + rx.map(body => ({ + status: 200, + response: {body} + })), + ); + } + + structure_search(sess) { + const dataset_uuids = this.data + .allowed_all_dataset_uuids( + sess.principal, + Constants.Perm.EditDataset + ) + .pipe( + rx.switchMap(uuids => { + const list = uuids.toArray?.() ?? []; + + if (list.length === 0) + return rx.of(IMap()); + + return rx.combineLatest( + list.map(uuid => + this._structure_resource_state(sess, uuid).pipe( + rx.map(body => [ + uuid, + { + status: 200, + body, + } + ]) + ) + ) + ).pipe( + rx.map(entries => IMap(entries)) + ); + }), + + rxu.shareLatest() + ); + + return this._build_search_source(dataset_uuids); + } + + + +} \ No newline at end of file diff --git a/acs-data-access/lib/utils.js b/acs-data-access/lib/utils.js new file mode 100644 index 000000000..ac6430d17 --- /dev/null +++ b/acs-data-access/lib/utils.js @@ -0,0 +1,38 @@ +export function csv_escape(value) { + if (value == null) return ""; + + const str = String(value); + + if ( + str.includes(",") || + str.includes('"') || + str.includes("\n") + ) { + return `"${str.replace(/"/g, '""')}"`; + } + + return str; +} + + +// const toTime = d => d ? new Date(d).getTime() : null; + +export function maxDate(a, b) { + + if (!a) return b + if (!b) return a + + return new Date(a) > new Date(b) + ? a + : b +} + +export function minDate(a, b) { + + if (!a) return b + if (!b) return a + + return new Date(a) < new Date(b) + ? a + : b +} \ No newline at end of file diff --git a/acs-data-access/lib/validate.js b/acs-data-access/lib/validate.js new file mode 100644 index 000000000..e2e74470b --- /dev/null +++ b/acs-data-access/lib/validate.js @@ -0,0 +1,50 @@ +/* + * ACS Data Access service + * Data validation routines + */ + +export const UUID_rx = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; +export const KRB_rx = /^[a-zA-Z0-9_./-]+@[A-Za-z0-9-.]+$/; + +export const booleans = { + undefined: false, + "true": true, "false": false, + "1": true, "0": false, + on: true, off: false, + yes: true, no: false, +}; + +export function valid_uuid(uuid) { + if (UUID_rx.test(uuid)) + return true; + //debug.log("debug", `Ignoring invalid UUID [${uuid}]`); + return false; +} + +export function valid_krb(krb) { + if (KRB_rx.test(krb)) + return true; + //debug.log("debug", `Ignoring invalid principal [${krb}]`); + return false; +} + +export function valid_grant (grant) { + if (!valid_uuid(grant.principal)) return false; + if (!valid_uuid(grant.permission)) return false; + if (!valid_uuid(grant.target)) return false; + return (grant.plural === true || grant.plural === false); +} + + +export function valid_datetime(datetime_str){ + // 1. Check the exact string format using RegEx + const regex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; + if (!regex.test(datetime_str)) return false; + + // 2. Check if the calendar date and time are physically valid + const timestamp = Date.parse(datetime_str); + if (isNaN(timestamp)) return false; + + // 3. Ensure it matches the original string (prevents date rolling like Feb 30 -> Mar 2) + return new Date(timestamp).toISOString() === datetime_str; +} \ No newline at end of file diff --git a/acs-data-access/package.json b/acs-data-access/package.json new file mode 100644 index 000000000..bfb1a2ce7 --- /dev/null +++ b/acs-data-access/package.json @@ -0,0 +1,32 @@ +{ + "name": "acs-data-access", + "version": "1.0.0", + "description": "Data Access Service supports creating, finding and downloading datasets via a WebAPI.", + "main": "server.js", + "type": "module", + "scripts": { + "test": "vitest run ./tests/current" + }, + "author": "", + "license": "ISC", + "dependencies": { + "@amrc-factoryplus/rx-client": "file:../lib/js-rx-client", + "@amrc-factoryplus/rx-util": "file:../lib/js-rx-util", + "@amrc-factoryplus/service-api": "file:../lib/js-service-api", + "@amrc-factoryplus/service-client": "file:../lib/js-service-client", + "@dotenvx/dotenvx": "^1.39.0", + "@influxdata/influxdb-client": "^1.35.0", + "archiver": "^8.0.0", + "deep-equal": "^2.2.3", + "express": "^5.0.1", + "p-limit": "^7.3.0", + "rxjs": "^7.8.2", + "stream": "^0.0.3", + "stream-size": "^0.0.6", + "uuid": "^11.0.5" + }, + "devDependencies": { + "eslint": "^9.23.0", + "vitest": "^4.1.8" + } +} diff --git a/acs-data-access/tests/current/download.test.js b/acs-data-access/tests/current/download.test.js new file mode 100644 index 000000000..75624222f --- /dev/null +++ b/acs-data-access/tests/current/download.test.js @@ -0,0 +1,118 @@ +import {ServiceClient} from "@amrc-factoryplus/service-client"; +import process from "node:process"; +import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest'; +import { DataAccess as Constants } from "../../lib/constants.js"; +import * as Temp_Data from '../temp_data.js'; + + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +describe('POST /data/:uuid', () => { + const TEST_PRINCIPAL=process.env.TEST_HUMAN_UUID; + + let test_fplus; + let admin_fplus; + let ALL_GRANTS = []; + + + beforeEach(async () => { + admin_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.ADMIN_USERNAME, + password: process.env.ADMIN_PASSWORD, + verbose: 'ALL' + }); + + test_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.TEST_HUMAN_USERNAME, + password: process.env.TEST_HUMAN_PASSWORD, + verbose: 'ALL' + }); + + await sleep(2000); + }); + + beforeAll( async () => { + admin_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.ADMIN_USERNAME, + password: process.env.ADMIN_PASSWORD, + verbose: 'ALL' + }); + }); + + // Clean up grants if not removed in case of failure. + afterAll(async () => { + for (let grant of ALL_GRANTS){ + await deleteGrant(grant); + } + }); + + + async function addGrant(principal, permission, target, plural){ + if(!principal || + !permission || + !target + ) return; + + const grant_uuid = await admin_fplus.Auth.add_grant({ + principal, + permission, + target, + plural + }); + ALL_GRANTS.push(grant_uuid); + return grant_uuid; + } + + async function deleteGrant(grant_uuid){ + if(!grant_uuid) return; + + await admin_fplus.Auth.delete_grant(grant_uuid) + ALL_GRANTS = ALL_GRANTS.filter(uuid => uuid !== grant_uuid); + } + + + test('Irrelevant permission', async () => { + const dataset_uuid = Temp_Data.Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset; + + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + dataset_uuid, + false + ); + + await sleep(3000); + + try { + await test_fplus.DataAccess.download_data(dataset_uuid); + + expect.fail('Expected to throw'); + } + catch (err) { + expect(err.status).toBe(403); + } + + await deleteGrant(grant); + + }, 30000); + + test('Invalid UUID', async () => { + const dataset_uuid = "xxx"; + + try { + await test_fplus.DataAccess.download_data(dataset_uuid); + + expect.fail('Expected to throw'); + } + catch (err) { + expect(err.status).toBe(422); + } + + }, 30000); + +}); \ No newline at end of file diff --git a/acs-data-access/tests/http/delete_dataset.test.js b/acs-data-access/tests/http/delete_dataset.test.js new file mode 100644 index 000000000..1f20e5f1c --- /dev/null +++ b/acs-data-access/tests/http/delete_dataset.test.js @@ -0,0 +1,137 @@ +import {ServiceClient} from "@amrc-factoryplus/service-client"; +import process from "node:process"; +import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest'; +import { DataAccess as Constants } from "../../lib/constants.js"; +import * as Temp_Data from '../temp_data.js'; + + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +describe('GET /delete/:uuid', () => { + const TEST_PRINCIPAL=process.env.TEST_HUMAN_UUID; + + let test_fplus; + let admin_fplus; + let ALL_GRANTS = []; + + let dataset_to_be_deleted; + + beforeEach(async () => { + admin_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.ADMIN_USERNAME, + password: process.env.ADMIN_PASSWORD, + verbose: 'ALL' + }); + + test_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.TEST_HUMAN_USERNAME, + password: process.env.TEST_HUMAN_PASSWORD, + verbose: 'ALL' + }); + + await sleep(2000); + }); + + beforeAll( async () => { + admin_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.ADMIN_USERNAME, + password: process.env.ADMIN_PASSWORD, + verbose: 'ALL' + }); + + + dataset_to_be_deleted = await admin_fplus.DataAccess.create_dataset(Constants.App.SparkplugSrc, Temp_Data.Valid_Body.SRC.config); + + console.log(`Created new dataset ${dataset_to_be_deleted}`); + }); + + // Clean up grants if not removed in case of failure. + afterAll(async () => { + for (let grant of ALL_GRANTS){ + await deleteGrant(grant); + } + }); + + + async function addGrant(principal, permission, target, plural){ + if(!principal || + !permission || + !target + ) return; + + const grant_uuid = await admin_fplus.Auth.add_grant({ + principal, + permission, + target, + plural + }); + ALL_GRANTS.push(grant_uuid); + return grant_uuid; + } + + async function deleteGrant(grant_uuid){ + if(!grant_uuid) return; + + await admin_fplus.Auth.delete_grant(grant_uuid) + ALL_GRANTS = ALL_GRANTS.filter(uuid => uuid !== grant_uuid); + } + + + test('Successful deletion', async () => { + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.DeleteDataset, + dataset_to_be_deleted, + false + ); + + await sleep(10000); + + const deleted = await test_fplus.DataAccess.delete_dataset(dataset_to_be_deleted); + + await deleteGrant(grant); + + expect(deleted).toBe(dataset_to_be_deleted); + + }, 30000); + + test('Irrelevant permission', async () => { + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + dataset_to_be_deleted, + false + ); + + await sleep(3000); + + try { + await test_fplus.DataAccess.delete_dataset(dataset_to_be_deleted); + + expect.fail('Expected to throw'); + } + catch (err) { + expect(err.status).not.toBe(200); + } + + await deleteGrant(grant); + + }, 30000); + + test('Invalid uuid', async () => { + + try { + await test_fplus.DataAccess.delete_dataset("xxx"); + + expect.fail('Expected to throw'); + } + catch (err) { + expect(err.status).not.toBe(200); + } + }, 30000); +}); \ No newline at end of file diff --git a/acs-data-access/tests/http/metadata_list.test.js b/acs-data-access/tests/http/metadata_list.test.js new file mode 100644 index 000000000..5e1336bcf --- /dev/null +++ b/acs-data-access/tests/http/metadata_list.test.js @@ -0,0 +1,604 @@ +import {ServiceClient} from "@amrc-factoryplus/service-client"; +import process from "node:process"; +import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest'; +import { DataAccess as Constants } from "../../lib/constants.js"; +import * as Temp_Data from '../temp_data.js'; + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +describe('GET /metadata', () => { + const TEST_PRINCIPAL=process.env.TEST_HUMAN_UUID; + + let test_fplus; + let admin_fplus; + let ALL_GRANTS = []; + + + beforeEach(async () => { + admin_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.ADMIN_USERNAME, + password: process.env.ADMIN_PASSWORD, + verbose: 'ALL' + }); + + test_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.TEST_HUMAN_USERNAME, + password: process.env.TEST_HUMAN_PASSWORD, + verbose: 'ALL' + }); + await sleep(500); + }); + + + // Clean up grants if not removed in case of failure. + afterAll(async () => { + for (let grant of ALL_GRANTS){ + await deleteGrant(grant); + } + }); + + + async function addGrant(principal, permission, target, plural){ + if(!principal || + !permission || + !target + ) return; + + const grant_uuid = await admin_fplus.Auth.add_grant({ + principal, + permission, + target, + plural + }); + ALL_GRANTS.push(grant_uuid); + return grant_uuid; + } + + async function deleteGrant(grant_uuid){ + if(!grant_uuid) return; + + await admin_fplus.Auth.delete_grant(grant_uuid) + ALL_GRANTS = ALL_GRANTS.filter(uuid => uuid !== grant_uuid); + } + + async function addConfig(app, obj, config){ + if(!app || + !obj || + !config + ) return; + + await admin_fplus.ConfigDB.put_config( app, obj, config ); + } + + async function removeConfig(app, obj){ + if(!app || !obj) return; + + await admin_fplus.ConfigDB.delete_config( app, obj ); + } + + test('Datasets exist but user has no permissions', async () => { + const res = await test_fplus.DataAccess.get_metadata_list(); + expect(res).toEqual([]); + }); + + test('User has irrelevant permission to one dataset', async () => { + const dataset_uuid = Test_Uuids.Src_Datasets.TestDeviceDataset; + + // Grant unrelated EditDataset permission to the user + const grant_uuid = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + dataset_uuid, + false + ); + await sleep(1000); + + + const res = await test_fplus.DataAccess.get_metadata_list(); + + await deleteGrant(grant_uuid); + + expect(res).toEqual([]); + }); + + test('User has relevant permission to one dataset', async () => { + const dataset_uuid = Test_Uuids.Src_Datasets.TestDeviceDataset; + + // Grant correct permission + const grant_uuid = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + dataset_uuid, + false + ); + await sleep(1000); + + const res = await test_fplus.DataAccess.get_metadata_list(); + + await deleteGrant(grant_uuid); + + expect(res).toContain(dataset_uuid); + + }); + + test('User has relevant permission to multiple datasets', async () => { + const datasets = [ + Test_Uuids.Src_Datasets.TestDeviceDataset, + Test_Uuids.Session_Datasets.TestSessionAllDataset + ]; + + // Add read grants + let grants = []; + for (let dataset_uuid of datasets){ + const grant_uuid = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + dataset_uuid, + false + ) + grants.push(grant_uuid); + } + + await sleep(1000); + + const res = await test_fplus.DataAccess.get_metadata_list(); + + // Revoke grants + for(let grant of grants){ + await deleteGrant(grant); + } + expect(res).toHaveLength(datasets.length); + expect(res).toEqual(expect.arrayContaining(datasets)); + }); + + test('Multiple datasets, some have relevant permission', async () => { + const datasets = { + Readable: Test_Uuids.Src_Datasets.TestDeviceDataset, + Unreadable: Test_Uuids.Session_Datasets.TestSessionAllDataset + }; + + // ======== add grants ====== + let grants = []; + const read_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + datasets.Readable, + false + ); + grants.push(read_grant); + + const edit_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + datasets.Unreadable, + false + ); + grants.push(edit_grant); + + await sleep(1000); + + // ========================= + + + const res = await test_fplus.DataAccess.get_metadata_list(); + + // Revoke grants + for(let grant of grants){ + await deleteGrant(grant); + } + expect(res).toHaveLength(1); + expect(res).toContain(datasets.Readable); + }); + + test('Multiple (simple SRC) datasets with Read permissions, invalid ones must be excluded', async () => { + const datasets = [ + Test_Uuids.Src_Datasets.TestDeviceDataset, + Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset, + Test_Uuids.Src_Datasets.Node2_TestFloutDeviceDataset + ]; + + + // ----- add grants --- + const grants = []; + for (let uuid of datasets){ + const read_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + uuid, + false + ); + grants.push(read_grant); + } + + await sleep(1000); + + // --------------------- + + + const invalid_dataset = Test_Uuids.Src_Datasets.TestDeviceDataset; + const invalid_structure = Constants.App.UnionComponents; + + + // make dataset invalid + await addConfig( + invalid_structure, + invalid_dataset, + [] + ); + await sleep(1000); + + // ========================= + + const res = await test_fplus.DataAccess.get_metadata_list(); + + await sleep(1000); + + // ==== remove invalid config ====== + await removeConfig(invalid_structure, invalid_dataset); + await sleep(500); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + expect(res).toHaveLength(2); + expect(res).not.toContain(invalid_dataset); + }, 30000); + + + test('Invalid Parent (Union) composed of only SRC does not invalidate children', async () => { + const invalid_parent_dataset = Test_Uuids.Union_Datasets.TestDoublesUnionDataset; + const invalid_structure = Constants.App.SparkplugSrc; + + const child_datasets = [ + Test_Uuids.Src_Datasets.TestDeviceDataset, + Test_Uuids.Src_Datasets.TestDoubleDeviceDataset + ]; + + const datasets = [ + invalid_parent_dataset, + ...child_datasets + ]; + + // ---- add READ grants --- + const grants = []; + for (let uuid of datasets){ + const read_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + uuid, + false + ); + grants.push(read_grant); + } + + await sleep(1000); + + + // ---- Make Union dataset invalid + await addConfig( + invalid_structure, + invalid_parent_dataset, + { + "source": Test_Uuids.Devices.Node2_TestDoubleDevice + } + ); + + await sleep(500); + + + const res = await test_fplus.DataAccess.get_metadata_list(); + + // ---- Make Union dataset valid again + await removeConfig( + invalid_structure, + invalid_parent_dataset + ); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + expect(res).toHaveLength(child_datasets.length); + expect(res).not.toContain(invalid_parent_dataset); + + }, 30000); + + + test('Invalid Parent (Union) composed of Session and Union does NOT invalidate children', async () => { + + const invalid_parent_dataset = Test_Uuids.Union_Datasets.TestNestedUnionDataset; + const invalid_structure = Constants.App.SparkplugSrc; + + const child_datasets = [ + Test_Uuids.Session_Datasets.SessionNode2DoubleDataset, + Test_Uuids.Union_Datasets.TestDoublesUnionDataset + ]; + + const datasets = [ + invalid_parent_dataset, + ...child_datasets + ]; + + // ---- add READ grants --- + const grants = []; + for (let uuid of datasets){ + const read_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + uuid, + false + ); + grants.push(read_grant); + } + + await sleep(1000); + + + // ---- Make Union dataset invalid + await addConfig( + invalid_structure, + invalid_parent_dataset, + { + "source": Test_Uuids.Devices.Node2_TestDoubleDevice + } + ); + + await sleep(500); + + + const res = await test_fplus.DataAccess.get_metadata_list(); + + // ---- Make Union dataset valid again + await removeConfig( + invalid_structure, + invalid_parent_dataset + ); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + expect(res).toHaveLength(child_datasets.length); + expect(res).not.toContain(invalid_parent_dataset); + }, 30000); + + + test('Invalid Parent (Session) dataset composed of (Union) child does NOT invalidate children', async () => { + const invalid_parent_dataset = Test_Uuids.Session_Datasets.TestSessionAllDataset; + const invalid_structure = Constants.App.SparkplugSrc; + + const child_datasets = [Test_Uuids.Union_Datasets.TestUnionAllDataset]; + + const datasets = [ + invalid_parent_dataset, + ...child_datasets + ]; + + // ---- add READ grants --- + const grants = []; + for (let uuid of datasets){ + const read_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + uuid, + false + ); + grants.push(read_grant); + } + + await sleep(1000); + + + // ---- Make Union dataset invalid + await addConfig( + invalid_structure, + invalid_parent_dataset, + { + "source": Test_Uuids.Devices.Node2_TestDoubleDevice + } + ); + + await sleep(500); + + const res = await test_fplus.DataAccess.get_metadata_list(); + + // ---- Make Union dataset valid again + await removeConfig( + invalid_structure, + invalid_parent_dataset + ); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + + expect(res).toHaveLength(child_datasets.length); + expect(res).not.toContain(invalid_parent_dataset); + + }, 30000); + + + + test('Invalid Child (SRC) invalidates Parent (Union)', async () => { + const invalid_child_dataset = Test_Uuids.Src_Datasets.TestDeviceDataset; + const invalid_structure = Constants.App.SessionLimits; + + const parent_dataset = Test_Uuids.Union_Datasets.TestUnionAllDataset; + + const datasets = [ + invalid_child_dataset, + parent_dataset + ]; + + // ---- add READ grants --- + const grants = []; + for (let uuid of datasets){ + const read_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + uuid, + false + ); + grants.push(read_grant); + } + + await sleep(1000); + + + // ---- Make Union dataset invalid + await addConfig( + invalid_structure, + invalid_child_dataset, + { + "source": "invalid", + "from": "invalid", + "to": "invalid" + } + ); + + await sleep(500); + + + const res = await test_fplus.DataAccess.get_metadata_list(); + + // ---- Make Union dataset valid again + await removeConfig( + invalid_structure, + invalid_child_dataset + ); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + + expect(res).toHaveLength(0); + + }, 30000); + + + + + test('Invalid Child (Union) invalidates Parent (Session)', async () => { + const invalid_child_dataset = Test_Uuids.Union_Datasets.TestUnionAllDataset; + const invalid_structure = Constants.App.SparkplugSrc; + const parent_dataset = Test_Uuids.Session_Datasets.TestSessionAllDataset; + + const datasets = [ + invalid_child_dataset, + parent_dataset + ]; + + // ---- add READ grants --- + const grants = []; + for (let uuid of datasets){ + const read_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + uuid, + false + ); + grants.push(read_grant); + } + + await sleep(1000); + + + // ---- Make Union dataset invalid + await addConfig( + invalid_structure, + invalid_child_dataset, + { + "source": "invalid" + } + ); + + await sleep(500); + + + const res = await test_fplus.DataAccess.get_metadata_list(); + + // ---- Make Union dataset valid again + await removeConfig( + invalid_structure, + invalid_child_dataset + ); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + + expect(res).toHaveLength(0); + + }, 30000); + + + + test('Invalid Child (Session) invalidates Parent (Union)', async () => { + const invalid_child_dataset = Test_Uuids.Session_Datasets.SessionNode2DoubleDataset; + const invalid_structure = Constants.App.SparkplugSrc; + + const valid_child_dataset = Test_Uuids.Union_Datasets.TestDoublesUnionDataset; + const parent_dataset = Test_Uuids.Union_Datasets.TestNestedUnionDataset; + + const datasets = [ + invalid_child_dataset, + parent_dataset, + valid_child_dataset + ]; + + // ---- add READ grants --- + const grants = []; + for (let uuid of datasets){ + const read_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + uuid, + false + ); + grants.push(read_grant); + } + + await sleep(1000); + + + // ---- Make Union dataset invalid + await addConfig( + invalid_structure, + invalid_child_dataset, + { + "source": "invalid" + } + ); + + await sleep(500); + + + const res = await test_fplus.DataAccess.get_metadata_list(); + + // ---- Make Union dataset valid again + await removeConfig( + invalid_structure, + invalid_child_dataset + ); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + + expect(res).not.toContain(parent_dataset); + expect(res).not.toContain(invalid_child_dataset); + expect(res).toContain(valid_child_dataset); + + }, 30000); +}); + + diff --git a/acs-data-access/tests/http/metadata_single.test.js b/acs-data-access/tests/http/metadata_single.test.js new file mode 100644 index 000000000..6f30aae98 --- /dev/null +++ b/acs-data-access/tests/http/metadata_single.test.js @@ -0,0 +1,316 @@ +import {ServiceClient} from "@amrc-factoryplus/service-client"; +import process from "node:process"; +import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest'; +import { DataAccess as Constants } from "../../lib/constants.js"; +import * as Temp_Data from '../temp_data.js'; + + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +describe('GET /metadata/:uuid', () => { + const TEST_PRINCIPAL=process.env.TEST_HUMAN_UUID; + + let test_fplus; + let admin_fplus; + let ALL_GRANTS = []; + + beforeEach(async () => { + admin_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.ADMIN_USERNAME, + password: process.env.ADMIN_PASSWORD, + verbose: 'ALL' + }); + + test_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.TEST_HUMAN_USERNAME, + password: process.env.TEST_HUMAN_PASSWORD, + verbose: 'ALL' + }); + await sleep(1000); + }); + + + // Clean up grants if not removed in case of failure. + afterAll(async () => { + for (let grant of ALL_GRANTS){ + await deleteGrant(grant); + } + }); + + + async function addGrant(principal, permission, target, plural){ + if(!principal || + !permission || + !target + ) return; + + const grant_uuid = await admin_fplus.Auth.add_grant({ + principal, + permission, + target, + plural + }); + ALL_GRANTS.push(grant_uuid); + return grant_uuid; + } + + async function deleteGrant(grant_uuid){ + if(!grant_uuid) return; + + await admin_fplus.Auth.delete_grant(grant_uuid) + ALL_GRANTS = ALL_GRANTS.filter(uuid => uuid !== grant_uuid); + } + + async function deleteDataset(uuid){ + if(!uuid) return; + + await admin_fplus.DataAccess.delete_dataset(uuid); + } + + async function addConfig(app, obj, config){ + if(!app || + !obj || + !config + ) return; + + await admin_fplus.ConfigDB.put_config( app, obj, config ); + } + + async function removeConfig(app, obj){ + if(!app || !obj) return; + + await admin_fplus.ConfigDB.delete_config( app, obj ); + } + + test('Dataset exist but no permission', async () => { + const testCase = Temp_Data.Test_Uuids.Src_Datasets.TestDeviceDataset; + + try { + const res = await test_fplus.DataAccess.get_single_metadata(testCase); + expect.fail('Expected to throw'); + } + catch (err) { + expect(err.status).not.toBe(200); + } + }); + + test('User has irrelevant permission to the dataset', async () => { + const testCase = Temp_Data.Test_Uuids.Src_Datasets.TestDeviceDataset; + + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + testCase, + false + ); + + try { + const res = await test_fplus.DataAccess.get_single_metadata(testCase); + expect.fail('Expected to throw'); + } + catch (err) { + expect(err.status).not.toBe(200); + } + finally{ + await deleteGrant(grant); + + } + }); + + test('Invalid uuid', async () => { + const testCase = 'xxx'; + + try { + const res = await test_fplus.DataAccess.get_single_metadata(testCase); + expect.fail('Expected to throw'); + } + catch (err) { + expect(err.status).not.toBe(200); + } + }); + + test('Invalid dataset -> Not found', async () => { + const testCase = Temp_Data.Test_Uuids.Session_Datasets.SessionNode2DoubleDataset; + const invalidApp = Constants.App.UnionComponents; + + // Grant + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + testCase, + false + ); + + await sleep(3600); + + const res_valid = await test_fplus.DataAccess.get_single_metadata(testCase); + + // make testCase invalid + await addConfig( + invalidApp, + testCase, + [] + ); + await sleep(3600); + + try{ + await test_fplus.DataAccess.get_single_metadata(testCase); + expect.fail('Expected to throw'); + }catch(err){ + expect(err.status).not.toBe(200); + } + + // make testCase valid + await removeConfig(invalidApp, testCase); + + // revoke grant + await deleteGrant(grant); + + expect(res_valid).toHaveProperty('uuid'); + + }, 40000); + + + /** Session type metadata object props: + * - uuid + * - name + * - from + * - to + * - function (opt) + * - metadata (opt) + * - parts + */ + test('Session format', async () => { + const testCase = Temp_Data.Test_Uuids.Session_Datasets.SessionNode2DoubleDataset; + + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + testCase, + false + ); + + await sleep(3600); + + const res = await test_fplus.DataAccess.get_single_metadata(testCase); + + console.log(res); + + await deleteGrant(grant); + + expect(res).toEqual(expect.objectContaining({ + 'uuid': testCase, + 'name': 'SessionNode2DoubleDataset', + 'from': '2026-06-01T08:52:00.000Z', + 'to': '2026-06-01T11:55:00.000Z', + 'parts': [testCase], + })); + + expect(res.function).toEqual(expect.arrayContaining( + [ + Temp_Data.Test_Uuids.Mes.MesDataset, + Temp_Data.Test_Uuids.Mes.MesEquipment, + Temp_Data.Test_Uuids.Mes.MesOperation, + Temp_Data.Test_Uuids.Mes.MesProduct, + Temp_Data.Test_Uuids.Mes.MesWorkorder, + ] + )); + expect(res.metadata).toHaveProperty(Temp_Data.Test_Uuids.Mes.App); + }); + + test('SRC format', async () => { + const testCase = Temp_Data.Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset; + + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + testCase, + false + ); + + await sleep(3600); + + const res = await test_fplus.DataAccess.get_single_metadata(testCase); + + console.log(res); + + await deleteGrant(grant); + + expect(res).not.haveOwnProperty('from'); + expect(res).not.haveOwnProperty('to'); + + + expect(res).toEqual(expect.objectContaining({ + 'uuid': testCase, + 'name': 'Node2_TestDoubleDeviceDataset', + 'parts': [testCase] + })); + + }); + + +// { +// uuid: '21e0dfa8-f044-4e87-8606-f528686205d8', +// name: 'TestNestedUnionDataset', +// from: '2026-04-02T13:00:00.000Z', +// to: '2026-06-01T11:55:00.000Z', +// parts: [ '21e0dfa8-f044-4e87-8606-f528686205d8' ] +// } + + +// { +// "to": "2026-06-01T11:55:00.000Z", +// "from": "2026-06-01T08:52:00.000Z", +// "source": "720ecd5a-c5d2-49d5-bf5a-8ca01dfdb7df" +// } + + +// { +// "to": "2026-06-01T10:00:00.000Z", +// "from": "2026-04-02T13:00:00.000Z", +// "source": "e2a4c530-dc0f-417d-b00b-329b0e90e033" +// } + test('Union format', async () => { + const testCase = Temp_Data.Test_Uuids.Union_Datasets.TestNestedUnionDataset; + + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + Constants.Class.Dataset, + true + ); + + await sleep(3600); + + const res = await test_fplus.DataAccess.get_single_metadata(testCase); + + console.log(res); + + await deleteGrant(grant); + + + expect(res).toEqual(expect.objectContaining({ + 'from': '2026-04-02T13:00:00.000Z', + 'to': '2026-06-01T11:55:00.000Z', + 'uuid': testCase, + 'name': 'TestNestedUnionDataset', + })); + + expect(res.parts).toEqual(expect.arrayContaining( + [ + Temp_Data.Test_Uuids.Session_Datasets.SessionNode2DoubleDataset, + Temp_Data.Test_Uuids.Session_Datasets.TestSessionAllDataset + ] + )); + expect(res.function).toEqual(expect.arrayContaining( + [ + Temp_Data.Test_Uuids.Mes.MesOperation, + Temp_Data.Test_Uuids.Mes.MesDataset + ] + )); + }); +}); \ No newline at end of file diff --git a/acs-data-access/tests/http/single_structure.test.js b/acs-data-access/tests/http/single_structure.test.js new file mode 100644 index 000000000..711cf4ec2 --- /dev/null +++ b/acs-data-access/tests/http/single_structure.test.js @@ -0,0 +1,199 @@ +import {ServiceClient} from "@amrc-factoryplus/service-client"; +import process from "node:process"; +import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest'; +import { DataAccess as Constants } from "../../lib/constants.js"; +import * as Temp_Data from '../temp_data.js'; + + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +describe('GET /structure/:uuid', () => { + const TEST_PRINCIPAL=process.env.TEST_HUMAN_UUID; + + let test_fplus; + let admin_fplus; + let ALL_GRANTS = []; + + beforeEach(async () => { + admin_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.ADMIN_USERNAME, + password: process.env.ADMIN_PASSWORD, + verbose: 'ALL' + }); + + test_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.TEST_HUMAN_USERNAME, + password: process.env.TEST_HUMAN_PASSWORD, + verbose: 'ALL' + }); + await sleep(1000); + }); + + + // Clean up grants if not removed in case of failure. + afterAll(async () => { + for (let grant of ALL_GRANTS){ + await deleteGrant(grant); + } + }); + + + async function addGrant(principal, permission, target, plural){ + if(!principal || + !permission || + !target + ) return; + + const grant_uuid = await admin_fplus.Auth.add_grant({ + principal, + permission, + target, + plural + }); + ALL_GRANTS.push(grant_uuid); + return grant_uuid; + } + + async function deleteGrant(grant_uuid){ + if(!grant_uuid) return; + + await admin_fplus.Auth.delete_grant(grant_uuid) + ALL_GRANTS = ALL_GRANTS.filter(uuid => uuid !== grant_uuid); + } + + async function deleteDataset(uuid){ + if(!uuid) return; + + await admin_fplus.DataAccess.delete_dataset(uuid); + } + + async function addConfig(app, obj, config){ + if(!app || + !obj || + !config + ) return; + + await admin_fplus.ConfigDB.put_config( app, obj, config ); + } + + async function removeConfig(app, obj){ + if(!app || !obj) return; + + await admin_fplus.ConfigDB.delete_config( app, obj ); + } + + test('Dataset exist but no permission', async () => { + const testCase = Temp_Data.Test_Uuids.Src_Datasets.TestDeviceDataset; + + try { + const res = await test_fplus.DataAccess.get_single_structure(testCase); + expect.fail('Expected to throw'); + } + catch (err) { + expect(err.status).not.toBe(200); + } + }); + + test('User has irrelevant permission to the dataset', async () => { + const testCase = Temp_Data.Test_Uuids.Src_Datasets.TestDeviceDataset; + + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + testCase, + false + ); + + try { + const res = await test_fplus.DataAccess.get_single_structure(testCase); + expect.fail('Expected to throw'); + } + catch (err) { + expect(err.status).not.toBe(200); + } + finally{ + await deleteGrant(grant); + } + }); + + test('Invalid uuid', async () => { + const testCase = 'xxx'; + + try { + const res = await test_fplus.DataAccess.get_single_structure(testCase); + expect.fail('Expected to throw'); + } + catch (err) { + expect(err.status).not.toBe(200); + } + }); + + + test('Invalid dataset -> Special UUID', async () => { + const testCase = Temp_Data.Test_Uuids.Session_Datasets.SessionNode2DoubleDataset; + const invalidApp = Constants.App.UnionComponents; + + // Grant + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + testCase, + false + ); + + await sleep(3600); + + + // make testCase invalid + await addConfig( + invalidApp, + testCase, + [] + ); + await sleep(3600); + + + const res_invalid = await test_fplus.DataAccess.get_single_structure(testCase); + await sleep(1000); + + + + await removeConfig(invalidApp, testCase); + await deleteGrant(grant); + + + expect(res_invalid.structure).toEqual(Constants.Special.InvalidDataset); + expect(res_invalid.config).toBe(null); + + }, 40000); + + test('SRC', async () => { + const testCase = Temp_Data.Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset; + + + // Grant + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + testCase, + false + ); + + await sleep(3600); + + const res = await test_fplus.DataAccess.get_single_structure(testCase); + console.log(res); + + + await deleteGrant(grant); + + expect(res.structure).toEqual(Constants.App.SparkplugSrc); + expect(res.config).not.toBe(null); + expect(res.config).toHaveProperty('source'); + }, 40000); + +}); \ No newline at end of file diff --git a/acs-data-access/tests/http/structure_create.test.js b/acs-data-access/tests/http/structure_create.test.js new file mode 100644 index 000000000..bf2d4c396 --- /dev/null +++ b/acs-data-access/tests/http/structure_create.test.js @@ -0,0 +1,324 @@ +import {ServiceClient} from "@amrc-factoryplus/service-client"; +import process from "node:process"; +import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest'; +import { DataAccess as Constants } from "../../lib/constants.js"; +import * as Temp_Data from '../temp_data.js'; + + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +describe('POST /structure', () => { + const TEST_PRINCIPAL=process.env.TEST_HUMAN_UUID; + + let test_fplus; + let admin_fplus; + let ALL_GRANTS = []; + + const ALL_INVALID_BODY_CASES = [ + ...Temp_Data.Invalid_Body_Structure, + ...Temp_Data.Invalid_Body_Config.SRC, + ...Temp_Data.Invalid_Body_Config.Session, + ...Temp_Data.Invalid_Body_Config.Union, + ]; + + const ALL_VALID_BODY_CASES = [ + Temp_Data.Valid_Body.SRC, + Temp_Data.Valid_Body.Session, + Temp_Data.Valid_Body.Union + ]; + + beforeEach(async () => { + admin_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.ADMIN_USERNAME, + password: process.env.ADMIN_PASSWORD, + verbose: 'ALL' + }); + + test_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.TEST_HUMAN_USERNAME, + password: process.env.TEST_HUMAN_PASSWORD, + verbose: 'ALL' + }); + await sleep(1000); + }); + + + // Clean up grants if not removed in case of failure. + afterAll(async () => { + for (let grant of ALL_GRANTS){ + await deleteGrant(grant); + } + }); + + + async function addGrant(principal, permission, target, plural){ + if(!principal || + !permission || + !target + ) return; + + const grant_uuid = await admin_fplus.Auth.add_grant({ + principal, + permission, + target, + plural + }); + ALL_GRANTS.push(grant_uuid); + return grant_uuid; + } + + async function deleteGrant(grant_uuid){ + if(!grant_uuid) return; + + await admin_fplus.Auth.delete_grant(grant_uuid) + ALL_GRANTS = ALL_GRANTS.filter(uuid => uuid !== grant_uuid); + } + + async function deleteDataset(uuid){ + if(!uuid) return; + + await admin_fplus.DataAccess.delete_dataset(uuid); + } + + + + test('User has no CreateDataset permission', async () => { + try { + await test_fplus.DataAccess.create_dataset( + Temp_Data.Valid_Body.SRC.structure, + Temp_Data.Valid_Body.SRC.config + ); + + expect.fail('Expected create_dataset to throw'); + } + catch (err) { + expect(err.status).not.toBe(200); + } + }); + + test('User has CreateDataset permission for irrelevant structure', async () => { + const irrelevant_structure = Constants.App.SessionLimits; + + const irrelevant_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.CreateDataset, + irrelevant_structure, + false + ); + await sleep(500); + + try { + await test_fplus.DataAccess.create_dataset( + Temp_Data.Valid_Body.Session.structure, + Temp_Data.Valid_Body.Session.config + ); + + expect.fail('Expected create_dataset to throw'); + } + catch (err) { + expect(err.status).not.toBe(200); + } + + await deleteGrant(irrelevant_grant); + }); + + + + test.each(ALL_INVALID_BODY_CASES)('Invalid body cases - case %#', async (testCase) => { + try { + + await test_fplus.DataAccess.create_dataset(testCase.structure, testCase.config); + expect.fail('Expected create_dataset to throw'); + + } catch (err) { + + expect(err.status).not.toBe(200); + } + }); + + test.each(ALL_VALID_BODY_CASES)('Missing 2nd level permission cases - case %#', async (testCase) => { + try{ + // Add 1st level grant + + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.CreateDataset, + testCase.structure, + false + ); + + await test_fplus.DataAccess.create_dataset(testCase.structure, testCase.config); + + + expect.fail('Expected create_dataset to throw'); + + // Revoke 1st level grant + await deleteGrant(grant); + } catch(err){ + console.log(err); + expect(err.status).not.toBe(200); + } + }); + + + test.each(ALL_VALID_BODY_CASES)('Create - case %#', async (testCase) => { + + let grants = []; + + // add 1st level grant + const grant_uuid = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.CreateDataset, + testCase.structure, + false + ); + grants.push(grant_uuid); + + await sleep(500); + + + if(testCase.structure === Constants.App.UnionComponents){ + for(let src of testCase.config){ + const g = await addGrant( + TEST_PRINCIPAL, + testCase.secondLevelPerm, + src, + false + ); + grants.push(g); + await sleep(1000); + } + }else{ + const g = await addGrant( + TEST_PRINCIPAL, + testCase.secondLevelPerm, + testCase.config.source, + false + ); + grants.push(g); + await sleep(1000); + } + + const dataset_uuid = await test_fplus.DataAccess.create_dataset( + testCase.structure, + testCase.config + ); + + console.log("HERE IS THE NEW OBJECT CREATED: ", dataset_uuid); + + for(let grant of grants){ + await deleteGrant(grant); + await sleep(1000); + } + + // await deleteConfigDBObj(obj_uuid); + await deleteDataset(dataset_uuid); + await sleep(1000); + + expect(dataset_uuid).not.toBe(undefined); + }, 30000); + + + // /* =============================================== */ + // /* -------- Test subclass relationships ---------- */ + // /* =============================================== */ + test('Session dataset becomes subclass of its source dataset', async () => { + const testCase = Temp_Data.Valid_Body.Session; + // 1. grants: CreateDataset and IncludeInSession for src + let grants = []; + + const create_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.CreateDataset, + testCase.structure, + false + ); + grants.push(create_grant); + + await sleep(1000); + + const scnd_lvl_grant = await addGrant( + TEST_PRINCIPAL, + testCase.secondLevelPerm, + testCase.config.source, + false + ); + + grants.push(scnd_lvl_grant); + await sleep(3000); + + // 2. dataset_uuid = create session dataset + const dataset_uuid = await test_fplus.DataAccess.create_dataset(testCase.structure, testCase.config); + + // 3. query the source's subclasses + const subclasses = await admin_fplus.ConfigDB.class_subclasses(testCase.config.source); + + // 4. revoke grants + for(let grant of grants){ + await deleteGrant(grant); + await sleep(1000); + } + + // 5. Delete dataset + await deleteDataset(dataset_uuid); + + + // 6. assert source's subclasses include new dataset_uuid + expect(subclasses).toContainEqual(dataset_uuid); + }, 30000); + + test('Union dataset becomes superclass of its sources ', async () => { + const testCase = Temp_Data.Valid_Body.Union; + // 1. grants: CreateDataset and UseInUnion for src + let grants = []; + + const create_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.CreateDataset, + testCase.structure, + false + ); + grants.push(create_grant); + + await sleep(500); + for (const s of testCase.config){ + const g = await addGrant( + TEST_PRINCIPAL, + testCase.secondLevelPerm, + s, + false + ); + grants.push(g); + await sleep(100); + } + + + // 2. dataset_uuid = create dataset + const dataset_uuid = await test_fplus.DataAccess.create_dataset(testCase.structure, testCase.config); + + await sleep(1000); + + // 3. query the dataset's subclasses + const subclasses = await admin_fplus.ConfigDB.class_subclasses(dataset_uuid); + + await sleep(500); + + + // 4. revoke grants + for(let grant of grants){ + await deleteGrant(grant); + await sleep(1000); + } + + // 5. delete dataset + await deleteDataset(dataset_uuid); + + // 6. assert new dataset's subclasses include all the config sources + expect(subclasses).toEqual(expect.arrayContaining(testCase.config)); + + }, 30000); +}); \ No newline at end of file diff --git a/acs-data-access/tests/http/structure_list.test.js b/acs-data-access/tests/http/structure_list.test.js new file mode 100644 index 000000000..9678d909c --- /dev/null +++ b/acs-data-access/tests/http/structure_list.test.js @@ -0,0 +1,598 @@ +import {ServiceClient} from "@amrc-factoryplus/service-client"; +import process from "node:process"; +import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest'; +import { DataAccess as Constants } from "../../lib/constants.js"; +import * as Temp_Data from '../temp_data.js'; + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +describe('GET /structure', () => { + const TEST_PRINCIPAL=process.env.TEST_HUMAN_UUID; + + let test_fplus; + let admin_fplus; + let ALL_GRANTS = []; + + beforeEach(async () => { + admin_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.ADMIN_USERNAME, + password: process.env.ADMIN_PASSWORD, + verbose: 'ALL' + }); + + test_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.TEST_HUMAN_USERNAME, + password: process.env.TEST_HUMAN_PASSWORD, + verbose: 'ALL' + }); + await sleep(500); + }); + + + // Clean up grants if not removed in case of failure. + afterAll(async () => { + for (let grant of ALL_GRANTS){ + await deleteGrant(grant); + } + }); + + + async function addGrant(principal, permission, target, plural){ + if(!principal || + !permission || + !target + ) return; + + const grant_uuid = await admin_fplus.Auth.add_grant({ + principal, + permission, + target, + plural + }); + ALL_GRANTS.push(grant_uuid); + return grant_uuid; + } + + async function deleteGrant(grant_uuid){ + if(!grant_uuid) return; + + await admin_fplus.Auth.delete_grant(grant_uuid) + ALL_GRANTS = ALL_GRANTS.filter(uuid => uuid !== grant_uuid); + } + + async function addConfig(app, obj, config){ + if(!app || + !obj || + !config + ) return; + + await admin_fplus.ConfigDB.put_config( app, obj, config ); + } + + async function removeConfig(app, obj){ + if(!app || !obj) return; + + await admin_fplus.ConfigDB.delete_config( app, obj ); + } + + test('Datasets exist but user has no permissions', async () => { + const res = await test_fplus.DataAccess.get_structure_list(); + expect(res).toEqual([]); + }); + + test('User has unrelated permission to one dataset', async () => { + const dataset_uuid = Test_Uuids.Src_Datasets.TestDeviceDataset; + + // Grant unrelated permission to the user + const grant_uuid = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + dataset_uuid, + false + ); + await sleep(1000); + + const res = await test_fplus.DataAccess.get_structure_list(); + + await deleteGrant(grant_uuid); + + expect(res).toEqual([]); + }); + + test('User has correct permission to one dataset', async () => { + const dataset_uuid = Test_Uuids.Src_Datasets.TestDeviceDataset; + + // Grant correct permission + const grant_uuid = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + dataset_uuid, + false + ); + await sleep(1000); + + const res = await test_fplus.DataAccess.get_structure_list(); + + await deleteGrant(grant_uuid); + + expect(res).toContain(dataset_uuid); + + }); + + test('User has correct permission to multiple datasets', async () => { + const datasets = [ + Test_Uuids.Src_Datasets.TestDeviceDataset, + Test_Uuids.Session_Datasets.TestSessionAllDataset, + ]; + + + // Add grants + let grants = []; + + for (let dataset_uuid of datasets){ + const grant_uuid = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + dataset_uuid, + false + ) + grants.push(grant_uuid); + } + + await sleep(1000); + + const res = await test_fplus.DataAccess.get_structure_list(); + + // Revoke grants + for(let grant of grants){ + await deleteGrant(grant); + } + expect(res).toHaveLength(datasets.length); + expect(res).toEqual(expect.arrayContaining(datasets)); + }); + + test('Multiple datasets, some have relevant permission', async () => { + const datasets = { + Relevant: Test_Uuids.Src_Datasets.TestDeviceDataset, + Irrelevant: Test_Uuids.Session_Datasets.TestSessionAllDataset + }; + + // ======== add grants ====== + let grants = []; + const relevant_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + datasets.Relevant, + false + ); + grants.push(relevant_grant); + + const irrelevant_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.ReadDataset, + datasets.Irrelevant, + false + ); + grants.push(irrelevant_grant); + + await sleep(1000); + + // ========================= + + + const res = await test_fplus.DataAccess.get_structure_list(); + + // Revoke grants + for(let grant of grants){ + await deleteGrant(grant); + } + expect(res).toHaveLength(1); + expect(res).toContain(datasets.Relevant); + expect(res).not.toContain(datasets.Irrelevant); + }); + + + test('Multiple (simple SRC) datasets with relevant permissions, invalid ones must be invcluded', async () => { + const datasets = [ + Test_Uuids.Src_Datasets.TestDeviceDataset, + Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset, + Test_Uuids.Src_Datasets.Node2_TestFloutDeviceDataset + ]; + + + // ----- add grants --- + const grants = []; + for (let uuid of datasets){ + const relevant_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + uuid, + false + ); + grants.push(relevant_grant); + } + + await sleep(1000); + + // --------------------- + + + const invalid_dataset = Test_Uuids.Src_Datasets.TestDeviceDataset; + const invalid_structure = Constants.App.UnionComponents; + + + // make dataset invalid + await addConfig( + invalid_structure, + invalid_dataset, + [] + ); + await sleep(1000); + + // ========================= + + const res = await test_fplus.DataAccess.get_structure_list(); + + await sleep(1000); + + // ==== remove invalid config ====== + await removeConfig(invalid_structure, invalid_dataset); + await sleep(500); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + expect(res).toHaveLength(datasets.length); + expect(res).toEqual(expect.arrayContaining(datasets)); + }, 30000); + + + test('Invalid Parent (Union) composed of only SRC - all included', async () => { + const invalid_parent_dataset = Test_Uuids.Union_Datasets.TestDoublesUnionDataset; + const invalid_structure = Constants.App.SparkplugSrc; + + const child_datasets = [ + Test_Uuids.Src_Datasets.TestDeviceDataset, + Test_Uuids.Src_Datasets.TestDoubleDeviceDataset + ]; + + const datasets = [ + invalid_parent_dataset, + ...child_datasets + ]; + + // ---- add grants --- + const grants = []; + for (let uuid of datasets){ + const relevant_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + uuid, + false + ); + grants.push(relevant_grant); + } + + await sleep(1000); + + + // ---- Make Union dataset invalid + await addConfig( + invalid_structure, + invalid_parent_dataset, + { + "source": Test_Uuids.Devices.Node2_TestDoubleDevice + } + ); + + await sleep(500); + + + const res = await test_fplus.DataAccess.get_structure_list(); + + // ---- Make Union dataset valid again + await removeConfig( + invalid_structure, + invalid_parent_dataset + ); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + expect(res).toHaveLength(datasets.length); + expect(res).toEqual(expect.arrayContaining(datasets)); + + }, 30000); + + test('Invalid Parent (Union) composed of Session and Union - all included', async () => { + + const invalid_parent_dataset = Test_Uuids.Union_Datasets.TestNestedUnionDataset; + const invalid_structure = Constants.App.SparkplugSrc; + + const child_datasets = [ + Test_Uuids.Session_Datasets.SessionNode2DoubleDataset, + Test_Uuids.Union_Datasets.TestDoublesUnionDataset + ]; + + const datasets = [ + invalid_parent_dataset, + ...child_datasets + ]; + + // ---- add grants --- + const grants = []; + for (let uuid of datasets){ + const relevant_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + uuid, + false + ); + grants.push(relevant_grant); + } + + await sleep(1000); + + + // ---- Make Union dataset invalid + await addConfig( + invalid_structure, + invalid_parent_dataset, + { + "source": Test_Uuids.Devices.Node2_TestDoubleDevice + } + ); + + await sleep(500); + + + const res = await test_fplus.DataAccess.get_structure_list(); + + // ---- Make Union dataset valid again + await removeConfig( + invalid_structure, + invalid_parent_dataset + ); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + expect(res).toHaveLength(datasets.length); + expect(res).toEqual(expect.arrayContaining(datasets)); + }, 30000); + + + test('Invalid Parent (Session) dataset composed of (Union) - all included', async () => { + const invalid_parent_dataset = Test_Uuids.Session_Datasets.TestSessionAllDataset; + const invalid_structure = Constants.App.SparkplugSrc; + + const child_datasets = [Test_Uuids.Union_Datasets.TestUnionAllDataset]; + + const datasets = [ + invalid_parent_dataset, + ...child_datasets + ]; + + // ---- add grants --- + const grants = []; + for (let uuid of datasets){ + const relevant_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + uuid, + false + ); + grants.push(relevant_grant); + } + + await sleep(1000); + + + // ---- Make Union dataset invalid + await addConfig( + invalid_structure, + invalid_parent_dataset, + { + "source": Test_Uuids.Devices.Node2_TestDoubleDevice + } + ); + + await sleep(500); + + const res = await test_fplus.DataAccess.get_structure_list(); + + // ---- Make Union dataset valid again + await removeConfig( + invalid_structure, + invalid_parent_dataset + ); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + + expect(res).toHaveLength(datasets.length); + expect(res).toEqual(expect.arrayContaining(datasets)); + + }, 30000); + + test('Invalid Child (SRC) invalidates Parent (Union) - all included', async () => { + const invalid_child_dataset = Test_Uuids.Src_Datasets.TestDeviceDataset; + const invalid_structure = Constants.App.SessionLimits; + + const parent_dataset = Test_Uuids.Union_Datasets.TestUnionAllDataset; + + const datasets = [ + invalid_child_dataset, + parent_dataset + ]; + + // ---- add grants --- + const grants = []; + for (let uuid of datasets){ + const relevant_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + uuid, + false + ); + grants.push(relevant_grant); + } + + await sleep(1000); + + + // ---- Make Union dataset invalid + await addConfig( + invalid_structure, + invalid_child_dataset, + { + "source": "invalid", + "from": "invalid", + "to": "invalid" + } + ); + + await sleep(500); + + + const res = await test_fplus.DataAccess.get_structure_list(); + + // ---- Make Union dataset valid again + await removeConfig( + invalid_structure, + invalid_child_dataset + ); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + + expect(res).toHaveLength(datasets.length); + expect(res).toEqual(expect.arrayContaining(datasets)); + + }, 30000); + + test('Invalid Child (Union) invalidates Parent (Session) - all included', async () => { + const invalid_child_dataset = Test_Uuids.Union_Datasets.TestUnionAllDataset; + const invalid_structure = Constants.App.SparkplugSrc; + const parent_dataset = Test_Uuids.Session_Datasets.TestSessionAllDataset; + + const datasets = [ + invalid_child_dataset, + parent_dataset + ]; + + // ---- add grants --- + const grants = []; + for (let uuid of datasets){ + const relevant_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + uuid, + false + ); + grants.push(relevant_grant); + } + + await sleep(1000); + + + // ---- Make Union dataset invalid + await addConfig( + invalid_structure, + invalid_child_dataset, + { + "source": "invalid" + } + ); + + await sleep(500); + + + const res = await test_fplus.DataAccess.get_structure_list(); + + // ---- Make Union dataset valid again + await removeConfig( + invalid_structure, + invalid_child_dataset + ); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + + expect(res).toHaveLength(datasets.length); + expect(res).toEqual(expect.arrayContaining(datasets)); + + }, 30000); + + + test('Invalid Child (Session) invalidates Parent (Union)', async () => { + const invalid_child_dataset = Test_Uuids.Session_Datasets.SessionNode2DoubleDataset; + const invalid_structure = Constants.App.SparkplugSrc; + + const valid_child_dataset = Test_Uuids.Union_Datasets.TestDoublesUnionDataset; + const parent_dataset = Test_Uuids.Union_Datasets.TestNestedUnionDataset; + + const datasets = [ + invalid_child_dataset, + parent_dataset, + valid_child_dataset + ]; + + // ---- add grants --- + const grants = []; + for (let uuid of datasets){ + const relevant_grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + uuid, + false + ); + grants.push(relevant_grant); + } + + await sleep(1000); + + + // ---- Make Union dataset invalid + await addConfig( + invalid_structure, + invalid_child_dataset, + { + "source": "invalid" + } + ); + + await sleep(500); + + + const res = await test_fplus.DataAccess.get_structure_list(); + + // ---- Make Union dataset valid again + await removeConfig( + invalid_structure, + invalid_child_dataset + ); + + // ==== Revoke grants ==== + for (let uuid of grants){ + await deleteGrant(uuid); + } + + expect(res).toHaveLength(datasets.length); + expect(res).toEqual(expect.arrayContaining(datasets)); + + }, 30000); +}); \ No newline at end of file diff --git a/acs-data-access/tests/http/structure_update.test.js b/acs-data-access/tests/http/structure_update.test.js new file mode 100644 index 000000000..3ed64c91f --- /dev/null +++ b/acs-data-access/tests/http/structure_update.test.js @@ -0,0 +1,580 @@ +import {ServiceClient} from "@amrc-factoryplus/service-client"; +import process from "node:process"; +import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest'; +import { DataAccess as Constants } from "../../lib/constants.js"; +import * as Temp_Data from '../temp_data.js'; + + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + + +describe('PUT /structure', () => { + const TEST_PRINCIPAL=process.env.TEST_HUMAN_UUID; + + let test_fplus; + let admin_fplus; + let ALL_GRANTS = []; + + const ALL_INVALID_BODY_CASES = [ + ...Temp_Data.Invalid_Body_Structure, + ...Temp_Data.Invalid_Body_Config.SRC, + ...Temp_Data.Invalid_Body_Config.Session, + ...Temp_Data.Invalid_Body_Config.Union, + ]; + + const ALL_VALID_BODY_CASES = [ + Temp_Data.Valid_Body.SRC, + Temp_Data.Valid_Body.Session, + Temp_Data.Valid_Body.Union + ]; + + beforeEach(async () => { + admin_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.ADMIN_USERNAME, + password: process.env.ADMIN_PASSWORD, + verbose: 'ALL' + }); + + test_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.TEST_HUMAN_USERNAME, + password: process.env.TEST_HUMAN_PASSWORD, + verbose: 'ALL' + }); + await sleep(1000); + }); + + beforeAll(async () => { + admin_fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.ADMIN_USERNAME, + password: process.env.ADMIN_PASSWORD, + verbose: 'ALL' + }); + }); + + // Clean up grants if not removed in case of failure. + afterAll(async () => { + for (let grant of ALL_GRANTS){ + await deleteGrant(grant); + } + + }); + + async function addGrant(principal, permission, target, plural){ + if(!principal || + !permission || + !target + ) return; + + const grant_uuid = await admin_fplus.Auth.add_grant({ + principal, + permission, + target, + plural + }); + ALL_GRANTS.push(grant_uuid); + return grant_uuid; + } + + async function deleteGrant(grant_uuid){ + if(!grant_uuid) return; + + await admin_fplus.Auth.delete_grant(grant_uuid) + ALL_GRANTS = ALL_GRANTS.filter(uuid => uuid !== grant_uuid); + } + + async function deleteDataset(uuid){ + if(!uuid) return; + + await admin_fplus.DataAccess.delete_dataset(uuid); + } + + async function createDataset(structure, config){ + if(!structure || !config) return; + + const uuid = await admin_fplus.DataAccess.create_dataset( + structure, + config + ) + + return uuid; + } + async function addConfig(app, obj, config){ + if(!app || + !obj || + !config + ) return; + + await admin_fplus.ConfigDB.put_config( app, obj, config ); + } + + async function removeConfig(app, obj){ + if(!app || !obj) return; + + await admin_fplus.ConfigDB.delete_config( app, obj ); + } + + + test('Irrelevant permission', async() => { + const testCase = Temp_Data.Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset; + + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.CreateDataset, + testCase, + false + ); + + try { + await test_fplus.DataAccess.update_dataset( + testCase, + Constants.App.SparkplugSrc, + Temp_Data.Valid_Body.SRC.config + ); + + expect.fail('Expected to throw'); + } + catch (err) { + expect(err.status).not.toBe(200); + } + await deleteGrant(grant); + }); + + + test('Invalid Dataset UUID', async () => { + try { + await test_fplus.DataAccess.update_dataset( + "xxx", + Constants.App.SparkplugSrc, + Temp_Data.Valid_Body.SRC.config + ); + + expect.fail('Expected to throw'); + } + catch (err) { + expect(err.status).not.toBe(200); + } + }); + + test.each(ALL_INVALID_BODY_CASES)('Invalid body cases - case %#', async (testCase) => { + try { + await test_fplus.DataAccess.update_dataset( + Temp_Data.Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset, + testCase.structure, + testCase.config + ); + expect.fail('Expected create_dataset to throw'); + + } catch (err) { + expect(err.status).not.toBe(200); + } + }, 30000); + + + test.each([ + { // Case 1: SRC + uuid: Temp_Data.Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset, + structure: Constants.App.UnionComponents, // intentionally wrong + config: Temp_Data.Valid_Body.Union.config // intentionally wrong + }, + { // Case 2: Session + uuid: Temp_Data.Test_Uuids.Session_Datasets.SessionNode2DoubleDataset, + structure: Constants.App.UnionComponents, // intentionally wrong + config: Temp_Data.Valid_Body.Union.config + }, + { + // Case 3: Union + uuid: Temp_Data.Test_Uuids.Union_Datasets.TestUnionAllDataset, + structure: Constants.App.SparkplugSrc, // intentionally wrong + config: Temp_Data.Valid_Body.SRC.config + } + ])('Unmatching structure for Valid datasets - case %#', async (testCase) => { + const grant = await addGrant( + TEST_PRINCIPAL, + Constants.Perm.EditDataset, + testCase.uuid, + false + ); + + await sleep(3600); + + try{ + await test_fplus.DataAccess.update_dataset( + testCase.uuid, + testCase.structure, + testCase.config + ); + }catch(err){ + expect(err.status).toBe(409); + } finally{ + await deleteGrant(grant) + } + }, 30000); + + + test.each([ + { + structure: Constants.App.SparkplugSrc, + config: { + source: Temp_Data.Test_Uuids.Devices.TestDevice + }, + secondLevelPerm: Constants.Perm.UseSparkplug + }, + { + structure: Constants.App.SparkplugSrc, + config: { + source: Temp_Data.Test_Uuids.Devices.TestDoubleDevice + }, + secondLevelPerm: Constants.Perm.UseSparkplug + }, + { + structure: Constants.App.SparkplugSrc, + config: { + source: Temp_Data.Test_Uuids.Devices.TestFloatDevice + }, + secondLevelPerm: Constants.Perm.UseSparkplug + }, + + ])('Update for Valid SRC - case %#', async (testCase) => { + const dataset_uuid = await createDataset(testCase.structure, testCase.config); + + const grant1 = await addGrant(TEST_PRINCIPAL, Constants.Perm.EditDataset, dataset_uuid, false); + + const new_source = Temp_Data.Test_Uuids.Devices.Node2_TestDoubleDevice; + const grant2 = await addGrant(TEST_PRINCIPAL, testCase.secondLevelPerm, new_source, false); + + await sleep(5000); + + await test_fplus.DataAccess.update_dataset( + dataset_uuid, + testCase.structure, + { + source: new_source + } + ); + await sleep(3000); + + const updated_config = await admin_fplus.ConfigDB.get_config( + testCase.structure, + dataset_uuid + ); + + + await deleteGrant(grant1); + await deleteGrant(grant2); + await deleteDataset(dataset_uuid); + + + expect(updated_config.source).toEqual(new_source); + + }, 30000); + + + + test.each([ + { + structure: Constants.App.SessionLimits, + config: { + source: Temp_Data.Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset, + from: "2026-06-01T08:52:00.000Z", + to: "2026-06-01T11:55:00.000Z", + }, + secondLevelPerm: Constants.Perm.UseForSession + }, + { + structure: Constants.App.SessionLimits, + config: { + source: Temp_Data.Test_Uuids.Src_Datasets.Node2_TestFloutDeviceDataset, + from: "2026-07-01T08:52:00.000Z", + to: "2026-08-01T11:55:00.000Z", + }, + secondLevelPerm: Constants.Perm.UseForSession + }, + + ])('Successful update for Valid Session - case %#', async (testCase) => { + + const dataset_uuid = await createDataset(testCase.structure, testCase.config); + const grant1 = await addGrant(TEST_PRINCIPAL, Constants.Perm.EditDataset, dataset_uuid, false); + + const new_config = { + source: Temp_Data.Test_Uuids.Src_Datasets.TestDeviceDataset, + from: "2026-07-01T08:52:00.000Z", + to: "2026-08-01T11:55:00.000Z", + }; + const grant2 = await addGrant(TEST_PRINCIPAL, testCase.secondLevelPerm, new_config.source, false); + + await sleep(4000); + + await test_fplus.DataAccess.update_dataset( + dataset_uuid, + testCase.structure, + new_config + ); + await sleep(3000); + + const updated_config = await admin_fplus.ConfigDB.get_config( + testCase.structure, + dataset_uuid + ); + + const old_source_subclasses = await admin_fplus.ConfigDB.class_direct_subclasses(testCase.config.source); + const new_source_subclasses = await admin_fplus.ConfigDB.class_direct_subclasses(new_config.source); + + await deleteDataset(dataset_uuid); + await deleteGrant(grant1); + await deleteGrant(grant2); + + expect(updated_config.source).toEqual(new_config.source); + expect(old_source_subclasses).not.toContain(dataset_uuid); + expect(new_source_subclasses).toContain(dataset_uuid); + + }, 40000); + + + + + test.each([ + { + structure: Constants.App.UnionComponents, + config: [ + Temp_Data.Test_Uuids.Session_Datasets.SessionNode2DoubleDataset, + Temp_Data.Test_Uuids.Session_Datasets.TestSessionAllDataset + ], + secondLevelPerm: Constants.Perm.IncludeInUnion + }, + { + structure: Constants.App.UnionComponents, + config: [ + Temp_Data.Test_Uuids.Union_Datasets.TestUnionAllDataset, + Temp_Data.Test_Uuids.Src_Datasets.Node2_TestFloutDeviceDataset + ], + secondLevelPerm: Constants.Perm.IncludeInUnion + }, + ])('Successful update for Valid Union', async (testCase) => { + const dataset_uuid = await createDataset(testCase.structure, testCase.config); + const grant1 = await addGrant(TEST_PRINCIPAL, Constants.Perm.EditDataset, dataset_uuid, false); + + const new_source = Temp_Data.Test_Uuids.Union_Datasets.TestNestedUnionDataset; + const grant2 = await addGrant(TEST_PRINCIPAL, testCase.secondLevelPerm, new_source, false); + + await sleep(4000); + + await test_fplus.DataAccess.update_dataset(dataset_uuid, testCase.structure, [new_source]); + + await sleep(2000); + + const updated_config = await admin_fplus.ConfigDB.get_config( + testCase.structure, + dataset_uuid + ); + + await sleep(5000) + + + + + const subclasses = await admin_fplus.ConfigDB.class_direct_subclasses(dataset_uuid); + + + + await deleteDataset(dataset_uuid); + await deleteGrant(grant1); + await deleteGrant(grant2); + + + expect(updated_config).toContain(new_source); + for (let old_src of testCase.config){ + expect(subclasses).not.toContain(old_src); + } + expect(subclasses).toContain(new_source); + }, 40000); + + + + test.each([ + { + structure: Constants.App.SparkplugSrc, + config: { + source: Temp_Data.Test_Uuids.Devices.TestDevice + }, + secondLevelPerm: Constants.Perm.UseSparkplug + }, + { + structure: Constants.App.SparkplugSrc, + config: { + source: Temp_Data.Test_Uuids.Devices.TestDoubleDevice + }, + secondLevelPerm: Constants.Perm.UseSparkplug + }, + { + structure: Constants.App.SparkplugSrc, + config: { + source: Temp_Data.Test_Uuids.Devices.TestFloatDevice + }, + secondLevelPerm: Constants.Perm.UseSparkplug + }, + ])('Successful update for Invalid SRC', async (testCase) => { + // create dataset and give edit grant + const dataset_uuid = await createDataset(testCase.structure, testCase.config); + const grant1 = await addGrant(TEST_PRINCIPAL, Constants.Perm.EditDataset, dataset_uuid, false); + + const new_source = Temp_Data.Test_Uuids.Devices.Node2_TestFloutDevice; + const grant2 = await addGrant(TEST_PRINCIPAL, testCase.secondLevelPerm, new_source, false); + + await sleep(4000); + + + // make it invalid + const invalid_structure = Constants.App.SessionLimits; + await addConfig(invalid_structure, dataset_uuid,{"source": "invalid source"} ) + + await sleep(4000); + + + // update dataset + await test_fplus.DataAccess.update_dataset(dataset_uuid, testCase.structure, {source: new_source}); + await sleep(4000); + + const updated_structure = await test_fplus.DataAccess.get_single_structure(dataset_uuid); + + // delete grants + await deleteGrant(grant1); + await deleteGrant(grant2); + + // delete dataset + await deleteDataset(dataset_uuid); + + expect(updated_structure.structure).toEqual(testCase.structure); + expect(updated_structure.config.source).toEqual(new_source); + }, 30000); + + + + + test.each([ + { + structure: Constants.App.SessionLimits, + config: { + source: Temp_Data.Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset, + from: "2026-06-01T08:52:00.000Z", + to: "2026-06-01T11:55:00.000Z", + }, + secondLevelPerm: Constants.Perm.UseForSession + }, + { + structure: Constants.App.SessionLimits, + config: { + source: Temp_Data.Test_Uuids.Src_Datasets.Node2_TestFloutDeviceDataset, + from: "2026-07-01T08:52:00.000Z", + to: "2026-08-01T11:55:00.000Z", + }, + secondLevelPerm: Constants.Perm.UseForSession + }, + + ])('Successful update for Invalid Session', async (testCase) => { + const dataset_uuid = await createDataset(testCase.structure, testCase.config); + const grant1 = await addGrant(TEST_PRINCIPAL, Constants.Perm.EditDataset, dataset_uuid, false); + + const new_config = { + source: Temp_Data.Test_Uuids.Src_Datasets.TestDeviceDataset, + from: "2026-07-01T08:52:00.000Z", + to: "2026-08-01T11:55:00.000Z", + }; + const grant2 = await addGrant(TEST_PRINCIPAL, testCase.secondLevelPerm, new_config.source, false); + + await sleep(4000); + + + // make it invalid + const invalid_structure = Constants.App.SparkplugSrc; + await addConfig(invalid_structure, dataset_uuid,{"source": "invalid source"} ) + + await sleep(4000); + + await test_fplus.DataAccess.update_dataset( + dataset_uuid, + testCase.structure, + new_config + ); + await sleep(3000); + + + const old_source_subclasses = await admin_fplus.ConfigDB.class_direct_subclasses(testCase.config.source); + const new_source_subclasses = await admin_fplus.ConfigDB.class_direct_subclasses(new_config.source); + + const updated_structure = await test_fplus.DataAccess.get_single_structure(dataset_uuid); + + await deleteDataset(dataset_uuid); + await deleteGrant(grant1); + await deleteGrant(grant2); + + expect(updated_structure.structure).toEqual(testCase.structure); + expect(updated_structure.config.source).toEqual(new_config.source); + expect(old_source_subclasses).toContain(dataset_uuid); + expect(new_source_subclasses).toContain(dataset_uuid); + + }, 40000); + + + + test.each([ + { + structure: Constants.App.UnionComponents, + config: [ + Temp_Data.Test_Uuids.Session_Datasets.SessionNode2DoubleDataset, + Temp_Data.Test_Uuids.Session_Datasets.TestSessionAllDataset + ], + secondLevelPerm: Constants.Perm.IncludeInUnion + }, + { + structure: Constants.App.UnionComponents, + config: [ + Temp_Data.Test_Uuids.Union_Datasets.TestUnionAllDataset, + Temp_Data.Test_Uuids.Src_Datasets.Node2_TestFloutDeviceDataset + ], + secondLevelPerm: Constants.Perm.IncludeInUnion + }, + ])('Successful update for Invalid Union', async (testCase) => { + const dataset_uuid = await createDataset(testCase.structure, testCase.config); + const grant1 = await addGrant(TEST_PRINCIPAL, Constants.Perm.EditDataset, dataset_uuid, false); + + const new_source = Temp_Data.Test_Uuids.Union_Datasets.TestNestedUnionDataset; + const grant2 = await addGrant(TEST_PRINCIPAL, testCase.secondLevelPerm, new_source, false); + + await sleep(4000); + + + // make it invalid + const invalid_structure = Constants.App.SparkplugSrc; + await addConfig(invalid_structure, dataset_uuid,{"source": "invalid source"} ) + + await sleep(4000); + + + + await test_fplus.DataAccess.update_dataset(dataset_uuid, testCase.structure, [new_source]); + + await sleep(4000); + + const updated_structure = await test_fplus.DataAccess.get_single_structure(dataset_uuid); + const subclasses = await admin_fplus.ConfigDB.class_direct_subclasses(dataset_uuid); + + await deleteDataset(dataset_uuid); + await deleteGrant(grant1); + await deleteGrant(grant2); + + expect(updated_structure.structure).toEqual(testCase.structure); + + expect(updated_structure.config).toContain(new_source); + + for (let old_src of testCase.config){ + expect(subclasses).toContain(old_src); + } + expect(subclasses).toContain(new_source); + }, 40000); + +}); \ No newline at end of file diff --git a/acs-data-access/tests/temp_data.js b/acs-data-access/tests/temp_data.js new file mode 100644 index 000000000..365e50fd2 --- /dev/null +++ b/acs-data-access/tests/temp_data.js @@ -0,0 +1,208 @@ +import { config } from "rxjs"; +import { DataAccess as Constants } from "../lib/constants.js"; + + +export const Test_Uuids = { + Devices: { + TestDevice: "5880686f-13f9-4089-b236-825d002bc911", + TestDoubleDevice: "1298e74a-37d3-4717-aba1-4c1e67b953c1", + TestFloatDevice: "34b6910c-85ec-445b-9071-0813891e6ff7", + + Node2_TestDoubleDevice: "d60d00bf-eae7-4f88-9ad2-cfc4c638ac7b", + Node2_TestFloutDevice: "961cb1c0-3d00-44e8-90bc-3fdf917746fc" + }, + + Src_Datasets: { + Node2_TestDoubleDeviceDataset: "720ecd5a-c5d2-49d5-bf5a-8ca01dfdb7df", + Node2_TestFloutDeviceDataset: "9975fb51-ed1c-4bc9-a845-331c801f140f", + TestDeviceDataset: "492194aa-4ced-44d1-96c1-4e09e4d52f45", + TestDoubleDeviceDataset: "3af79d1f-bdc2-41a8-bf2b-bd16af8a92a2", + TestFloatDeviceDataset: "3dc230e9-ab36-4729-8d93-6d5c421f7d7f", + }, + + Union_Datasets: { + TestDoublesUnionDataset: "f0f1da5d-7a9d-431a-afd1-eec467ed21cb", + TestNestedUnionDataset: "21e0dfa8-f044-4e87-8606-f528686205d8", + TestUnionAllDataset: "e2a4c530-dc0f-417d-b00b-329b0e90e033", + }, + + Session_Datasets: { + SessionNode2DoubleDataset: "811da559-727b-4808-8d99-c31f90520966", + TestSessionAllDataset: "74cac8a1-88a6-4b08-b297-a3c05a388ec5", + }, + Mes: { + MesDataset: '586205bf-81c6-4091-9d2c-f3c0465ebdc4', + MesEquipment: '4c93ddc1-e610-4efe-91e3-a355f9ba1a09', + MesOperation: 'bd0354eb-b8f7-4bd9-8407-0588e545603c', + MesProduct: '4a089748-b26b-4f12-8f1a-164bfba97809', + MesWorkorder: 'b416e44c-c57e-4486-9431-64c425f1b2c6', + App: 'af178f0c-3b1e-44f2-9724-5cf06e8fd056' + }, +} + +export const Valid_Body = { + SRC: { + structure: Constants.App.SparkplugSrc, + config: { + source: Test_Uuids.Devices.Node2_TestDoubleDevice + }, + secondLevelPerm: Constants.Perm.UseSparkplug + }, + + Session: { + structure: Constants.App.SessionLimits, + config: { + source: Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset, + from: "2026-06-01T08:52:00.000Z", + to: "2026-06-01T11:55:00.000Z", + }, + secondLevelPerm: Constants.Perm.UseForSession + }, + + Union: { + structure: Constants.App.UnionComponents, + config: [ + Test_Uuids.Session_Datasets.SessionNode2DoubleDataset, + Test_Uuids.Union_Datasets.TestDoublesUnionDataset + ], + secondLevelPerm: Constants.Perm.IncludeInUnion + } +} + + +export const Invalid_Body_Structure = [ + // Invalid_No_Structure + { + config: { + source: Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset + } + }, + + // Invalid_Bad_Structure_Uuid + { + structure: 'xxx', + config: { + source: Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset + } + }, + + // Invalid_Unhandled_Structure_Type + { + structure: '74cac8a1-88a6-4b08-b297-a3c05a388ec5', + config: { + source: Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset + } + } +] + +export const Invalid_Body_Config = { + SRC:[ + // no config + { + structure: Constants.App.SparkplugSrc + }, + + // Invalid_No_Source + { + structure: Constants.App.SparkplugSrc, + config: {} + }, + + // Invalid_Bad_Source_Uuid for SRC + { + structure: Constants.App.SparkplugSrc, + config: { + source: 'xxx' + } + }, + + // Invalid_No_Config for SRC + { + structure: Constants.App.SparkplugSrc, + undefined + }, + + // Invalid_Config_Format for SRC + { + structure: Constants.App.SparkplugSrc, + config: [] + }, + + // Invalid_Bad_Child_Uuid for SRC + { + structure: Constants.App.SparkplugSrc, + config: { + source: "xxx" + } + } + ], + Session:[ + // Invalid_No_Source: + { + structure: Constants.App.SessionLimits, + config: { + from: "2026-06-01T08:52:00.000Z", + to: "2026-06-01T11:55:00.000Z", + } + }, + + // Invalid_No_From: + { + structure: Constants.App.SessionLimits, + config: { + source: Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset, + to: "2026-06-01T11:55:00.000Z", + } + }, + + // Invalid_No_To: + { + structure: Constants.App.SessionLimits, + config: { + source: Test_Uuids.Src_Datasets.Node2_TestDoubleDeviceDataset, + from: "2026-06-01T08:52:00.000Z", + } + }, + + // Invalid config format + { + structure: Constants.App.SessionLimits, + config: [] + }, + + // Empty config + { + structure: Constants.App.SessionLimits, + config: {} + }, + + // No config + { + structure: Constants.App.SessionLimits + }, + ], + Union: [ + // no config + { + structure: Constants.App.UnionComponents, + }, + + // config wrong format + { + structure: Constants.App.UnionComponents, + config: {} + }, + + // invalid source uuid + { + structure: Constants.App.UnionComponents, + config: [ + Test_Uuids.Session_Datasets.SessionNode2DoubleDataset, + "xxx", + Test_Uuids.Session_Datasets.TestSessionAllDataset, + ] + }, + ], +} + + diff --git a/acs-data-access/tests/tst-http-api.js b/acs-data-access/tests/tst-http-api.js new file mode 100644 index 000000000..893236ced --- /dev/null +++ b/acs-data-access/tests/tst-http-api.js @@ -0,0 +1,40 @@ + + +import {ServiceClient} from "@amrc-factoryplus/service-client"; +import process from "node:process"; + + +/** + * This is a test script that uses rx-client DataAccess client interface + * to subscribe to DataAccess Notify API. + */ + + + // .get_metadata_list(); + // .get_single_metadata(DATASET_UUID) +async function main(){ + const fplus = new ServiceClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.TEST_HUMAN_USERNAME, + password: process.env.TEST_HUMAN_PASSWORD, + verbose: 'ALL' + }); + + + const uuid = "a7594958-4b03-45e4-8ac0-4af8d1e77e3f"; + const res = await fplus.DataAccess.delete_dataset(uuid); + + + + + // const res = await fplus.DataAccess.download_data(DATASET_UUID) + + // const res = await fplus.DataAccess.get_metadata_list(); + + console.log("Test Data Access HTTP"); + console.log(res); + console.log(""); +} + + +main().catch(console.error); \ No newline at end of file diff --git a/acs-data-access/tests/tst-influx-directly.js b/acs-data-access/tests/tst-influx-directly.js new file mode 100644 index 000000000..c8320a86f --- /dev/null +++ b/acs-data-access/tests/tst-influx-directly.js @@ -0,0 +1,212 @@ +import {InfluxDB, flux} from '@influxdata/influxdb-client'; +import { count } from 'rxjs'; + +const url = process.env.INFLUXDB_URL; +const token = process.env.INFLUXDB_TOKEN; +const org = process.env.INFLUXDB_ORG; +const bucket = process.env.INFLUXDB_BUCKET; + +const influxDB = new InfluxDB({ url, token }); +const queryApi = influxDB.getQueryApi(org); + +/** + * Get all data for a specific topLevelInstance tag. + */ +async function getAllDataForTag(topLevelInstance) { + const flux = ` + from(bucket: "${bucket}") + |> range(start: 0) + |> filter(fn: (r) => r["topLevelInstance"] == "${topLevelInstance}") + `; + + const rows = []; + + return new Promise((resolve, reject) => { + queryApi.queryRows(flux, { + next(row, tableMeta) { + rows.push(tableMeta.toObject(row)); + }, + error(error) { + reject(error); + }, + complete() { + resolve(rows); + } + }); + }); +} + +/** + * Get first and last timestamps for a specific topLevelInstance tag. + */ +async function getTimestampRange(topLevelInstance) { + const firstQuery = ` + from(bucket: "${bucket}") + |> range(start: 0) + |> filter(fn: (r) => r["topLevelInstance"] == "${topLevelInstance}") + |> sort(columns: ["_time"]) + |> limit(n: 1) + `; + + const lastQuery = ` + from(bucket: "${bucket}") + |> range(start: 0) + |> filter(fn: (r) => r["topLevelInstance"] == "${topLevelInstance}") + |> sort(columns: ["_time"], desc: true) + |> limit(n: 1) + `; + + async function runSingleRowQuery(flux) { + return new Promise((resolve, reject) => { + let result = null; + + queryApi.queryRows(flux, { + next(row, tableMeta) { + result = tableMeta.toObject(row); + }, + error(error) { + reject(error); + }, + complete() { + resolve(result); + } + }); + }); + } + + const [first, last] = await Promise.all([ + runSingleRowQuery(firstQuery), + runSingleRowQuery(lastQuery) + ]); + + return { + firstTimestamp: first?._time || null, + lastTimestamp: last?._time || null + }; +} + + +async function countRowsForTag(topLevelInstance) { + const flux = ` + from(bucket: "${bucket}") + |> range(start: 0) + |> filter(fn: (r) => r["topLevelInstance"] == "${topLevelInstance}") + |> group() + |> count(column: "_value") + |> sum(column: "_value") + `; + + return new Promise((resolve, reject) => { + let total = 0; + + queryApi.queryRows(flux, { + next(row, tableMeta) { + const obj = tableMeta.toObject(row); + total = obj._value || 0; + }, + error: reject, + complete: () => resolve(total), + }); + }); +} + + +async function getAllTagsAndFieldsForTag(topLevelInstance) { + const flux = ` + from(bucket: "${bucket}") + |> range(start: 0) + |> filter(fn: (r) => r["topLevelInstance"] == "${topLevelInstance}") + |> limit(n: 1000) + `; + + return new Promise((resolve, reject) => { + const tags = new Set(); + const fields = new Set(); + + queryApi.queryRows(flux, { + next(row, tableMeta) { + const obj = tableMeta.toObject(row); + + // Fields + if (obj._field) { + fields.add(obj._field); + } + + // Tags = all string keys except reserved Influx columns + for (const key of Object.keys(obj)) { + if ( + !["_time", "_value", "_field", "_measurement"].includes(key) + ) { + tags.add(key); + } + } + }, + error(err) { + reject(err); + }, + complete() { + resolve({ + tags: [...tags], + fields: [...fields], + }); + }, + }); + }); +} + +/** + * Example usage + */ + +const SESSION_DATASETS = { + "TestSessionAllDataset": "74cac8a1-88a6-4b08-b297-a3c05a388ec5" +} + +const UNION_DATASETS = { + "TestUnionAllDataset": "e2a4c530-dc0f-417d-b00b-329b0e90e033" +} + +const SRC_DATASETS = { + "TestDeviceDataset": "492194aa-4ced-44d1-96c1-4e09e4d52f45", + "TestDoubleDeviceDataset": "3af79d1f-bdc2-41a8-bf2b-bd16af8a92a2", + "TestFloatDeviceDataset": "3dc230e9-ab36-4729-8d93-6d5c421f7d7f", + + "Node2_TestDoubleDeviceDataset": "720ecd5a-c5d2-49d5-bf5a-8ca01dfdb7df", + + "Node2_TestFloutDeviceDataset": "9975fb51-ed1c-4bc9-a845-331c801f140f", +} + +// NODE 1 +const DEVICES_MAP = { + "TestDevice": "5880686f-13f9-4089-b236-825d002bc911", + "TestDoubleDevice": "1298e74a-37d3-4717-aba1-4c1e67b953c1", + "TestFloatDevice": "34b6910c-85ec-445b-9071-0813891e6ff7", + + "Node2_TestDoubleDevice": "d60d00bf-eae7-4f88-9ad2-cfc4c638ac7b", + "Node2_TestFloutDevice": "961cb1c0-3d00-44e8-90bc-3fdf917746fc" +} + +async function main() { + // device uuid + const topLevelInstance = DEVICES_MAP.TestDoubleDevice; + + console.log(`Fetching data for ${topLevelInstance}...`); + + // const data = await getAllDataForTag(topLevelInstance); + + // console.log(`Found ${data.length} rows`); + + // Show first few rows + // console.dir(data.slice(0, 5), { depth: null }); + + const rows_count = await countRowsForTag(topLevelInstance); + console.log("Rows count:", rows_count); + + const time_range = await getTimestampRange(topLevelInstance); + console.log('Timestamp range:', time_range); + + const fields = await getAllTagsAndFieldsForTag(topLevelInstance); + console.log("Fields:", fields); +} + +main().catch(console.error); \ No newline at end of file diff --git a/acs-data-access/tests/tst-notify-api.js b/acs-data-access/tests/tst-notify-api.js new file mode 100644 index 000000000..6387909a4 --- /dev/null +++ b/acs-data-access/tests/tst-notify-api.js @@ -0,0 +1,71 @@ + + +import {RxClient} from "@amrc-factoryplus/rx-client"; +import * as rxu from "@amrc-factoryplus/rx-util"; +import * as rx from "rxjs"; +import readline from "node:readline"; +import process from "node:process"; + + +/** + * This is a test script that uses rx-client DataAccess client interface + * to subscribe to DataAccess Notify API. + */ +async function main(){ + const fplus = new RxClient({ + directory_url: process.env.DIRECTORY_URL, + username: process.env.HUMAN_USERNAME, + password: process.env.HUMAN_PASSWORD, + verbose: 'ALL' + }); + + const DATASET_UUID = ""; + + const subscription = fplus.DataAccess + // .watch_metadata_list() + // .watch_single_metadata(DATASET_UUID) + // .search_metadata() + // .watch_structure_list() + // .watch_single_structure(DATASET_UUID) + .search_structure() + .subscribe({ + next: value => { + console.log("TEST: UPDATE from DataAccess"); + console.log(value?.toJS()); + console.log(""); + }, + + error: err => { + console.error("ERROR"); + console.error(err); + }, + + complete: () => { + console.log("COMPLETE"); + }, + }); + + // ---------------------------------------- + // Keep process alive + quit on q + // ---------------------------------------- + readline.emitKeypressEvents(process.stdin); + + if(process.stdin.isTTY) + process.stdin.setRawMode(true); + + console.log("Press q to quit"); + + process.stdin.on("keypress", (_, key) => { + if(key.name === "q"){ + console.log("\nQuitting"); + subscription.unsubscribe(); + process.exit(0); + } + }); +} + + +main().catch(err => { + console.error(err); + process.exit(1); +}) \ No newline at end of file diff --git a/deploy/templates/auth/principals/services.yaml b/deploy/templates/auth/principals/services.yaml index d26914260..0838385b1 100644 --- a/deploy/templates/auth/principals/services.yaml +++ b/deploy/templates/auth/principals/services.yaml @@ -153,4 +153,4 @@ spec: additionalPrincipals: - HTTP/i3x.{{.Values.acs.baseUrl | required "values.acs.baseUrl is required"}}@{{ .Values.identity.realm | required "values.identity.realm is required!" }} secret: i3x-keytabs/server -{{- end }} +{{- end }} \ No newline at end of file diff --git a/deploy/templates/data-access/data-access.yaml b/deploy/templates/data-access/data-access.yaml new file mode 100644 index 000000000..d33ddfba3 --- /dev/null +++ b/deploy/templates/data-access/data-access.yaml @@ -0,0 +1,86 @@ +{{- if .Values.dataAccess.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: data-access + namespace: {{ .Release.Namespace }} + labels: + component: data-access +spec: + replicas: 1 + selector: + matchLabels: + component: data-access + template: + metadata: + labels: + component: data-access + factory-plus.service: data-access + spec: + {{- with .Values.acs.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: krb5-conf + configMap: + name: krb5-conf + - name: krb5-keytabs + secret: + secretName: data-access-keytabs + + containers: + - name: data-access +{{ include "amrc-connectivity-stack.image" (list . .Values.dataAccess) | indent 10 }} + command: [ "/usr/bin/k5start", "-Uf", "/keytabs/client" ] + args: [ "node", "bin/api.js" ] + env: + - name: KRB5_CONFIG + value: /config/krb5-conf/krb5.conf + - name: CLIENT_KEYTAB + value: /keytabs/client + - name: SERVER_KEYTAB + value: /keytabs/server + - name: HOSTNAME + value: data-access.{{ .Release.Namespace }}.svc.cluster.local + - name: REALM + value: {{ .Values.identity.realm | required "values.identity.realm is required!" }} + - name: ROOT_PRINCIPAL + value: admin@{{ .Values.identity.realm | required "values.identity.realm is required!" }} + - name: VERBOSE + value: {{.Values.dataAccess.verbosity | quote | required "values.dataAccess.verbosity is required!"}} + - name: PORT + value: "8080" +{{ include "amrc-connectivity-stack.cache-max-age" (list . "dataAccess") | indent 12 }} + - name: DIRECTORY_URL + value: http://directory.{{ .Release.Namespace }}.svc.cluster.local + - name: INFLUXDB_URL + value: http://acs-influxdb2.{{ .Release.Namespace }}.svc.cluster.local + - name: INFLUXDB_ORG + value: default + - name: INFLUXDB_BUCKET + value: default + - name: INFLUXDB_TOKEN + valueFrom: + secretKeyRef: + name: influxdb-auth + key: admin-token + volumeMounts: + - mountPath: /config/krb5-conf + name: krb5-conf + - mountPath: /keytabs + name: krb5-keytabs +--- +apiVersion: v1 +kind: Service +metadata: + name: data-access + namespace: {{ .Release.Namespace }} +spec: + ports: + - name: "web" + port: 80 + targetPort: 8080 + selector: + factory-plus.service: data-access +{{- end -}} diff --git a/deploy/templates/data-access/ingress.yaml b/deploy/templates/data-access/ingress.yaml new file mode 100644 index 000000000..f381287cb --- /dev/null +++ b/deploy/templates/data-access/ingress.yaml @@ -0,0 +1,23 @@ +{{ if .Values.dataAccess.enabled }} +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: data-access-ingressroute + namespace: {{ .Release.Namespace }} +spec: + entryPoints: + - {{ .Values.acs.secure | ternary "websecure" "web" }} + routes: + - match: Host(`data-access.{{.Values.acs.baseUrl | required "values.acs.baseUrl is required"}}`) + kind: Rule + services: + - name: data-access + port: 80 + namespace: {{ .Release.Namespace }} + {{- if .Values.acs.secure }} + tls: + secretName: {{ coalesce .Values.dataAccess.tlsSecretName .Values.acs.tlsSecretName }} + domains: + - main: data-access.{{.Values.acs.baseUrl | required "values.acs.baseUrl is required"}} + {{- end -}} +{{- end -}} diff --git a/lib/js-rx-client/lib/auth.js b/lib/js-rx-client/lib/auth.js index d74b8a5a6..843b69270 100644 --- a/lib/js-rx-client/lib/auth.js +++ b/lib/js-rx-client/lib/auth.js @@ -35,4 +35,30 @@ export class Auth extends Interfaces.Auth { const res = await rx.firstValueFrom(this._watch_acl(path)); return [res.status, res.body]; } + + /** + * We have not handled bootstrap acls + * @param {*} principal + * @returns + */ + watch_acl (principal) { + const url = urljoin("v2", "acl", "kerberos", principal); + return rxx.rx( + this._watch_acl(url), + + rx.filter(r => r.status == 200), + rx.map(r => r.body) + ); + } + + watch_acl_with_perm (principal, permission) { + return rxx.rx( + this.watch_acl(principal), + rx.map(arr => + arr.filter(x => x.permission == permission) + .map(x => x.target) + ), + rx.map(arr => new Set(arr)) + ); + } } diff --git a/lib/js-rx-client/lib/data-access.js b/lib/js-rx-client/lib/data-access.js new file mode 100644 index 000000000..fcadd573e --- /dev/null +++ b/lib/js-rx-client/lib/data-access.js @@ -0,0 +1,55 @@ +/* + * Factory+ Rx interface + * Data Access notify + * Copyright 2026 University of Sheffield + */ + +import * as imm from "immutable"; +import * as rx from "rxjs"; + +import { Interfaces, urljoin } from "@amrc-factoryplus/service-client"; +import * as rxx from "@amrc-factoryplus/rx-util"; + +import { NotifyV2 } from "./notify-v2.js"; + +/** Extended DataAccess interface class. + * This supports all the methods from the base DataAccess service + * interface as well as methods to access the notify API. + */ +export class DataAccess extends Interfaces.DataAccess { + constructor(fplus){ + super(fplus); + + /** A NotifyV2 object for the DataAccess service. + * @type NotifyV2 */ + this.notify = new NotifyV2(this); + } + + /** + * Watch Dataset uuids with READ_DATASET permission + * @returns + */ + watch_metadata_list(){ + return this.notify.watch(`v2/metadata/`); + } + + watch_single_metadata(uuid){ + return this.notify.watch(`v2/metadata/${uuid}`); + } + + search_metadata(filter){ + return this.notify.search(`v2/metadata/`, filter); + } + + watch_structure_list(){ + return this.notify.watch('v2/structure/'); + } + + watch_single_structure(uuid){ + return this.notify.watch(`v2/structure/${uuid}`); + } + + search_structure(filter){ + return this.notify.search('v2/structure/', filter); + } +} \ No newline at end of file diff --git a/lib/js-rx-client/lib/interfaces.js b/lib/js-rx-client/lib/interfaces.js index 002856c00..c38c4dc80 100644 --- a/lib/js-rx-client/lib/interfaces.js +++ b/lib/js-rx-client/lib/interfaces.js @@ -9,4 +9,4 @@ export { Auth } from "./auth.js"; export { ConfigDB } from "./configdb.js"; - +export { DataAccess } from './data-access.js'; diff --git a/lib/js-rx-client/lib/rx-client.js b/lib/js-rx-client/lib/rx-client.js index 2282cf16d..34a189648 100644 --- a/lib/js-rx-client/lib/rx-client.js +++ b/lib/js-rx-client/lib/rx-client.js @@ -20,6 +20,7 @@ export class RxClient extends ServiceClient { RxClient.define_interfaces( ["ConfigDB", SI.ConfigDB, ``], ["Auth", SI.Auth, ``], + ["DataAccess", SI.DataAccess, ``], ); /** diff --git a/lib/js-service-client/lib/interfaces.js b/lib/js-service-client/lib/interfaces.js index 0b32ae233..c5f13d3d9 100644 --- a/lib/js-service-client/lib/interfaces.js +++ b/lib/js-service-client/lib/interfaces.js @@ -13,3 +13,4 @@ export { Fetch } from "./service/fetch.js"; export { Git } from "./service/git.js"; export { MQTTInterface } from "./service/mqtt.js"; export { Files } from "./service/files.js"; +export {DataAccess} from "./service/data-access.js"; diff --git a/lib/js-service-client/lib/service-client.js b/lib/js-service-client/lib/service-client.js index e0efacadc..3dcdf8f0b 100644 --- a/lib/js-service-client/lib/service-client.js +++ b/lib/js-service-client/lib/service-client.js @@ -142,4 +142,5 @@ ServiceClient.define_interfaces( ["Git", SI.Git, ``], ["MQTT", SI.MQTTInterface, `mqtt_client`], ["Files", SI.Files, ``], + ["DataAccess", SI.DataAccess, ``] ); diff --git a/lib/js-service-client/lib/service/data-access.js b/lib/js-service-client/lib/service/data-access.js new file mode 100644 index 000000000..89545eef0 --- /dev/null +++ b/lib/js-service-client/lib/service/data-access.js @@ -0,0 +1,221 @@ +/* + * Factory+ NodeJS Utilities + * Data Access service interface. + */ + +import { Service } from "../uuids.js"; +import { ServiceInterface } from "./service-interface.js"; + +let nodeDepsPromise; + +async function loadNodeDeps() { + if (!nodeDepsPromise) { + nodeDepsPromise = (async () => { + const fs = await import("node:fs"); + const path = await import("node:path"); + const { pipeline } = await import("node:stream/promises"); + const { Readable } = await import("node:stream"); + + return { + fs: fs.default, + path: path.default, + pipeline, + Readable + }; + })(); + } + + return nodeDepsPromise; +} + + +/** + * Interface to the Data Access service + */ + +export class DataAccess extends ServiceInterface{ + constructor(fplus){ + super(fplus); + this.service = Service.DataAccess; + this.log = fplus.debug.bound("data-access"); + } + + async delete_dataset(uuid){ + const [st, json] = await this.fetch(`v1/delete/${uuid}`); + if(st == 404) return ""; + if(st != 200) + this.throw (`Can't delete dataset ${uuid}`, st, json); + + return json; + } + + /** + * Returns list of dataset uuids with READ_DATASET permission. + */ + async get_metadata_list(){ + const [st, json] = await this.fetch('v1/metadata'); + if(st == 404) return []; + if(st == 403) + this.throw(`Unauthorised get dataset uuids`, st); + if(st != 200) + this.throw(`Can't get dataset uuids`, st); + return json; + } + + + /** + * Returns json object with dataset metadata + */ + async get_single_metadata(uuid){ + const [st, json] = await this.fetch(`v1/metadata/${uuid}`); + if(st == 404) return {}; + if(st == 403) + this.throw(`Unauthorised to access dataset ${uuid}`, st); + if(st != 200) + this.throw(`Can't get metadata for dataset ${uuid}`, st); + return json; + } + + /** + * Download dataset CSV stream + * Returns a readable stream + */ + /** + * Returns a stream of dataset CSV data. + */ + async download_data(uuid, measurement=undefined, output_dir = ".") { + if(this.fplus.opts.browser){ + this.throw(`Method not supported in browser.`); + } + // Dynamically import the node libs if running in node + const { fs, path, pipeline, Readable } = await loadNodeDeps(); + + const body = {}; + + if (measurement !== undefined) { + body.measurement = measurement; + } + + const [st, stream, _, headers] = await this.fetch({ + url: `v1/data/${uuid}`, + method: "POST", + accept: "application/zip", + response_type: "stream", + body + }); + + if (st === 404) return; + + if (st === 403) { + this.throw( + `Unauthorised to access dataset ${uuid}`, + st + ); + } + + if (st !== 200) { + this.throw( + `Can't download data for dataset ${uuid}`, + st + ); + } + + const disposition = + headers.get("Content-Disposition") ?? ""; + + const filename = + /filename\*?=(?:UTF-8'')?("?)([^";]+)\1/i.exec(disposition)?.[2] + ?? `${uuid}.zip`; + + const safeFilename = filename.replace(/[\/\\]/g, "_"); + + fs.mkdirSync(output_dir, { recursive: true }); + + const filepath = path.join(output_dir, safeFilename); + + const nodeStream = + stream instanceof Readable + ? stream + : Readable.fromWeb(stream); + + nodeStream.on("error", err => { + this.throw(`Download stream error: ${err.message}`); + }); + + await pipeline( + nodeStream, + fs.createWriteStream(filepath) + ); + + return filepath; + } + + + async get_structure_list(){ + const [st, json] = await this.fetch('v1/structure'); + if(st == 404) return []; + if(st == 403) + this.throw(`Unauthorised to get dataset uuids`, st); + if(st != 200) + this.throw(`Can't get dataset uuids`, st); + return json; + } + + /** + * + * @param {*} structure is one of the Dataset Definition app uuids src, session or union. + * @param {*} config is the config entry content of dataset + */ + async create_dataset(structure, config){ + const [st, json] = await this.fetch({ + url:'v1/structure', + method: 'POST', + body: { + structure, + config + } + }); + + if(st == 404) return ""; + if(st == 403) + this.throw(`Unauthorised to create a dataset`, st); + if(st != 200) + this.throw(`Can't create a dataset`, st); + return json; + } + + + + /** + * Returns dataset definition (content of the config entry) + */ + + async get_single_structure(uuid){ + const [st, json] = await this.fetch(`v1/structure/${uuid}`); + if(st == 404) return {}; + if(st == 403) + this.throw(`Unauthorised to access dataset ${uuid}`, st); + if(st != 200) + this.throw(`Can't get dataset definition ${uuid}`, st); + return json; + } + + + async update_dataset(uuid, structure, config){ + const [st, json] = await this.fetch({ + url: `v1/structure/${uuid}`, + method: 'PUT', + body: { + structure, + config + } + }); + + if(st == 404) return ""; + if(st == 403) + this.throw(`Unauthorised to update dataset ${uuid}`, st); + if(st != 200) + this.throw(`Can't update dataset definition ${uuid}`, st); + return json; + } +} \ No newline at end of file diff --git a/lib/js-service-client/lib/service/discovery.js b/lib/js-service-client/lib/service/discovery.js index 2179f9dd0..b9fc24f06 100644 --- a/lib/js-service-client/lib/service/discovery.js +++ b/lib/js-service-client/lib/service/discovery.js @@ -35,6 +35,7 @@ export class Discovery extends ServiceInterface { [opts.directory_url, Service.Directory], [opts.mqtt_url, Service.MQTT], [opts.files_url, Service.Files], + [opts.dataAccess_url, Service.DataAccess] ]; for (const [pr, srv] of presets) { if (pr == null) continue;