diff --git a/front_end/openVRE/apache/check_tool_access.lua b/front_end/openVRE/apache/check_tool_access.lua index 25e34547..7c704797 100644 --- a/front_end/openVRE/apache/check_tool_access.lua +++ b/front_end/openVRE/apache/check_tool_access.lua @@ -6,14 +6,14 @@ function check_access(r) local tool_segment = r.uri:match("^/interactive%-tool/([^/]+)") if not tool_segment then r:err("check_tool_access: no tool_segment found, passing through") - return 200 + return apache2.OK end r:err("check_tool_access: tool_segment=" .. tool_segment) local project_id = tool_segment:match("__PROJ.+$") if not project_id then r:err("check_tool_access: could not parse project_id from: " .. tool_segment) - return 403 + return apache2.HTTP_FORBIDDEN end r:err("check_tool_access: project_id=" .. project_id) @@ -30,9 +30,9 @@ function check_access(r) r:err("check_tool_access: http status=" .. tostring(status)) if status == "200" then - return 200 + return apache2.OK else r:err("check_tool_access: denied user=" .. user .. " project=" .. project_id) - return 403 + return apache2.HTTP_FORBIDDEN end end \ No newline at end of file diff --git a/front_end/openVRE/apache/server.conf b/front_end/openVRE/apache/server.conf index 016146d9..da9dc2b9 100644 --- a/front_end/openVRE/apache/server.conf +++ b/front_end/openVRE/apache/server.conf @@ -39,6 +39,8 @@ Require valid-user + RewriteEngine on + RewriteRule ^/interactive-tool/([^/]+)/?(.*) http://$1:8787/$2 [P,L] ErrorLog /var/log/apache/vre-error.log diff --git a/front_end/openVRE/config/globals.inc.php.sample b/front_end/openVRE/config/globals.inc.php.sample index c108f941..ee10069e 100644 --- a/front_end/openVRE/config/globals.inc.php.sample +++ b/front_end/openVRE/config/globals.inc.php.sample @@ -43,6 +43,7 @@ $GLOBALS['logs_datetime'] = "dd.M.Y HH:mm:ss"; // Interactive tools $GLOBALS['NETWORK_IP'] = "172.17.0.1"; # default host ip for docker +$GLOBALS['NETWORK_NAME'] = getenv('NETWORK_NAME'); $GLOBALS['interactive_range_start_port'] = 9001; $GLOBALS['max_parallel_independent_tools'] = 10; $GLOBALS['toolsPath'] = "/tools/"; diff --git a/front_end/openVRE/public/assets/pages/scripts/actions-home.js b/front_end/openVRE/public/assets/pages/scripts/actions-home.js index a00fdd8f..bc68f117 100644 --- a/front_end/openVRE/public/assets/pages/scripts/actions-home.js +++ b/front_end/openVRE/public/assets/pages/scripts/actions-home.js @@ -698,7 +698,7 @@ function closeModalTool() { } -function goInteractiveTool(toolContainerName) { - location.href = "interactive-tool/" + toolContainerName + "/"; +function goInteractiveTool(id) { + window.open('launch-interactive/?pid=' + id, "_blank"); } diff --git a/front_end/openVRE/public/getdata/uploadForm.php b/front_end/openVRE/public/getdata/uploadForm.php index 2f09bfa2..140f4731 100644 --- a/front_end/openVRE/public/getdata/uploadForm.php +++ b/front_end/openVRE/public/getdata/uploadForm.php @@ -2,6 +2,7 @@ require __DIR__ . "/../../config/bootstrap.php"; redirectOutside(); +$includeEgaFiles = false; ?> @@ -51,7 +52,8 @@ them over the area below. You can also create a text file from a sequence or load a file - to your workspace from an external URL.

+ to your workspace from an external URL. +

@@ -79,9 +81,9 @@
  • Load file from an external URL
    • -
  • +
    @@ -127,7 +129,7 @@ if (isset($_SESSION['errorData'])) { if (isset($_SESSION['errorData']['Info'])) { - ?> + ?>
    -
    +
    - + -
    -
    - Error downloading file, please, try again. -
    -
    - - -
    - - - -
    -

    For password-protected sites, include the credentials in the URL. Format: [protocol]://[username]:[password]@server/path/to/file.
    Notice that strange characters should be UTF-8 codified.

    + +
    + Error downloading file, please, try again.
    - +
    + + +
    + + + +
    +

    For password-protected sites, include the credentials in the URL. Format: [protocol]://[username]:[password]@server/path/to/file.
    Notice that strange characters should be UTF-8 codified.

    +
    + -
    -
    - 20% Complete +
    +
    + 20% Complete +
    +
    +
    + +
    +

    Load data from EGA

    + getMessage(); + } - -
    -
    -

    Load data from EGA

    - getMessage(); - } - - try { - require 'egaFiles.php'; - } catch (Exception $e) { - $_SESSION['errorData']['Error']['EGA'] = 'Failed to fetch EGA files: ' . $e->getMessage(); - } - - ?> - - -
    - -
    - - $txts) { - print "$subTitle
    "; - foreach ($txts as $txt) { - print "
    $txt
    "; + try { + require 'egaFiles.php'; + } catch (Exception $e) { + $_SESSION['errorData']['Error']['EGA'] = 'Failed to fetch EGA files: ' . $e->getMessage(); } - } - unset($_SESSION['errorData']); - ?> -
    - + ?> + + +
    + +
    + + $txts) { + print "$subTitle
    "; + foreach ($txts as $txt) { + print "
    $txt
    "; + } + } + unset($_SESSION['errorData']); + ?> +
    + + +
    +
    +
    +
    -
    + - - - - + - + ?> \ No newline at end of file diff --git a/front_end/openVRE/public/help/general.php b/front_end/openVRE/public/help/general.php index 7a5fb6ba..10ca1c54 100644 --- a/front_end/openVRE/public/help/general.php +++ b/front_end/openVRE/public/help/general.php @@ -36,38 +36,24 @@ -

    What is the euCanSHare Virtual Research Envirnoment (VRE)?

    +

    What is the Virtual Research Envirnoment (VRE)?

    -

    - The Disc4All VRE is a computational platform designed for Data Sharing, Intervertevral Disc Degeneration Analysis, Machine Learning and Bioinformatics Data Analysis. +

    + The VRE is a computational platform designed for Data Sharing, Intervertevral Disc Degeneration Analysis, Machine Learning and Bioinformatics Data Analysis.

    - -

    - -

    - The Disc4All VRE is being developed so as to cater for the needs of the scientific community, especially clinical researchers who need an objective and quantitative way to analyse their data in an unbiased way and make predictions with state-of-the art techniques. Its main objectives are: -

    -

    -

    -

    - The main components of the Disc4All VRE are: -

    - -

    + The VRE is being developed so as to cater for the needs of the scientific community, especially clinical researchers who need an objective and quantitative way to analyse their data in an unbiased way and make predictions with state-of-the art techniques. Its main objectives are: +

    +

    +

    @@ -78,4 +64,4 @@ require "../htmlib/footer.inc.php"; require "../htmlib/js.inc.php"; - ?> + ?> \ No newline at end of file diff --git a/front_end/openVRE/public/help/tools.php b/front_end/openVRE/public/help/tools.php index c9f1330f..f5b0b702 100644 --- a/front_end/openVRE/public/help/tools.php +++ b/front_end/openVRE/public/help/tools.php @@ -1,6 +1,6 @@ -
    - - - - - -
    - -
    - - -
    -
      -
    • - Home - -
      • -
    • - Help - -
      • -
    • - Tools -
      • -
    +
    + + + + + +
    + +
    + + +
    +
      +
    • + Home + +
      • +
    • + Help + +
      • +
    • + Tools +
      • +
    +
    + + +

    Select a tool +

    + + + +

    +

    +
    + + + +
    +
    +
    All
    + + +
    + +
    - - -

    Select a tool -

    - - - -

    -

    -
    - - - -
    -
    -
    All
    - - -
    - - -
    -
    - - ##pending##  + ##pending##  ##execution## ##mtime## @@ -34,7 +34,7 @@ diff --git a/front_end/openVRE/public/htmlib/footer.inc.php b/front_end/openVRE/public/htmlib/footer.inc.php index 24a53675..0f23d1ad 100644 --- a/front_end/openVRE/public/htmlib/footer.inc.php +++ b/front_end/openVRE/public/htmlib/footer.inc.php @@ -85,7 +85,7 @@ Our Cookie Policy
    - Understand + Understand
    diff --git a/front_end/openVRE/public/htmlib/js.inc.php b/front_end/openVRE/public/htmlib/js.inc.php index 06034cde..b2c72174 100644 --- a/front_end/openVRE/public/htmlib/js.inc.php +++ b/front_end/openVRE/public/htmlib/js.inc.php @@ -146,7 +146,7 @@ - + @@ -163,15 +163,11 @@ - + - - - - + + +
    + + + + +
    +
    +
    + +
    +
    +
    +
    + +
    + +
    + +
    +
    + + + + + + + + + + + + + + + +
    +
    + +
    + +

    + + + + " + target="_blank"> + + + Open Interactive Session + + + + + + +
    + +
    +
    +
    +
    + + + This page refreshes automatically every 3 seconds. + + + + + + + + + + +
    + +
    + +
    + +
    + + + +
    + +
    + +

    Launcher logs

    + + "> + Job Standard Output + + +
    + + + +
    + +
    + + + +
    +
    + +
    + + \ No newline at end of file diff --git a/front_end/openVRE/public/phplib/classes/LoggerFactory.php b/front_end/openVRE/public/phplib/classes/LoggerFactory.php index 3ed215d1..7a62b25e 100644 --- a/front_end/openVRE/public/phplib/classes/LoggerFactory.php +++ b/front_end/openVRE/public/phplib/classes/LoggerFactory.php @@ -2,16 +2,21 @@ namespace OpenVRE; -use Monolog\Logger; +require_once __DIR__ . "/../db.inc.php"; + +use Monolog\Handler\MongoDBHandler; use Monolog\Handler\StreamHandler; use Monolog\Formatter\LineFormatter; use Monolog\Level; +use Monolog\Logger; +use Monolog\Processor\UidProcessor; +use Monolog\Processor\PsrLogMessageProcessor; class LoggerFactory { private static array $loggers = []; - public static function getLogger(string $channel = 'app'): Logger + public static function getLogger(string $channel = 'app-stdout'): Logger { if (isset(self::$loggers[$channel])) { return self::$loggers[$channel]; @@ -34,4 +39,26 @@ public static function getLogger(string $channel = 'app'): Logger return $logger; } + + + public static function getPersistentLogger(): Logger + { + if (isset(self::$loggers['app-mongodb'])) { + return self::$loggers['app-mongodb']; + } + + $logger = new Logger('app-mongodb'); + $logsCollection = 'action_logs'; + $mongodb = new MongoDBHandler($GLOBALS['mongodbClient'], getenv('MONGO_MAIN_DB'), $logsCollection, level::Info); + $logger->pushHandler($mongodb); + $logger->pushProcessor(new UidProcessor()); + $logger->pushProcessor(new PsrLogMessageProcessor()); + $logger->pushProcessor(function ($record) { + $record['extra']['userId'] = $_SESSION['User']['id']; + return $record; + }); + self::$loggers['app-mongodb'] = $logger; + + return $logger; + } } diff --git a/front_end/openVRE/public/phplib/classes/ProcessSGE.php b/front_end/openVRE/public/phplib/classes/ProcessSGE.php index 47888b13..2d326db3 100644 --- a/front_end/openVRE/public/phplib/classes/ProcessSGE.php +++ b/front_end/openVRE/public/phplib/classes/ProcessSGE.php @@ -41,7 +41,7 @@ class ProcessSGE private Logger $logger; - public function __construct($cl = false, $workDir = "", $queue = "srv.q", $jobname = "", $cpu = 1, $mem = 0, $logFile = "job_output.log", $errFile = "job_error.log") + public function __construct($cl = false, $workDir = "", $queue = "local.q", $jobname = "", $cpu = 1, $mem = 0, $logFile = "job_output.log", $errFile = "job_error.log") { $this->logger = LoggerFactory::getLogger("Process SGE interface"); $current_user = posix_getpwuid(posix_geteuid()); @@ -94,7 +94,7 @@ private function runCom() public function setFullCommand() { $workDir = $this->workDir; - $command = QSUB . " -N '" . $this->jobname . "' -wd $workDir -q " . $this->queue . " -o " . $this->logFile . " -e " . $this->errFile; + $command = QSUB . " -notify -N '" . $this->jobname . "' -wd $workDir -q " . $this->queue . " -o " . $this->logFile . " -e " . $this->errFile; if ($this->cpu > 1) { $command .= " -l cpu=" . $this->cpu; } @@ -131,6 +131,7 @@ public function getRunningJobInfo($pid) if (is_null($jobState[0])) { $job['state'] = "FINISHING"; + LoggerFactory::getPersistentLogger()->info("Job {pid} finished", array("pid" => $pid)); } else { list($pid, $state) = explode("\t", $jobState[0]); $job['state'] = $this->jobState[$state]; @@ -177,7 +178,8 @@ public function stop($pid = null) $command = QDEL . ' ' . $pid; exec($command, $r); $res = join(" ", $r); - log_addInfo($jobid, "SGE/qdel: " . $res); + $this->logger->info("Job stopped: " . $res); + LoggerFactory::getPersistentLogger()->info("Job {pid} stopped", array("pid" => $pid)); if (preg_match('/has deleted/i', $res) || preg_match('/registered the job \d+ for deletion/', $res)) { return array(true, $res); } else { diff --git a/front_end/openVRE/public/phplib/classes/ProcessSlurm.php b/front_end/openVRE/public/phplib/classes/ProcessSlurm.php index f362efdc..aee6aed1 100644 --- a/front_end/openVRE/public/phplib/classes/ProcessSlurm.php +++ b/front_end/openVRE/public/phplib/classes/ProcessSlurm.php @@ -200,6 +200,7 @@ public function getRunningJobInfo($pid) if (!$line) { $this->logger->debug("ProcessSlurm: getRunningJobInfo: job not running anymore. State: FINISHING"); + LoggerFactory::getPersistentLogger()->info("Job {pid} finished.", array("pid" => $pid)); $job['state'] = "FINISHING"; // log message like SGE version: $this->logger->debug("ProcessSlurm: getRunningJobInfo: log message added"); diff --git a/front_end/openVRE/public/phplib/classes/SwiftClient.py b/front_end/openVRE/public/phplib/classes/SwiftClient.py deleted file mode 100644 index 433b2766..00000000 --- a/front_end/openVRE/public/phplib/classes/SwiftClient.py +++ /dev/null @@ -1,60 +0,0 @@ -import swiftclient - - - -class SwiftClientExample: - def __init__(self, auth_url, username, password, project_name, user_domain_name, project_domain_name): - self.auth_url = auth_url - self.username = username - self.password = password - self.project_name = project_name - self.user_domain_name = user_domain_name - self.project_domain_name = project_domain_name - self.conn = None - - def authenticate(self): - self.conn = swiftclient.Connection( - authurl=self.auth_url, - user=self.username, - key=self.password - ) - - def list_containers(self): - if self.conn is None: - return "Not authenticated." - - containers = self.conn.get_account()[1] - return [container["name"] for container in containers] - - def download_object(self, container_name, object_name): - if self.conn is None: - return "Not authenticated." - - try: - response, content = self.conn.get_object(container_name, object_name) - with open(object_name, "wb") as f: - f.write(content) - return f"Object '{object_name}' downloaded and saved successfully." - except swiftclient.exceptions.ClientException as e: - return f"Failed to download object '{object_name}': {e}" - - def close_connection(self): - if self.conn is not None: - self.conn.close() - - - -if __name__ == "__main__": - - auth_url="https://ncloud.bsc.es:5000/v3" - username="bsc23829" - password="Maio23!" - project_name="bsc22Disc4All" - user_domain_name="bsc-compute" - project_domain_name="bsc22Disc4All" - swift_client = SwiftClientExample(auth_url, username, password, project_name, user_domain_name, project_domain_name) - swift_client.authenticate() - - swift_client.list_containers() - - swift_client.close_connection() diff --git a/front_end/openVRE/public/phplib/classes/Tooljob.php b/front_end/openVRE/public/phplib/classes/Tooljob.php index 5f92480a..43e1d33c 100644 --- a/front_end/openVRE/public/phplib/classes/Tooljob.php +++ b/front_end/openVRE/public/phplib/classes/Tooljob.php @@ -1048,7 +1048,7 @@ protected function getFreePort() } - protected function setBashCommandDockerSgeInteractive($tool, $cmd_envs) + protected function setBashCommandDockerSgeInteractive($tool, $customToolParameters) { $this->job_type = "interactive"; $container_port = $tool['infrastructure']['container_port']; @@ -1058,7 +1058,7 @@ protected function setBashCommandDockerSgeInteractive($tool, $cmd_envs) throw new UnexpectedValueException("No free ports available to run the interactive tool."); } - $networkName = $GLOBALS['networkName']; + $networkName = $GLOBALS['NETWORK_NAME']; $cmd = <<containerName \ --net $networkName \ - $cmd_envs \ + $customToolParameters \ --hostname $this->containerName \ -p $hostPort:{$tool['infrastructure']['container_port']} {$tool['infrastructure']['container_image']} {$tool['infrastructure']['executable']}); EOF; @@ -1105,7 +1105,7 @@ protected function setBashCommandDockerSgeInteractive($tool, $cmd_envs) } - protected function setBashCommandDockerCompose($tool, $cmd_envs) + protected function setBashCommandDockerCompose($tool, $customToolParameters) { $this->job_type = "interactive"; $dockerComposeFile = $GLOBALS['toolsPath'] . $tool['infrastructure']['docker_path']; @@ -1136,7 +1136,28 @@ protected function setBashCommandDockerCompose($tool, $cmd_envs) echo '# End time:' \$(date) >> $this->log_file_virtual; EOF; - return $cmd . "\n" . $monitorContainer . $cmd_envs; + return $cmd . "\n" . $monitorContainer . $customToolParameters; + } + + + protected function isToolRunning() + { + $ch = curl_init(); + $defaultInternalPort = 8787; + curl_setopt_array($ch, [ + CURLOPT_URL => $this->containerName . ":" . $defaultInternalPort, + CURLOPT_NOBODY => true, + CURLOPT_TIMEOUT => 5, + CURLOPT_CONNECTTIMEOUT => 5, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => false, + ]); + + curl_exec($ch); + $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + + return $httpCode !== 0; } @@ -1148,23 +1169,46 @@ protected function setBashCommandDockerSge($tool) } $this->containerName = $tool['infrastructure']['container_image'] . "_" . $_SESSION['User']['activeProject']; - $cmd_envs = ""; + $customToolParameters = ""; $envReplacements = ['$this->containerName' => $this->containerName]; foreach ($tool['infrastructure']['container_env'] as $env_key => $env_value) { $env_value = str_replace(array_keys($envReplacements), array_values($envReplacements), $env_value); - $cmd_envs .= "-e $env_key=$env_value "; + $customToolParameters .= "-e $env_key=$env_value "; } foreach ($tool['infrastructure']['volumes'] as $hostDir => $containerDir) { $userHomeDir = $this->root_dir_volumes . "/" . $this->project; - $cmd_envs .= "-v $userHomeDir" . "$hostDir:$containerDir "; + $customToolParameters .= "-v $userHomeDir" . "$hostDir:$containerDir "; + + $user = getUserById($_SESSION['User']['_id']); + $dataDir = $user['id'] . "/" . $user['activeProject']; + $upDirId = createGSDirBNS($dataDir . $hostDir, 1); + getProjectLogger()->info("Creating directory:" . $dataDir . $hostDir . "($upDirId)"); + addMetadataToFile($upDirId, array( + "expiration" => -1, + "description" => "Uploaded personal data" + )); + + $dataDirP = $GLOBALS['dataDir'] . "/$dataDir"; + if (!is_dir("$dataDirP" . $hostDir)) { + mkdir("$dataDirP" . $hostDir, 0775); + } + } + + if (!empty($tool['infrastructure']['user'])) { + $customToolParameters .= "--user " . escapeshellarg($tool['infrastructure']['user'] . " "); } if ($tool['infrastructure']['interactive']) { if ($tool['infrastructure']['docker_type'] == "compose") { - $cmd = $this->setBashCommandDockerCompose($tool, $cmd_envs); + $cmd = $this->setBashCommandDockerCompose($tool, $customToolParameters); } else { - $cmd = $this->setBashCommandDockerSgeInteractive($tool, $cmd_envs); + if ($this->isToolRunning()) { + $toolUrl = $GLOBALS['URL'] . "interactive-tool/" . $this->containerName . "/"; + $_SESSION['errorData']['Error'][] = "There is already a running instance of this tool at: " . $toolUrl . ""; + } + + $cmd = $this->setBashCommandDockerSgeInteractive($tool, $customToolParameters); } } else { $cmd_vre = $tool['infrastructure']['executable'] . @@ -1175,7 +1219,7 @@ protected function setBashCommandDockerSge($tool) $cmd = "docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock -d" . - " " . $cmd_envs . + " " . $customToolParameters . "--memory=" . $tool['infrastructure']['memory'] . "g" . " -v " . $this->pub_dir_volumes . ":" . $GLOBALS['shared'] . "public_tmp/ " . " -v " . $this->root_dir_volumes . ":" . $GLOBALS['shared'] . "userdata_tmp/{$_SESSION['User']['id']}" . @@ -1244,9 +1288,9 @@ protected function setBashCmd_docker_EGA($tool) " --out_metadata " . $this->stageout_file_virtual . " --log_file " . $this->log_file_virtual; - $cmd_envs = ""; + $customToolParameters = ""; foreach ($tool['infrastructure']['container_env'][0] as $env_key => $env_value) { - $cmd_envs .= "-e $env_key=$env_value "; + $customToolParameters .= "-e $env_key=$env_value "; } $vaultKey = $_SESSION['userVaultInfo']['vaultKey']; @@ -1261,7 +1305,7 @@ protected function setBashCmd_docker_EGA($tool) } $cmd = "docker run --device /dev/fuse --security-opt apparmor:unconfined --cap-add SYS_ADMIN -v /var/run/docker.sock:/var/run/docker.sock " . - " " . $cmd_envs . + " " . $customToolParameters . " -v " . $this->pub_dir_host . ":" . $GLOBALS['shared'] . "public_tmp/ " . " -v " . $this->root_dir_host . "/" . $_SESSION['User']['id'] . ":" . $GLOBALS['shared'] . "userdata_tmp/" . $_SESSION['User']['id'] . " --tmpfs " . "/clean_files:rw,uid=1000,gid=1000" . @@ -1443,6 +1487,7 @@ protected function enqueue($tool) $pid = execJob($this->working_dir, $this->submission_file, $queue, $cpus, $memory, $this->stdout_file, $this->stderr_file, $jobManager, $this->toolId, $jobOptions); $this->logger->info("Tool job submitted to SGE queue '$queue' (PID=$pid)"); + LoggerFactory::getPersistentLogger()->info("Job {pid} for tool {toolId} submitted to SGE queue {queue}", array( "toolId" => $this->toolId, "queue" => $queue, "pid" => $pid)); $this->pid = $pid; return $pid; diff --git a/front_end/openVRE/public/phplib/db.inc.php b/front_end/openVRE/public/phplib/db.inc.php index 38ce06bd..46824e0b 100644 --- a/front_end/openVRE/public/phplib/db.inc.php +++ b/front_end/openVRE/public/phplib/db.inc.php @@ -20,7 +20,7 @@ // create handlers $dbname = getenv('MONGO_MAIN_DB'); - +$GLOBALS['mongodbClient'] = $VREConn; $GLOBALS['db'] = $VREConn->$dbname; $GLOBALS['usersCol'] = $GLOBALS['db']->users; $GLOBALS['filesCol'] = $GLOBALS['db']->files; @@ -28,10 +28,10 @@ $GLOBALS['logMailCol'] = $GLOBALS['db']->checkMail; $GLOBALS['toolsCol'] = $GLOBALS['db']->tools; $GLOBALS['visualizersCol'] = $GLOBALS['db']->visualizers; -$GLOBALS['fileFormatsCol'] = $GLOBALS['db']->file_formats; +$GLOBALS['fileFormatsCol'] = $GLOBALS['db']->file_formats; $GLOBALS['dataTypesCol'] = $GLOBALS['db']->data_types; $GLOBALS['helpsCol'] = $GLOBALS['db']->helps; $GLOBALS['sampleDataCol'] = $GLOBALS['db']->sampleData; -$GLOBALS['logExecutionsCol'] = $GLOBALS['db']->log_executions; +$GLOBALS['actionLogs'] = $GLOBALS['db']->action_logs; //adding new cred for SITES collection $GLOBALS['sitesCol'] = $GLOBALS['db']->sites; diff --git a/front_end/openVRE/public/phplib/interactive_tools.inc.php b/front_end/openVRE/public/phplib/interactive_tools.inc.php new file mode 100644 index 00000000..2d41f7ee --- /dev/null +++ b/front_end/openVRE/public/phplib/interactive_tools.inc.php @@ -0,0 +1,110 @@ + false, + "reload" => false, + "title" => "Job not found", + "message" => "The requested interactive session could not be found." + ]; + } + + $job = $jobs[$pid]; + + /*---------------------------------------------------- + * Job state + *---------------------------------------------------*/ + + if ($job['state'] == "PENDING") { + return [ + "ready" => false, + "reload" => true, + "title" => "Waiting for scheduler", + "message" => "The interactive session is waiting for compute resources." + ]; + } + + if ($job['state'] != "RUNNING") { + return [ + "ready" => false, + "reload" => false, + "title" => "Session finished", + "message" => "The interactive session is no longer running." + ]; + } + + /*---------------------------------------------------- + * Wait until stdout exists + *---------------------------------------------------*/ + + if (!is_file($job['stdout_file'])) { + return [ + "ready" => false, + "reload" => true, + "title" => "Starting container", + "message" => "Waiting for launcher output..." + ]; + } + + $stdout = file_get_contents($job['stdout_file']); + + /*---------------------------------------------------- + * Save metadata in the session (KEEP THIS) + *---------------------------------------------------*/ + + /* + if (preg_match('/ExposedPort: (\d+)/', $stdout, $matches)) { + $_SESSION['User']['lastjobs'][$pid]['interactive_tool']['port'] = $matches[1]; + } + + if (preg_match('/ContainerID: (\w+)/', $stdout, $matches)) { + $_SESSION['User']['lastjobs'][$pid]['interactive_tool']['container_id'] = $matches[1]; + } + + if (preg_match('/ContainerName: (\S+)/', $stdout, $matches)) { + $_SESSION['User']['lastjobs'][$pid]['interactive_tool']['containerName'] = $matches[1]; + } + */ + + /*---------------------------------------------------- + * Has the service started? + *---------------------------------------------------*/ + + if (strpos($stdout, "Service UP") === false) { + + return [ + "ready" => false, + "reload" => true, + "title" => "Preparing interactive session", + "message" => "The container is running. Waiting for the application to become available..." + ]; + } + + /*---------------------------------------------------- + * Mark service ready + *---------------------------------------------------*/ + + $url = $GLOBALS['SERVER'] + . $interactiveToolprefix + . $job['containerName'] + . "/"; + + return [ + "ready" => true, + "reload" => false, + "title" => "Interactive session ready", + "message" => "Your session is ready.", + "url" => $url, + "job" => $job + ]; +} diff --git a/front_end/openVRE/public/phplib/mail.inc.php b/front_end/openVRE/public/phplib/mail.inc.php index 33b87696..186c91d0 100644 --- a/front_end/openVRE/public/phplib/mail.inc.php +++ b/front_end/openVRE/public/phplib/mail.inc.php @@ -21,8 +21,8 @@ function sendEmail($recipient, $subject, $body, $reply = null, $bcc = null) $reply = $GLOBALS['ADMINMAIL']; } - $mail->AddReplyTo($reply, $GLOBALS['FROMNAME']); - $mail->SetFrom($reply, $GLOBALS['FROMNAME']); + $mail->SetFrom(getenv('MAIL_USER'), $GLOBALS['FROMNAME']); + $mail->AddReplyTo($reply); $mail->Subject = $subject; $mail->Body = $body; $mail->AddAddress($recipient); diff --git a/front_end/openVRE/public/phplib/processJob.inc.php b/front_end/openVRE/public/phplib/processJob.inc.php index 5447a205..b93485c1 100644 --- a/front_end/openVRE/public/phplib/processJob.inc.php +++ b/front_end/openVRE/public/phplib/processJob.inc.php @@ -25,16 +25,19 @@ function execJob($workDir, $shFile, $queue, $cpus = 1, $mem = 0, $logFile = "job if (is_null($_SESSION['User']['id'])) { getJobProcessLogger()->error("User ID not found in session."); + LoggerFactory::getPersistentLogger()->error("User ID {userId} not found in session.", array('userId' => $_SESSION['User']['id'])); throw new NotFoundException("User ID not found in session."); } if (!file_exists($shFile)) { getJobProcessLogger()->error("Shell script file does not exist: $shFile"); + LoggerFactory::getPersistentLogger()->error("Shell script file {shFile} does not exist", array('shFile' => $shFile)); throw new NotFoundException("Shell script file does not exist: $shFile"); } if (!is_dir($workDir)) { getJobProcessLogger()->error("Working directory does not exist: $workDir"); + LoggerFactory::getPersistentLogger()->error("Working directory {workDir} does not exist", array('workDir' => $workDir)); throw new NotFoundException("Working directory does not exist: $workDir"); } @@ -251,6 +254,7 @@ function delJob($pid, $launcherType = null, $login = null) } $_SESSION['errorData']['Info'][] = "Job successfully cancelled"; + LoggerFactory::getPersistentLogger()->info("Job {pid} successfully cancelled", array('pid' => $pid)); // wait to make qdel/terminateActivity effective sleep(15); diff --git a/front_end/openVRE/public/phplib/projects.inc.php b/front_end/openVRE/public/phplib/projects.inc.php index be0c3c70..60957c35 100644 --- a/front_end/openVRE/public/phplib/projects.inc.php +++ b/front_end/openVRE/public/phplib/projects.inc.php @@ -3,9 +3,7 @@ use League\OAuth2\Client\Token\AccessToken; use OpenVRE\LoggerFactory; use OpenVRE\NotFoundException; -use OpenVRE\Oauth2Provider; use OpenVRE\UserType; -use OpenVRE\VaultClient; function getProjectLogger() @@ -19,6 +17,7 @@ function getProjectLogger() return $logger; } + function prepUserWorkSpace($homeDir, $projectDir, $sampleData = "", $projectData = array(), $verbose = false, $asRoot = 0) { // set current directory @@ -1280,6 +1279,7 @@ function updatePendingFiles($sessionId) // and consequently reload workspace (checkPendingJobs.php) } else { getProjectLogger()->info("Automatic job update detects job $pid is not running anymore"); + LoggerFactory::getPersistentLogger()->info("Automatic job update detects job {pid} is not running anymore", array('pid' => $pid)); $SGE_updated[$pid] = $job; $SGE_updated[$pid]['state'] = "NOT_RUNNING"; } @@ -1356,6 +1356,7 @@ function processRunningJobInfo($job, $jobProcess, $pid, $title, $descrip, &$file function processFinishedJobInfo($job, $pid, $title, &$filesPending) { getProjectLogger()->info("Workspace reload detects job $pid is not running anymore"); + LoggerFactory::getPersistentLogger()->info("Workspace reload detects job {pid} is not running anymore", array('pid' => $pid)); unset($_SESSION['errorData']); $job_in_err = 0; @@ -1366,6 +1367,7 @@ function processFinishedJobInfo($job, $pid, $title, &$filesPending) getProjectLogger()->error("Tool '" . $job['toolId'] . "' received from JobTool not registered"); getProjectLogger()->error("Cannot obtain results from '$title' in folder '" . basename($job['working_dir']) . "'. Job metadata is not valid."); getProjectLogger()->error("Failed to register $pid job outfiles. Job metadata has toolId '" . $job['toolId'] . "'"); + LoggerFactory::getPersistentLogger()->error("Failed to register {pid} job outfiles. Tool {toolId} not registered.", array('pid' => $pid, 'toolId' => $job['toolId'])); $job_in_err = 1; return; } @@ -1377,6 +1379,7 @@ function processFinishedJobInfo($job, $pid, $title, &$filesPending) getProjectLogger()->debug("Finished building output from toolINFO + stageout_file + stageout_data: " . json_encode($outs_files)); if (empty($outs_files)) { getProjectLogger()->warning("Failed to register $pid job outfiles. Output file list empty."); + LoggerFactory::getPersistentLogger()->warning("Failed to register {pid} job outfiles. Output file list empty.", array('pid' => $pid)); $job_in_err = 1; } @@ -1414,7 +1417,7 @@ function processFinishedJobInfo($job, $pid, $title, &$filesPending) print "
    Recovering path from remote_paths: $remote_path
    "; $_SESSION['errorData']['Error'][] = "Recovering path from remote_paths: $remote_path"; } - // this is right (?) + $out_data['path'] = $remote_path; } else { if ($is_required) { @@ -1423,7 +1426,7 @@ function processFinishedJobInfo($job, $pid, $title, &$filesPending) $msg .= ". No 'path' and no usable 'remote_paths' found."; $msg .= ". Job metadata: " . print_r($out_data, true); $_SESSION['errorData']['Error'][] = $msg; - log_addOutregister($pid, $msg); + LoggerFactory::getPersistentLogger()->error("Job output file {outName} not created.", array('outName' => $out_name)); $job_in_err = 1; } continue; @@ -1481,8 +1484,9 @@ function processFinishedJobInfo($job, $pid, $title, &$filesPending) // job successfully finished and already in mongo. Update medatada if ($fileId) { - getProjectLogger()->debug("JOB $pid finished successfully."); + getProjectLogger()->info("JOB $pid finished successfully."); getProjectLogger()->debug("Updating only outfile $out_name '$rfn' metadata from job $pid"); + LoggerFactory::getPersistentLogger()->info("Job {pid} finished successfully.", array('pid' => $pid)); list($out_vre, $metadata) = getVREfile_fromFile($out_data); addMetadataToFile($fileId, $metadata); } elseif (is_file($rfn) || is_dir($rfn) || isset($out_data['meta_data']['remote_paths'][0]['remote_path'])) { // job successfully finished but not yet on mongo. Save output @@ -1547,6 +1551,7 @@ function processFinishedJobInfo($job, $pid, $title, &$filesPending) if ($job_in_err) { getProjectLogger()->error("Failed to register all job outfiles"); getProjectLogger()->error("JOB $pid FINISHED but with errors"); + LoggerFactory::getPersistentLogger()->error("Job {pid} finished with errors. Failed to register all job outfiles.", array('pid' => $pid)); $logFileP = $job['log_file']; $logFile = fromAbsPath_toPath($job['log_file']); @@ -1570,7 +1575,8 @@ function processFinishedJobInfo($job, $pid, $title, &$filesPending) } } } else { - getProjectLogger()->debug("JOB $pid finished successfully."); + getProjectLogger()->info("JOB $pid finished successfully."); + LoggerFactory::getPersistentLogger()->info("Job {pid} finished successfully.", array('pid' => $pid)); } } @@ -1756,7 +1762,7 @@ function build_outputs_list($tool, $stageout_job, $stageout_file) array_push($stageout_meta[$out['name']], $out); } - } elseif ($tool['external'] !== false) { + } elseif ($tool['external'] !== false && !$tool['infrastructure']['interactive']) { $_SESSION['errorData']['Warning'][] = date("h:i:s") . ": Tool stageout file '" . $stageout_file . "' is not found"; getProjectLogger()->warning("Tool stageout file '" . $stageout_file . "' is not found"); } @@ -1777,20 +1783,6 @@ function build_outputs_list($tool, $stageout_job, $stageout_file) array_push($stageout_data[$out['name']], $out); } } - if ($debug) { - - print "\n__________FROM FILE________________\n"; - print json_encode($stageout_meta, JSON_PRETTY_PRINT); - - print "\n__________FROM JOB________________\n"; - print json_encode($stageout_data, JSON_PRETTY_PRINT); - - // Merge FILE + JOB (job overrides file) - $stageout_meta = array_merge($stageout_meta, $stageout_data); - - print "\n__________MERGED (FILE + JOB)________________\n"; - print json_encode($stageout_meta, JSON_PRETTY_PRINT); - } // merging file data from tool and stageout_file $outs_meta = array(); @@ -1913,14 +1905,20 @@ function downloadFile($rfn) $fhandle = fopen($rfn, 'r'); fseek($fhandle, $offset); // seek to the requested offset, this is 0 if it's not a partial content request - $data = fread($fhandle, $length); fclose($fhandle); header('HTTP/1.1 206 Partial Content'); header('Content-Range: bytes ' . $offset . '-' . ($offset + $length) . '/' . $size); } - header("Content-Disposition: attachment;filename=" . $filename); - header('Content-Type: ' . $content_type); + + if (strtolower($fileExtension) === 'html' || strtolower($fileExtension) === 'htm') { + header("Content-Disposition: inline; filename=\"$filename\""); + header("Content-Type: text/html; charset=UTF-8"); + } else { + header("Content-Disposition: attachment; filename=\"$filename\""); + header("Content-Type: " . $content_type); + } + header("Accept-Ranges: bytes"); header("Pragma: public"); header("Expires: -1"); @@ -2283,3 +2281,130 @@ function moveFiles($fns, $target_fn) */ } } + + +function syncProjectFiles($projects) +{ + foreach ($projects as $projectId => $projectAttributes) { + $projectFullPath = $GLOBALS['dataDir'] . "/" . $projectAttributes['path']; + + if (!is_dir($projectFullPath)) { + continue; // skip projects whose root path doesn't exist + } + + syncDirectoryRecursive($projectFullPath, $projectAttributes['path'], $projectId, null); + } +} + + +function syncDirectoryRecursive($fullFolderPath, $relativeFolderPath, $projectId, $parentDirId) +{ + $entries = scandir($fullFolderPath); + + foreach ($entries as $entry) { + // skip hidden files/folders (this also covers "." and "..") + if ($entry[0] === '.') { + continue; + } + + $relativePath = $relativeFolderPath . "/" . $entry; + $fullPath = $fullFolderPath . "/" . $entry; + + if (is_dir($fullPath)) { + // skip folders named "run*" (e.g. "run001") + if (fnmatch('run*', $entry)) { + continue; + } + + // ensure this subdirectory itself is registered, then recurse into it + $subDirId = ensureDirRegistered($relativePath, $projectId, $parentDirId); + syncDirectoryRecursive($fullPath, $relativePath, $projectId, $subDirId); + } else { + ensureFileRegistered($relativePath, $fullPath, $projectId, $parentDirId); + } + } +} + +function ensureDirRegistered($relativePath, $projectId, $parentDirId) +{ + $existingId = getGSFileId_fromPath($relativePath); + if ($existingId) { + return $existingId; + } + + $dirId = createLabel(); + + $mongoDirDocument = array( + '_id' => $dirId, + 'mtime' => new MongoDB\BSON\UTCDateTime(strtotime("now") * 1000), + 'owner' => $_SESSION['User']['id'], + 'path' => $relativePath, + 'project' => $projectId, + 'parentDir' => $parentDirId, + 'type' => "dir", + 'files' => [] + ); + + $GLOBALS['filesCol']->updateOne( + ['_id' => $dirId], + ['$set' => $mongoDirDocument], + ['upsert' => true] + ); + + if ($parentDirId) { + $GLOBALS['filesCol']->updateOne( + ['_id' => $parentDirId], + ['$addToSet' => ['files' => $dirId]] + ); + } + + return $dirId; +} + +function ensureFileRegistered($relativePath, $fullPath, $projectId, $parentDirId) +{ + if (getGSFileId_fromPath($relativePath)) { + return; // already registered + } + + $fileId = createLabel(); + + $mongoFileDocument = array( + '_id' => $fileId, + 'mtime' => new MongoDB\BSON\UTCDateTime(strtotime("now") * 1000), + 'owner' => $_SESSION['User']['id'], + 'size' => filesize($fullPath), + 'path' => $relativePath, + 'project' => $projectId, + 'parentDir' => $parentDirId, + 'type' => "file" + ); + + $mongoFileMetadataDocument = array( + '_id' => $fileId, + 'compressed' => false, + 'data_type' => null, + 'format' => null, + 'validated' => true, + 'visible' => true + ); + + $GLOBALS['filesCol']->updateOne( + ['_id' => $fileId], + ['$set' => $mongoFileDocument], + ['upsert' => true] + ); + + $GLOBALS['filesMetaCol']->updateOne( + ['_id' => $fileId], + ['$set' => $mongoFileMetadataDocument], + ['upsert' => true] + ); + + if ($parentDirId) { + $GLOBALS['filesCol']->updateOne( + ['_id' => $parentDirId], + ['$addToSet' => ['files' => $fileId]] + ); + } +} diff --git a/front_end/openVRE/public/user/usrProfile.php b/front_end/openVRE/public/user/usrProfile.php index e5eebac4..63411302 100644 --- a/front_end/openVRE/public/user/usrProfile.php +++ b/front_end/openVRE/public/user/usrProfile.php @@ -3,6 +3,10 @@ require __DIR__ . "/../../config/bootstrap.php"; redirectOutside(); +$includeHpc = false; +$includeEgaFiles = false; +$includeOpenStack = false; + ?> @@ -396,78 +400,104 @@ class="icon-question tooltips" data-container="body" class="caption-subject font-blue-madison bold uppercase">Linked Accounts - + +
    + +

    HPC resources access (via SSH)

    + +
    + +

    + + Do you have an account to an HPC facility? + + Link it and you'll be able to launch jobs there. Data will + be transferred via SSH, from/to your HPC home directory. +

    +
    + +
    + +
    + +
    +
    -
    -
    - -
    - -
    +
    + +
    + +
    -
    - -   Generate new pair of Keys - -   Delete Account -
    -
    - -
    --> - +
    + + + +
    + +
    + +
    + +
    + +
    + +
    + +   Generate new pair of Keys + +   Delete Account +
    +
    + +
    + + + + +
    -
    @@ -489,10 +519,12 @@ class="btn green">   Link
    - -
    - + + + + +
    - + +
    diff --git a/front_end/openVRE/public/workspace/index.php b/front_end/openVRE/public/workspace/index.php index ba925755..56a745af 100644 --- a/front_end/openVRE/public/workspace/index.php +++ b/front_end/openVRE/public/workspace/index.php @@ -30,6 +30,7 @@ // project list $projects = getProjects_byOwner(); +syncProjectFiles($projects); //update files workspace content (job and files) $allFiles = getFilesToDisplay(array('_id' => $_SESSION['User']['dataDir'])); diff --git a/mongodb/init_documents/action_logs.json b/mongodb/init_documents/action_logs.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/mongodb/init_documents/action_logs.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/mongodb/mongo-init.sh b/mongodb/mongo-init.sh index ac833071..9521dec7 100644 --- a/mongodb/mongo-init.sh +++ b/mongodb/mongo-init.sh @@ -24,9 +24,25 @@ if (db.getUser("$MONGO_INITDB_USERNAME") === null) { EOF for mongo_document in /init_documents/*.json; do - mongoimport --db ${MONGO_MAIN_DB} \ - --jsonArray \ - --username ${MONGO_INITDB_USERNAME} \ - --password ${MONGO_INITDB_PASSWORD} \ - --file $mongo_document -done + collection_name=$(basename "$mongo_document" .json) + + # Strip all whitespace and compare to "[]" to detect an empty array + stripped_content=$(tr -d '[:space:]' < "$mongo_document") + + if [ "$stripped_content" == "[]" ]; then + echo "Creating empty collection: ${collection_name}" + mongosh "${MONGO_MAIN_DB}" \ + --username "${MONGO_INITDB_USERNAME}" \ + --password "${MONGO_INITDB_PASSWORD}" \ + --authenticationDatabase "${MONGO_MAIN_DB}" \ + --eval "db.createCollection('${collection_name}')" + else + echo "Importing into collection: ${collection_name}" + mongoimport --db "${MONGO_MAIN_DB}" \ + --collection "${collection_name}" \ + --jsonArray \ + --username "${MONGO_INITDB_USERNAME}" \ + --password "${MONGO_INITDB_PASSWORD}" \ + --file "$mongo_document" + fi +done \ No newline at end of file diff --git a/sge/setup_gridengine.sh b/sge/setup_gridengine.sh index 86d47d14..b720f19b 100644 --- a/sge/setup_gridengine.sh +++ b/sge/setup_gridengine.sh @@ -39,7 +39,7 @@ qtype BATCH INTERACTIVE ckpt_list NONE pe_list make rerun FALSE -slots 10 +slots 4 tmpdir /tmp shell /bin/bash prolog NONE diff --git a/vault/config/init-config.hcl.sample b/vault/config/init-config.hcl.sample new file mode 100644 index 00000000..c799a59d --- /dev/null +++ b/vault/config/init-config.hcl.sample @@ -0,0 +1,16 @@ +api_addr = "https://vault-server:8200" +cluster_addr = "https://vault-server:8201" +cluster_name = "vault-cluster" +disable_mlock = true +ui = true + +listener "tcp" { +address = "0.0.0.0:8200" +tls_cert_file = "/etc/ssl/certs/vault.crt" +tls_key_file = "/etc/ssl/certs/vault.key" +} + +storage "raft" { +path = "/vault/data" +node_id = "vault-server-node" +} \ No newline at end of file diff --git a/vault/vault-init-prod.md b/vault/vault-init-prod.md new file mode 100644 index 00000000..e42a89d3 --- /dev/null +++ b/vault/vault-init-prod.md @@ -0,0 +1,114 @@ + +# Vault Setup Manual Steps + +## Check Vault user and group IDs + +```bash +docker compose run --rm vault-server /bin/sh -c "id -u vault && id -g vault" +``` + +Set the variables (adjust if needed): + +```bash +VAULT_USER=100 # Change if different +VAULT_GROUP=1000 # Change if different +``` + +## Copy and update the config file + +```bash +cp vault/config/init-config.hcl.sample vault/config/init-config.hcl +``` + +And update values according to your setup. + +## Fix data folder ownership inside the container + +By default, the folder is owned by `root`. Update it: + +```bash +docker compose run --rm --entrypoint /bin/sh vault-server -c "chown -R $VAULT_USER:$VAULT_GROUP /vault/data" +``` + +## Start the Vault and Keycloak containers + +```bash +docker compose up vault-server keycloak -d +``` + +## Enter the container and install basic tools + +```bash +docker exec -it vault-server /bin/sh +``` + +Inside the container: + +```bash +apk add jq +``` + +## Initialize and unseal Vault + +```bash +vault operator init +vault operator unseal +vault login +``` + +## Configure JWT Authentication + +Enable JWT signature verification method: +```bash +vault auth enable jwt +``` + +Navigate to config directory: + +```bash +cd vault/config +``` + +Configure JWT and create a role: + +```bash +vault write auth/jwt/config oidc_discovery_url="$KEYCLOAK_SERVER/realms/$KEYCLOAK_REALM" \ + bound_issuer="$KEYCLOAK_SERVER/realms/$KEYCLOAK_REALM" + +vault write auth/jwt/role/user-role \ + role_type="jwt" \ + bound_audiences="account" \ + user_claim="sub" \ + policies="user-policy" \ + ttl="1h" +``` + +Retrieve the JWT accessor: + +```bash +JWT_ACCESSOR=$(vault auth list -format=json | jq -r '."jwt/".accessor') +``` + +Generate the user policy file: + +```bash +sed "s/JWT_ACCESSOR/$JWT_ACCESSOR/g" jwt-user-policies-template.hcl > jwt-user-policies.hcl +``` + +Write the policy: + +```bash +vault policy write user-policy jwt-user-policies.hcl +``` + +## Enable KV secrets engine + +```bash +vault secrets enable -path=secret -version=1 kv +``` + +## Revoke root token (optional but recommended) + +```bash +vault token revoke "" +``` \ No newline at end of file