diff --git a/docs/source/DeveloperGuide/Systems/ResourceStatus/index.rst b/docs/source/DeveloperGuide/Systems/ResourceStatus/index.rst index 65e2f3335ea..320053def2c 100644 --- a/docs/source/DeveloperGuide/Systems/ResourceStatus/index.rst +++ b/docs/source/DeveloperGuide/Systems/ResourceStatus/index.rst @@ -115,7 +115,7 @@ Cache tables for metrics used by policies. * - PolicyResult - Policy evaluation results (Element, Name, PolicyName, Status, Reason) * - SpaceTokenOccupancyCache - - Storage space usage (Endpoint, Token, Free, Guaranteed) + - Storage space usage (Token, Free, Guaranteed) * - TransferCache - Transfer quality metrics (SourceName, DestinationName, Metric, Value) diff --git a/src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py b/src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py index a76573318d1..f8de8b63b80 100644 --- a/src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py +++ b/src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py @@ -651,80 +651,5 @@ def addOrModifyPolicyResult( return self._getRPC().addOrModify("PolicyResult", prepareDict(columnNames, columnValues)) - # SpaceTokenOccupancyCache Methods ........................................... - - def selectSpaceTokenOccupancyCache( - self, endpoint=None, token=None, total=None, guaranteed=None, free=None, lastCheckTime=None, meta=None - ): - """ - Gets from SpaceTokenOccupancyCache all rows that match the parameters given. - - :param endpoint: endpoint - :type endpoint: string, list - :param token: name of the token - :type token: string, list - :param total: total terabytes - :type total: integer, list - :param guaranteed: guaranteed terabytes - :type guaranteed: integer, list - :param free: free terabytes - :type free: integer, list - :param lastCheckTime: time-stamp from which the result is effective - :type lastCheckTime: datetime, list - :param dict meta: metadata for the mysql query. Currently it is being used only for column selection. - For example: meta={'columns': ['Name']} will return only the 'Name' column. - :return: S_OK() || S_ERROR() - """ - columnNames = ["Endpoint", "Token", "Total", "Guaranteed", "Free", "LastCheckTime", "Meta"] - columnValues = [endpoint, token, total, guaranteed, free, lastCheckTime, meta] - - return self._getRPC().select("SpaceTokenOccupancyCache", prepareDict(columnNames, columnValues)) - - def deleteSpaceTokenOccupancyCache( - self, endpoint=None, token=None, total=None, guaranteed=None, free=None, lastCheckTime=None - ): - """ - Deletes from SpaceTokenOccupancyCache all rows that match the parameters given. - - :param endpoint: endpoint - :type endpoint: string, list - :param token: name of the token - :type token: string, list - :param total: total terabytes - :type total: integer, list - :param guaranteed: guaranteed terabytes - :type guaranteed: integer, list - :param free: free terabytes - :type free: integer, list - :param lastCheckTime: time-stamp from which the result is effective - :type lastCheckTime: datetime, list - :return: S_OK() || S_ERROR() - """ - columnNames = ["Endpoint", "Token", "Total", "Guaranteed", "Free", "LastCheckTime"] - columnValues = [endpoint, token, total, guaranteed, free, lastCheckTime] - - return self._getRPC().delete("SpaceTokenOccupancyCache", prepareDict(columnNames, columnValues)) - - def addOrModifySpaceTokenOccupancyCache( - self, endpoint=None, token=None, total=None, guaranteed=None, free=None, lastCheckTime=None - ): - """ - Adds or updates-if-duplicated to SpaceTokenOccupancyCache. Using `site` and `token` - to query the database, decides whether to insert or update the table. - - :param endpoint: endpoint - :type endpoint: string, list - :param str token: name of the token - :param int total: total terabytes - :param int guaranteed: guaranteed terabytes - :param int free: free terabytes - :param datetime lastCheckTime: time-stamp from which the result is effective - :return: S_OK() || S_ERROR() - """ - columnNames = ["Endpoint", "Token", "Total", "Guaranteed", "Free", "LastCheckTime"] - columnValues = [endpoint, token, total, guaranteed, free, lastCheckTime] - - return self._getRPC().addOrModify("SpaceTokenOccupancyCache", prepareDict(columnNames, columnValues)) - # EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF diff --git a/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py b/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py index f96e45ee132..525ac720e75 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py @@ -1,10 +1,10 @@ -""" FreeDiskSpaceCommand - The Command gets the free space that is left in a Storage Element +"""FreeDiskSpaceCommand +The Command gets the free space that is left in a Storage Element - Note: there are, still, many references to "space tokens", - for example ResourceManagementClient().selectSpaceTokenOccupancyCache(token=elementName) - This is for historical reasons, and shoud be fixed one day. - For the moment, when you see "token" or "space token" here, just read "StorageElement". +Note: there are, still, many references to "space tokens", +for example the SpaceTokenOccupancyCache table. +This is for historical reasons, and shoud be fixed one day. +For the moment, when you see "token" or "space token" here, just read "StorageElement". """ @@ -19,8 +19,8 @@ from DIRAC.Core.Utilities.TimeUtilities import DiracTime from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers from DIRAC.Resources.Storage.StorageElement import StorageElement -from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient from DIRAC.ResourceStatusSystem.Command.Command import Command +from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB from DIRAC.ResourceStatusSystem.Utilities import CSHelpers @@ -32,7 +32,7 @@ class FreeDiskSpaceCommand(Command): def __init__(self, args=None, clients=None): super().__init__(args, clients=clients) - self.rmClient = ResourceManagementClient() + self.rmDB = ResourceManagementDB() def _prepareCommand(self): """ @@ -80,7 +80,7 @@ def doNew(self, masterParams=None): free = occupancy["Free"] total = occupancy["Total"] - results = {"Endpoint": "Deprecated", "Free": free, "Total": total, "ElementName": elementName} + results = {"Free": free, "Total": total, "ElementName": elementName} result = self._storeCommand(results) if not result["OK"]: return result @@ -93,22 +93,23 @@ def _storeCommand(self, results): and adds records to the StorageOccupancy accounting. :param dict results: something like {'ElementName': 'CERN-HIST-EOS', - 'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server', 'Free': 3264963586.10073, 'Total': 8000000000.0} :returns: S_OK/S_ERROR dict """ # Stores in cache - res = self.rmClient.addOrModifySpaceTokenOccupancyCache( - endpoint=results["Endpoint"], - lastCheckTime=DiracTime.utcnow(), - free=results["Free"], - total=results["Total"], - token=results["ElementName"], + res = self.rmDB.addOrModify( + "SpaceTokenOccupancyCache", + { + "Token": results["ElementName"], + "Free": results["Free"], + "Total": results["Total"], + "LastCheckTime": DiracTime.utcnow(), + }, ) if not res["OK"]: - self.log.error("Error calling addOrModifySpaceTokenOccupancyCache", res["Message"]) + self.log.error("Error calling addOrModify on SpaceTokenOccupancyCache", res["Message"]) return res # Now proceed with the accounting @@ -117,8 +118,6 @@ def _storeCommand(self, results): return siteRes accountingDict = { - "StorageElement": results["ElementName"], - "Endpoint": results["Endpoint"], "Site": siteRes["Value"] if siteRes["Value"] else "unassigned", } @@ -151,7 +150,7 @@ def doCache(self): return params elementName, unit = params["Value"] - result = self.rmClient.selectSpaceTokenOccupancyCache(token=elementName) + result = self.rmDB.select("SpaceTokenOccupancyCache", {"Token": elementName}) if not result["OK"]: return result @@ -192,16 +191,16 @@ def doMaster(self): return self._cleanCommand() def _cleanCommand(self, toDelete=None): - """Clean the spaceTokenOccupancy table from old endpoints + """Clean the spaceTokenOccupancy table from old SEs - :param tuple toDelete: endpoint to remove (endpoint, storage_element_name), - e.g. ('httpg://srm-lhcb.cern.ch:8443/srm/managerv2', CERN-RAW) + :param tuple toDelete: storage_element_name """ if not toDelete: toDelete = [] - res = self.rmClient.selectSpaceTokenOccupancyCache( - meta={"older": ["LastCheckTime", DiracTime.utcnow() - timedelta(hours=6)]} + res = self.rmDB.select( + "SpaceTokenOccupancyCache", + {"meta": {"older": ["LastCheckTime", DiracTime.utcnow() - timedelta(hours=6)]}}, ) if not res["OK"]: return res @@ -223,7 +222,7 @@ def _cleanCommand(self, toDelete=None): toDelete = [toDelete] for ep in toDelete: - res = self.rmClient.deleteSpaceTokenOccupancyCache(ep[0], ep[1]) + res = self.rmDB.delete("SpaceTokenOccupancyCache", {"Token": ep[0]}) if not res["OK"]: self.log.warn("Could not delete entry from SpaceTokenOccupancyCache", res["Message"]) diff --git a/src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py b/src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py index 014d2755ef1..7d1e724840b 100644 --- a/src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py +++ b/src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py @@ -278,7 +278,6 @@ class SpaceTokenOccupancyCache(rmsBase): __tablename__ = "SpaceTokenOccupancyCache" __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} - endpoint = Column("Endpoint", String(128), nullable=False, primary_key=True) token = Column("Token", String(64), nullable=False, primary_key=True) guaranteed = Column("Guaranteed", Float(asdecimal=False), nullable=False, server_default="0") free = Column("Free", Float(asdecimal=False), nullable=False, server_default="0") @@ -293,7 +292,6 @@ def fromDict(self, dictionary): :type arguments: dict """ - self.endpoint = dictionary.get("Endpoint", self.endpoint) self.token = dictionary.get("Token", self.token) self.guaranteed = dictionary.get("Guaranteed", self.guaranteed) self.free = dictionary.get("Free", self.free) @@ -307,7 +305,7 @@ def fromDict(self, dictionary): def toList(self): """Simply returns a list of column values""" - return [self.endpoint, self.token, self.guaranteed, self.free, self.total, self.lastchecktime] + return [self.token, self.guaranteed, self.free, self.total, self.lastchecktime] class TransferCache(rmsBase): diff --git a/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py b/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py index d99bd44eb10..59fa78a2f5e 100644 --- a/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py +++ b/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py @@ -9,13 +9,14 @@ from datetime import datetime, timedelta # DIRAC -from DIRAC import S_OK, gConfig, S_ERROR +from DIRAC import S_ERROR, S_OK, gConfig +from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSiteCEMapping, getSites from DIRAC.Core.DISET.RequestHandler import RequestHandler from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.Utilities.SiteSEMapping import getSEHosts, getStorageElementsHosts from DIRAC.Core.Utilities.TimeUtilities import DiracTime -from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites, getSiteCEMapping from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers +from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB class PublisherHandlerMixin: @@ -40,6 +41,8 @@ def initializeHandler(cls, serviceInfoDict): resourceManagementClientClass = result["Value"] cls.rmClient = resourceManagementClientClass() + cls.rmDB = ResourceManagementDB() + return S_OK() types_getSites = [] @@ -331,6 +334,21 @@ def export_getCachedDowntimes(self, element, elementType, name, severity): return result + types_selectSpaceTokenOccupancyCache = [(str, list, type(None))] + + @classmethod + def export_selectSpaceTokenOccupancyCache(cls, token): + """ + Gets space token occupancy from the SpaceTokenOccupancyCache table. + + :param token: name of the token (StorageElement) + :type token: str or list + :return: S_OK() || S_ERROR() + """ + + params = {"Token": token} if token else {} + return cls.rmDB.select("SpaceTokenOccupancyCache", params) + types_setStatus = [str] * 7 def export_setStatus(self, element, name, statusType, status, elementType, username, lastCheckTime): diff --git a/src/DIRAC/ResourceStatusSystem/Service/ResourceManagementHandler.py b/src/DIRAC/ResourceStatusSystem/Service/ResourceManagementHandler.py index 1eddad62a8f..94ee26a3de8 100644 --- a/src/DIRAC/ResourceStatusSystem/Service/ResourceManagementHandler.py +++ b/src/DIRAC/ResourceStatusSystem/Service/ResourceManagementHandler.py @@ -1,11 +1,12 @@ -""" ResourceManagementHandler +"""ResourceManagementHandler - Module that allows users to access the ResourceManagementDB remotely. +Module that allows users to access the ResourceManagementDB remotely. """ -from DIRAC import gConfig, S_OK + +from DIRAC import S_OK, gConfig from DIRAC.Core.DISET.RequestHandler import RequestHandler, getServiceOption -from DIRAC.ResourceStatusSystem.Utilities import Synchronizer from DIRAC.ResourceStatusSystem.Service.ResourceStatusHandler import loadResourceStatusComponent +from DIRAC.ResourceStatusSystem.Utilities import Synchronizer class ResourceManagementHandlerMixin: diff --git a/tests/Integration/ResourceStatusSystem/Test_ResourceManagement.py b/tests/Integration/ResourceStatusSystem/Test_ResourceManagement.py index f721fdf871b..f21389a8b3c 100644 --- a/tests/Integration/ResourceStatusSystem/Test_ResourceManagement.py +++ b/tests/Integration/ResourceStatusSystem/Test_ResourceManagement.py @@ -223,49 +223,6 @@ def test_PolicyResult(rmClient): assert not res["Value"], res["Value"] -def test_SpaceTokenOccupancy(rmClient): - """ - SpaceTokenOccupancy table - """ - - res = rmClient.deleteSpaceTokenOccupancyCache("endpoint", "token") # just making sure it's not there (yet) - assert res["OK"] is True, res["Message"] - - # TEST addOrModifySpaceTokenOccupancy - res = rmClient.addOrModifySpaceTokenOccupancyCache( - "endpoint", "token", 500.0, 1000.0, 200.0, datetime.datetime.now() - ) - assert res["OK"] is True, res["Message"] - - res = rmClient.selectSpaceTokenOccupancyCache("endpoint", "token") - assert res["OK"] is True, res["Message"] - # check if the name that we got is equal to the previously added 'token' - assert res["Value"][0][1] == "token" - - res = rmClient.addOrModifySpaceTokenOccupancyCache("endpoint", "token", free=100.0) - assert res["OK"] is True, res["Message"] - - res = rmClient.selectSpaceTokenOccupancyCache("endpoint", "token") - # check if the result has changed - assert res["Value"][0][3] == 100.0 - - res = rmClient.addOrModifySpaceTokenOccupancyCache("endpoint", "token", free=0.0) - assert res["OK"] is True, res["Message"] - - res = rmClient.selectSpaceTokenOccupancyCache("endpoint", "token") - # check if the result has changed - assert res["Value"][0][3] == 0.0 - - # TEST deleteSpaceTokenOccupancy - # ............................................................................... - res = rmClient.deleteSpaceTokenOccupancyCache("endpoint", "token") - assert res["OK"] is True, res["Message"] - - res = rmClient.selectSpaceTokenOccupancyCache("endpoint", "token") - assert res["OK"] is True, res["Message"] - assert not res["Value"], res["Value"] - - def test_Transfer(rmClient): """ TransferOccupancy table