Add API to create scopes

This commit adds an API enabling the POST operation to create scopes in
an ad hoc fashion. This is useful for operators to register scopes
before they are created as resources in the collected backend so their
processing can be discarded right away, for example for trial
projects/accounts.

Otherwise, we need to wait for them to create resources, then for
example Ceilometer has to monitor these resources, persist measures in
Gnocchi, then CloudKitty has to discover the scopes and finally we can
disable their processing.

Change-Id: I3e947d36c9d5d5da07115d35dde578ae300cbe5c

Author: rafaelweingartner

cloudkitty/api/v2/scope/state.py

@@ -183,7 +183,10 @@ def put(self,
         voluptuous.Required('scope_key'): vutils.get_string_type(),
         voluptuous.Required('fetcher'): vutils.get_string_type(),
         voluptuous.Required('collector'): vutils.get_string_type(),
+        # This "state" property should be removed in the next release.
         voluptuous.Required('state'): vutils.get_string_type(),
+        voluptuous.Optional('last_processed_timestamp'):
+            voluptuous.Coerce(tzutils.dt_from_iso),
         voluptuous.Required('active'): bool,
         voluptuous.Required('scope_activation_toggle_date'):
             vutils.get_string_type()
@@ -226,6 +229,79 @@ def patch(self, scope_id, scope_key=None, fetcher=None,
             'fetcher': update_storage_scope.fetcher,
             'collector': update_storage_scope.collector,
             'state': update_storage_scope.state.isoformat(),
+            'last_processed_timestamp':
+                update_storage_scope.last_processed_timestamp.isoformat(),
+            'active': update_storage_scope.active,
+            'scope_activation_toggle_date':
+                update_storage_scope.scope_activation_toggle_date.isoformat()
+        }
+
+    @api_utils.add_input_schema('body', {
+        voluptuous.Required('scope_id'):
+            api_utils.SingleQueryParam(str),
+        voluptuous.Optional('scope_key'):
+            api_utils.SingleQueryParam(str),
+        voluptuous.Optional('fetcher'):
+            api_utils.SingleQueryParam(str),
+        voluptuous.Optional('collector'):
+            api_utils.SingleQueryParam(str),
+        voluptuous.Optional('active'):
+            api_utils.SingleQueryParam(bool),
+    })
+    @api_utils.add_output_schema({
+        voluptuous.Required('scope_id'): vutils.get_string_type(),
+        voluptuous.Required('scope_key'): vutils.get_string_type(),
+        voluptuous.Required('fetcher'): vutils.get_string_type(),
+        voluptuous.Required('collector'): vutils.get_string_type(),
+        # This "state" property should be removed in the next release.
+        voluptuous.Required('state'): vutils.get_string_type(),
+        voluptuous.Optional('last_processed_timestamp'):
+            voluptuous.Coerce(tzutils.dt_from_iso),
+        voluptuous.Required('active'): bool,
+        voluptuous.Required('scope_activation_toggle_date'):
+            vutils.get_string_type()
+    })
+    def post(self, scope_id, scope_key=None, fetcher=None, collector=None,
+             active=None):
+
+        policy.authorize(
+            flask.request.context,
+            'scope:post_state',
+            {'tenant_id': scope_id or flask.request.context.project_id}
+        )
+        results = self._storage_state.get_all(identifier=scope_id)
+
+        if len(results) >= 1:
+            LOG.debug("There is already a scope with ID [%s], "
+                      "scopes found: [%s].", scope_id, results)
+            raise http_exceptions.NotFound("Cannot create a scope with an "
+                                           "already existing scope_id: %s."
+                                           % scope_id)
+
+        LOG.debug("Creating storage scope with data: [scope_id=%s, "
+                  "scope_key=%s, fetcher=%s, collector=%s, active=%s].",
+                  scope_id, scope_key, fetcher, collector, active)
+
+        self._storage_state.create_scope(scope_id, None, fetcher=fetcher,
+                                         collector=collector,
+                                         scope_key=scope_key, active=active)
+
+        storage_scopes = self._storage_state.get_all(
+            identifier=scope_id)
+
+        update_storage_scope = storage_scopes[0]
+        last_processed_timestamp = None
+        if update_storage_scope.last_processed_timestamp.isoformat():
+            last_processed_timestamp =\
+                update_storage_scope.last_processed_timestamp.isoformat()
+
+        return {
+            'scope_id': update_storage_scope.identifier,
+            'scope_key': update_storage_scope.scope_key,
+            'fetcher': update_storage_scope.fetcher,
+            'collector': update_storage_scope.collector,
+            'state': last_processed_timestamp,
+            'last_processed_timestamp': last_processed_timestamp,
             'active': update_storage_scope.active,
             'scope_activation_toggle_date':
                 update_storage_scope.scope_activation_toggle_date.isoformat()

cloudkitty/common/policies/v2/scope.py

@@ -36,7 +36,12 @@
         description='Enables operators to patch a storage scope',
         operations=[{'path': '/v2/scope',
                      'method': 'PATCH'}]),
-
+    policy.DocumentedRuleDefault(
+        name='scope:post_state',
+        check_str=base.ROLE_ADMIN,
+        description='Enables operators to create a storage scope',
+        operations=[{'path': '/v2/scope',
+                     'method': 'POST'}]),
 ]
 
 

cloudkitty/storage_state/__init__.py

@@ -168,6 +168,8 @@ def set_last_processed_timestamp(
             collector=None, scope_key=None):
         """Set the last processed timestamp of a scope.
 
+        If the scope does not exist yet in the database, it will create it.
+
         :param identifier: Identifier of the scope
         :type identifier: str
         :param last_processed_timestamp: last processed timestamp of the scope
@@ -191,18 +193,52 @@ def set_last_processed_timestamp(
                 r.last_processed_timestamp = last_processed_timestamp
                 session.commit()
         else:
-            state_object = self.model(
-                identifier=identifier,
-                last_processed_timestamp=last_processed_timestamp,
-                fetcher=fetcher,
-                collector=collector,
-                scope_key=scope_key,
-            )
-            session.add(state_object)
-            session.commit()
-
+            self.create_scope(identifier, last_processed_timestamp,
+                              fetcher=fetcher, collector=collector,
+                              scope_key=scope_key)
         session.close()
 
+    def create_scope(self, identifier, last_processed_timestamp, fetcher=None,
+                     collector=None, scope_key=None, active=True,
+                     session=None):
+        """Creates a scope in the database.
+
+        :param identifier: Identifier of the scope
+        :type identifier: str
+        :param last_processed_timestamp: last processed timestamp of the scope
+        :type last_processed_timestamp: datetime.datetime
+        :param fetcher: Fetcher associated to the scope
+        :type fetcher: str
+        :param collector: Collector associated to the scope
+        :type collector: str
+        :param scope_key: scope_key associated to the scope
+        :type scope_key: str
+        :param active: indicates if the scope is active
+        :type active: bool
+        :param session: the current database session to be reused
+        :type session: object
+        """
+
+        is_session_reused = True
+        if not session:
+            session = db.get_session()
+            session.begin()
+            is_session_reused = False
+
+        state_object = self.model(
+            identifier=identifier,
+            last_processed_timestamp=last_processed_timestamp,
+            fetcher=fetcher,
+            collector=collector,
+            scope_key=scope_key,
+            active=active
+        )
+        session.add(state_object)
+        session.commit()
+
+        if not is_session_reused:
+            session.close()
+
     def get_state(self, identifier,
                   fetcher=None, collector=None, scope_key=None):
         LOG.warning("The method 'get_state' is deprecated."

doc/source/_static/cloudkitty.policy.yaml.sample

@@ -101,6 +101,10 @@
 # PATCH  /v2/scope
 #"scope:patch_state": "role:admin"
 
+# Enables operators to create a storage scope
+# POST  /v2/scope
+#"scope:post_state": "role:admin"
+
 # Get a rating summary
 # GET  /v2/summary
 #"summary:get_summary": "rule:admin_or_owner"

doc/source/api-reference/v2/scope/scope.inc

@@ -127,6 +127,56 @@ Response
    - active: active_key_resp
 
 
+Response Example
+----------------
+
+.. literalinclude:: ./api_samples/scope/scope_get.json
+   :language: javascript
+
+Create a scope
+================================
+
+Create a scope.
+
+.. rest_method::  POST /v2/scope
+
+.. rest_parameters:: scope/scope_parameters.yml
+
+   - collector: collector
+   - fetcher: fetcher
+   - scope_id: scope_id
+   - scope_key: scope_key
+   - active: active_body
+
+Status codes
+------------
+
+.. rest_status_code:: success http_status.yml
+
+   - 200
+
+.. rest_status_code:: error http_status.yml
+
+   - 400
+   - 403
+   - 404
+   - 405
+
+Response
+--------
+
+.. rest_parameters:: scope/scope_parameters.yml
+
+   - scope_id: scope_id_resp
+   - scope_key: scope_key_resp
+   - fetcher: fetcher_resp
+   - collector: collector_resp
+   - state: state
+   - last_processed_timestamp: last_processed_timestamp
+   - active: active_key_resp
+   - scope_activation_toggle_date: scope_activation_toggle_date
+
+
 Response Example
 ----------------
 

doc/source/api-reference/v2/scope/scope_parameters.yml

@@ -88,6 +88,14 @@ last_processed_timestamp:
   type: iso8601 timestamp
   required: true
 
+scope_activation_toggle_date:
+  in: body
+  description: |
+    It represents the last time the scope was activated/deactivated via the
+    PATCH API.
+  type: iso8601 timestamp
+  required: true
+
 scope_id_body:
   <<: *scope_id
   in: body

releasenotes/notes/post-api-create-scope-739098144706a1cf.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Introduce an API to create scopes with a POST request. This is useful for
+    operators to register scopes before they are created as resources in the
+    collected backend and disable their processing without waiting for the
+    scopes to be discovered by CloudKitty.