improve editing functions - update values, upsert values

Author: stephanie

odmtools/controller/logicEditTools.py

@@ -222,7 +222,7 @@ def restore(self):
             self._script("edit_service.restore()\n", 'black')
             Publisher.sendMessage("scroll")
 
-    def saveFactory(self, var=None, method=None, qcl=None):
+    def saveFactory(self, var=None, method=None, qcl=None, action= None, actionby= None):
         """
 
         :param var:
@@ -236,7 +236,7 @@ def saveFactory(self, var=None, method=None, qcl=None):
         values['method'] = ("new_method" if method else None)
         values['qcl'] = ("new_qcl" if qcl else None)
         #values['override'] = override
-        return values['var'], values['method'], values['qcl']#, values['isSave']
+        return values['var'], values['method'], values['qcl'], action, actionby#, values['isSave']
 
 
     def save(self, var=None, method=None, qcl=None):
@@ -257,7 +257,7 @@ def save(self, var=None, method=None, qcl=None):
             Publisher.sendMessage("scroll")
         return result
 
-    def save_as(self, var=None, method=None, qcl=None):
+    def save_as(self, var=None, method=None, qcl=None, action = None, actionby = None):
         """
 
         :param var:
@@ -266,16 +266,16 @@ def save_as(self, var=None, method=None, qcl=None):
         :param override:
         :return:
         """
-        result = self._edit_service.save_as(var=var, method=method, qcl=qcl)
+        result = self._edit_service.save_as(var=var, method=method, qcl=qcl, action = action, actionby = actionby)
         if self._record:
             self._script(
-                "edit_service.save_as(%s, %s, %s)\n" % (self.saveFactory(var, method, qcl)),
+                "edit_service.save_as(%s, %s, %s, %s, %s)\n" % (self.saveFactory(var, method, qcl, action, actionby)),
                 'black')
             #self._script("edit_service.save(%s, %s, %s, saveAs=%s)\n" % (var, method, qcl, isSave), 'black')
             Publisher.sendMessage("scroll")
         return result
 
-    def save_appending(self, var = None, method =None, qcl = None, overwrite = False):
+    def save_appending(self, result = None, overwrite = False):
         """
 
         :param var:
@@ -284,14 +284,14 @@ def save_appending(self, var = None, method =None, qcl = None, overwrite = False
         :param override:
         :return:
         """
-        result = self._edit_service.save_appending(var=var, method=method, qcl=qcl, overwrite= overwrite)
+        result = self._edit_service.save_appending(result= result,  overwrite= overwrite)
         if result:
             print "Save worked!"
 
             if self._record:
 
                 self._script(
-                    "edit_service.save_appending(%s, %s, %s, " % self.saveFactory(var, method, qcl)+str(overwrite )+")\n",
+                    "edit_service.save_appending(%s, %s)\n" % (result, str(overwrite )),
                     'black')
                 #self._script("edit_service.save(%s, %s, %s, saveAs=%s)\n" % (var, method, qcl, isSave), 'black')
                 Publisher.sendMessage("scroll")
@@ -302,7 +302,7 @@ def save_appending(self, var = None, method =None, qcl = None, overwrite = False
         return result
 
 
-    def save_existing(self, var=None, method=None, qcl=None):
+    def save_existing(self, result=None):
         """
 
         :param var:
@@ -311,14 +311,14 @@ def save_existing(self, var=None, method=None, qcl=None):
         :param override:
         :return:
         """
-        result = self._edit_service.save_existing(var=var, method=method, qcl=qcl)
+        result = self._edit_service.save_existing(result = result)
         if result:
             print "Save worked!"
 
             if self._record:
 
                 self._script(
-                    "edit_service.save_existing(%s, %s, %s)\n" % (self.saveFactory(var, method, qcl)),
+                    "edit_service.save_existing(%s)\n" % (result),
                     'black')
                 #self._script("edit_service.save(%s, %s, %s, saveAs=%s)\n" % (var, method, qcl, isSave), 'black')
                 Publisher.sendMessage("scroll")

odmtools/controller/pageExisting.py

@@ -70,8 +70,8 @@ def enableButtons(self, isEnabled):
 
     def get_selected_series(self):
         selectedObject = self.pnlExisting.olvSeriesList.GetSelectedObject()
-        result = self.series_service.get_series(selectedObject.ResultID)
-        return result.FeatureActionObj.ActionObj.MethodObj, result.ProcessingLevelObj, result.VariableObj
+        result = selectedObject.ResultObj#self.series_service.get_series(selectedObject.ResultID)
+        return result.FeatureActionObj.ActionObj.MethodObj, result.ProcessingLevelObj, result.VariableObj, result
 
     def initTable(self, dbservice, site_id):
         """Set up columns and objects to be used in the objectlistview to be visible in the series_service selector"""

odmtools/gui/wizSave.py

@@ -262,7 +262,7 @@ def get_metadata(self):
 
         elif self.pgIntro.pnlIntroduction.rbSaveExisting.GetValue():
             # selected an existing series
-            method, processing_level, variable = self.pgExisting.get_selected_series()
+            method, processing_level, variable, result = self.pgExisting.get_selected_series()
 
         # Create action
         action = Actions()
@@ -460,25 +460,27 @@ def on_wizard_finished(self, event):
             action_by.AffiliationObj = affiliation
 
             # result = self.series_service.getResult(var=variable, meth=method, proc=proc_level, action=action, actionby=action_by)
-            result = self.record_service._edit_service.getResult(var=variable, meth=method, proc=proc_level, action=action, actionby=action_by)
+            result = self.pgExisting.pnlExisting.olvSeriesList.GetSelectedObject().ResultObj
+
+            #result = self.record_service._edit_service.getResult(var=variable, meth=method, proc=proc_level, action=action, actionby=action_by)
 
             try:
                 if rbSave:
                     result = self.record_service.save()
                 elif rbSaveAsNew:
-                    result = self.record_service.saveAs(variable=variable, method=method, proc_level=proc_level,
+                    result = self.record_service.save_as(variable=variable, method=method, proc_level=proc_level,
                                                         action=action, action_by=action_by)
                 elif rbSaveAsExisting:
                     if overwrite:
-                        result = self.record_service.saveExisting(result=result)
+                        result = self.record_service.save_existing(result=result)
                     elif append:
                         #TODO send in just the result
                         #def save_appending(self, var = None, method =None, qcl = None, overwrite = False):
                         #TODO if i require that original or new is selected I can call once with overwrite = original
                         if original:
-                            result = self.record_service.saveAppend(result=result, overwrite=False)
+                            result = self.record_service.save_appending(result=result, overwrite=False)
                         elif new:
-                            result = self.record_service.saveAppend(result=result, overwrite=True)
+                            result = self.record_service.save_appending(result=result, overwrite=True)
 
                 Publisher.sendMessage("refreshSeries")
 

odmtools/odmservices/edit_service.py

@@ -483,20 +483,24 @@ def restore(self):
         self.reset_filter()
 
     def save(self,  result=None):
-        values = self.memDB.df
+        values = self.memDB.getDataValuesDF()
+
         if not result:
             result = self.memDB.series_service.get_series(series_id = values['resultid'][0])
+        else:
+            values["resultid"] = result.ResultID
+
         # update result
         result.ValueCount = 0
         self.updateResult(result)
         # upsert values
-        self.memDB.series_service.upsert(values)
+        self.memDB.series_service.upsert_values(values)
         # save new annotations
         self.add_annotations(self.memDB.annotation_list)
         return result
 
-    def saveAppend(self,  result, overwrite=True):
-        values = self.memDB.df
+    def save_appending(self, result, overwrite=True):
+        values = self.memDB.getDataValuesDF()
 
         # get save result
 
@@ -517,22 +521,21 @@ def saveAppend(self,  result, overwrite=True):
         return result
 
 
-    def saveExisting(self, result):
-        values = self.memdB.df
+    def save_existing(self, result):
+        #values = self.memDB.getDataValuesDF()
         # get save result
 
         # set in df
-        values["resultid"]=result.ResultID
+        #values["resultid"]=result.ResultID
 
         # save(values)
         self.save(result)
-        #self.memDB.series_service.upsert_values(values)
 
         return result
 
-    def saveAs(self, variable, method, proc_level, action, action_by):
+    def save_as(self, variable, method, proc_level, action, action_by):
         #save as new series
-        values = self.memDB.df
+        values = self.memDB.getDataValuesDF()
         # get all annotations for series
         annolist= self.memDB.series_service.get_annotations_by_result(values["resultid"][0])
         annolist['valueid']
@@ -555,9 +558,9 @@ def saveAs(self, variable, method, proc_level, action, action_by):
         return result
 
     def getResult(self, var, meth, proc, action, actionby):
-
+        values = self.memDB.getDataValuesDF()
         # copy old
-        result = self.memDB.series_service.get_series(self.memDB.df["resultid"][0])
+        result = self.memDB.series_service.get_series(values["resultid"][0])
         sampling_feature = result.FeatureActionObj.SamplingFeatureObj
         # self.memDB.series_service._session.expunge(result)
 
@@ -595,7 +598,7 @@ def getResult(self, var, meth, proc, action, actionby):
     def updateResult(self, Result):
         form = "%Y-%m-%d %H:%M:%S"
         # get pd
-        values = self.memDB.df
+        values = self.memDB.getDataValuesDF()
 
         # update count, dates,
         Action = Result.FeatureActionObj.ActionObj

odmtools/odmservices/series_service.py

@@ -3,7 +3,7 @@
 from odm2api.ODM2.services import ReadODM2,  UpdateODM2, DeleteODM2, CreateODM2
 from odm2api import serviceBase
 from odm2api.ODM2.models import *
-from odmtools.odmservices.to_sql_newrows import clean_df_db_dups, delete, update
+from odmtools.odmservices.to_sql_newrows import get_insert, get_delete, get_update
 import datetime
 from odmtools.common.logger import LoggerTool
 import pandas as pd
@@ -624,47 +624,69 @@ def get_current_time_and_utcoffset(self):
     #     return True
 
 
-    def update_values(self, updates):
-        '''
-        updates : time series result values, pandas dataframe
-        '''
-        setSchema(self.mem_service._session_factory.engine)
 
-        stmt = (TimeSeriesResultValues.__table__.update().
-                where(TimeSeriesResultValues.ValueDateTime == bindparam('id')).
-                values(datavalue=bindparam('value'))
-                )
 
-        self.create._session.execute(stmt, updates["datavalue"].to_dict(orient='dict'))
-        #self.mem_service._session.query(TSRV).filter_by
 
-        # self.updateDF()
 
     def insert_annotations(self, annotations):
         annotations.to_sql(name="timeseriesresultvalueannotations", if_exists='append', con=self._session_factory.engine, index=False)
 
+    def _get_df_query(self, values):
+
+        resid = str(values['resultid'][0])
+        sd = values['valuedatetime'].min()
+        ed = values['valuedatetime'].max()
+        q = self.read._session.query(TimeSeriesResultValues).\
+            filter(TimeSeriesResultValues.ResultID == resid)#.\
+            #filter(TimeSeriesResultValues.ValueDateTime.between(sd, ed))
+        return q.statement.compile(dialect=self._session_factory.engine.dialect)
+
 
     def upsert_values(self, values):
         setSchema(self._session_factory.engine)
-        newvals= clean_df_db_dups(df = values, dup_cols = ["valuedatetime", "resultid"], tablename="timeseriesresultvalues", engine = self._session_factory.engine,
-                                         filter_continuous_col="valuedatetime", filter_categorical_col="resultid")
-        self.insert_values(newvals)
-        delvals = delete(df = values, dup_cols = ["valuedatetime", "resultid"], tablename="timeseriesresultvalues", engine = self._session_factory.engine,
-                        filter_continuous_col = "valuedatetime", filter_categorical_col = "resultid")
-        self.delete_dvs(delvals["valuedatetime"])
+        query = self._get_df_query(values)
+        newvals= get_insert(df = values, query = query, dup_cols = ["valuedatetime", "resultid"], engine = self._session_factory.engine)
+        if not newvals.empty:
+            self.insert_values(newvals)
+        delvals = get_delete(df = values, query = query, dup_cols = ["valuedatetime", "resultid"], engine = self._session_factory.engine)
+        if not delvals.empty:
+            self.delete_dvs(delvals["valuedatetime"].tolist())
 
-        upvals = update(df = values,dup_cols = ["valuedatetime", "resultid"], tablename="timeseriesresultvalues", engine = self._session_factory.engine,
-                               filter_continuous_col="valuedatetime", filter_categorical_col="resultid")
-        self.update(upvals)
+        upvals = get_update(df = values, query = query, dup_cols = ["valuedatetime", "resultid"], engine = self._session_factory.engine)
+        if not upvals.empty:
+            self.update_values(upvals)
+
+        self._session.commit()
 
     def insert_values(self, values):
         """
 
         :param values: pandas dataframe
         :return:
         """
-        values.to_sql(name="timeseriesresultvalues", if_exists='append', con=self._session_factory.engine, index=False)
-#
+        values.to_sql(name="TimeSeriesResultValues",
+                      schema=TimeSeriesResultValues.__table_args__['schema'],
+                      if_exists='append',
+                      chunksize=1000,
+                      con=self._session_factory.engine,
+                      index=False)
+
+
+    def update_values(self, updates):
+        '''
+        updates : time series result values, pandas dataframe
+        '''
+        setSchema(self._session_factory.engine)
+
+        stmt = (TimeSeriesResultValues.__table__.update().
+                where(TimeSeriesResultValues.ValueDateTime == bindparam('id')).
+                values(datavalue=bindparam('value'))
+                )
+        update_list = [{"value": row["datavalue"], "id": index.to_pydatetime()} for index, row in updates.iterrows()]
+        # update_list = {'value':updates["datavalue"].tolist(), 'id':updates.index.to_pydatetime().tolist()}
+        vals = self.create._session.execute(stmt, update_list)
+
+        
 #     def create_new_series(self, data_values, site_id, variable_id, method_id, source_id, qcl_id):
 #         """
 #