Data Table shows annotation code

Need to optimize and clean

Author: Francisco Arrieta

odmtools/controller/frmDataTable.py

@@ -13,17 +13,14 @@ def __init__(self, parent, **kwargs):
         self.memDB = None
         DataTable.__init__(self, parent, **kwargs)
 
+    def init(self, memDB):
+        self.memDB = memDB
+        self.olvDataTable.init(self.memDB)
 
-    def init_publishers(self):
         Publisher.subscribe(self.onChangeSelection, "changeTableSelection")
         Publisher.subscribe(self.onRefresh, "refreshTable")
         Publisher.subscribe(self.olvDataTable.onDeselectAll, "deselectAllDataTable")
 
-    def init(self, memDB):
-        self.memDB = memDB
-        self.olvDataTable.init(self.memDB)
-        self.init_publishers()
-
     def onItemSelected(self, event):
         pass
 

odmtools/controller/olvDataTable.py

@@ -20,6 +20,8 @@ def __init__(self, parent, **kwargs):
         self.sortedColumnIndex = -1
         self.currentItem = None
         self.dataframe = None
+        self.annotations = None
+        self.data = None
 
     def init(self, memDB):
         self.memDB = memDB
@@ -28,19 +30,54 @@ def init(self, memDB):
         self.oddRowsBackColor = wx.Colour(191, 217, 217)
 
         self.dataframe = self.memDB.getDataValuesDF()
-        sort_by_index = list(self.dataframe.columns).index("valuedatetime")
-        self.dataframe.sort_values(self.dataframe.columns[sort_by_index], inplace=True)
-        self.dataObjects = self.dataframe.values.tolist()
+        self.annotations = self.memDB.get_annotations()
+        # sort_by_index = list(self.dataframe.columns).index("valuedatetime")
+        # columns = self.memDB.get_columns_with_annotations()
+        # self.dataframe.sort_values(self.dataframe.columns[sort_by_index], inplace=True)
+        # self.dataObjects = self.dataframe.values.tolist()
+        self.dataObjects = self.__merge_dataframe_with_annotations()
+        col = self.memDB.get_columns_with_annotations()
+
+        # columns = \
+        #     [ColumnDefn(x.strip(), align="left", valueGetter=i, minimumWidth=125, width=125,
+        #                       stringConverter='%Y-%m-%d %H:%M:%S' if "valuedatetime" == x.lower() else '%s')
+        #            for x, i in self.memDB.getEditColumns()]
         columns = \
             [ColumnDefn(x.strip(), align="left", valueGetter=i, minimumWidth=125, width=125,
-                              stringConverter='%Y-%m-%d %H:%M:%S' if "valuedatetime" == x.lower() else '%s')
-                   for x, i in self.memDB.getEditColumns()]
+                        stringConverter='%Y-%m-%d %H:%M:%S' if "valuedatetime" == x.lower() else '%s')
+             for x, i in col]
+
         self.SetColumns(columns)
 
 
         self.SetObjectGetter(self.ObjectGetter)
         self.SetItemCount(len(self.dataframe))
 
+    def __merge_dataframe_with_annotations(self):
+        sort_by_index = self.dataframe.columns.tolist().index("valuedatetime")
+        self.dataframe.sort_values(self.dataframe.columns[sort_by_index], inplace=True)
+
+        data_list = self.dataframe.values.tolist()
+        anno_list = self.annotations.values.tolist()
+        data = data_list
+
+        anno = {}
+        for i in range(0, len(anno_list)):
+            value_id = anno_list[i][1]
+            annotation_code = anno_list[i][-1]
+            if value_id in anno:
+                anno[value_id].append(annotation_code)
+            else:
+                anno[value_id] = [annotation_code]
+
+        for key, value in anno.iteritems():
+            for i in range(0, len(data_list)):
+                if key in data_list[i]:
+                    data_list[i].append(value)
+                    break
+
+        return data
+
     def EnableSorting(self):
         self.Bind(wx.EVT_LIST_COL_CLICK, self.onColSelected)
         if not self.smallImageList:

odmtools/odmdata/memory_database.py

@@ -25,6 +25,7 @@ def __init__(self, taskserver=None):
         self.df = None
         # Series_Service handles remote database
         self.series_service = None
+        self.results_annotations = None
 
         # Memory_service handles in memory database
         sm = ServiceManager()
@@ -77,6 +78,16 @@ def getDataValuesDF(self):
         logging.debug("done updating memory dataframe")
         return self.df
 
+    def get_annotations(self, query_db_again=False):
+        # self.mem_service._session.commit()
+        # setSchema(self.mem_service._session_factory.engine)
+        if self.results_annotations is None or query_db_again:
+            result_id = self.df.resultid[0]
+            annotation = self.series_service.get_annotations_by_result(resultid=result_id)
+            self.results_annotations = annotation
+
+        return self.results_annotations
+
     def getDataValues(self):
         # TODO: fix me! this commit location is only temoporarily. should be flushing so that we can restore
         self.mem_service._session.commit()
@@ -94,6 +105,27 @@ def getEditColumns(self):
         return [(x, i) for (i, x) in enumerate(columns)]
         # return [(x, i) for (i, x) in enumerate(self.df.columns)]
 
+    def get_columns_with_annotations(self):
+        """
+        If results_annotations has not been set then
+        :return:
+        """
+
+        if self.results_annotations is None or self.df is None:
+            print "self.df and self.results_annotations must be a pandas dataframe. Currently they are None"
+            return []
+
+        columns = []
+        columns.extend(self.df.columns.tolist())
+
+        annotation_columns = self.results_annotations.columns.tolist()
+        index = annotation_columns.index("annotationcode")
+        annotation_code_column = annotation_columns[index]
+
+        columns.append(annotation_code_column)
+
+        return [(x, i) for (i, x) in enumerate(columns)]
+
     def getDataValuesforGraph(self, seriesID, noDataValue, startDate=None, endDate=None):
         return self.series_service.get_plot_values(seriesID, noDataValue, startDate, endDate)
 
@@ -250,7 +282,7 @@ def initEditValues(self, seriesID):
             logger.debug("Load series from db")
 
             self.series = self.series_service.get_series(seriesID)
-            self.df = self.series_service.get_values(series_id= seriesID)
+            self.df = self.series_service.get_values(series_id=seriesID)
 
             self.editLoaded = True
 

odmtools/odmservices/series_service.py

@@ -116,26 +116,6 @@ def get_variables_by_site_code(self, site_code):
         q = self._session.query(Variables).filter(Variables.VariableID.in_(var_ids))
         return q.all()
 
-        # Data Value Methods
-    def get_values(self, series_id=None):
-        '''
-
-        :param series_id:  Series id
-        :return: pandas dataframe
-        '''
-
-        setSchema(self._session_factory.engine)
-        q = self.read._session.query(TimeSeriesResultValues)
-        if series_id:
-            q = q.filter_by(ResultID=series_id)
-        q = q.order_by(TimeSeriesResultValues.ValueDateTime)
-        query = q.statement.compile(dialect=self._session_factory.engine.dialect)
-        data = pd.read_sql_query(sql=query,
-                                 con=self._session_factory.engine,
-                                 params=query.params)
-        data.set_index(data['valuedatetime'], inplace=True)
-        return data
-
 # Series Catalog methods
     def get_series_by_site(self , site_id):
         # try:
@@ -359,42 +339,55 @@ def get_series_from_filter(self):
         # Pass in probably a Series object, match it against the database
         pass
 
-#
+
     #Data Value Methods
     def get_values(self, series_id=None):
-        '''
-        :param series_id:  Series id
+        """
+        :param series_id:
         :return: pandas dataframe
-        '''
-        #series= self.get_series_by_id(series_id)
-        # if series:
-        #     q = self._edit_session.query(DataValue).filter_by(
-        #             site_id=series.site_id,
-        #             variable_id=series.variable_id,
-        #             method_id=series.method_id,
-        #             source_id=series.source_id,
-        #             quality_control_level_id=series.quality_control_level_id)
-        #
-        #     query=q.statement.compile(dialect=self._session_factory.engine.dialect)
-        #     data= pd.read_sql_query(sql= query,
-        #                       con = self._session_factory.engine,
-        #                       params = query.params )
-        #     #return data.set_index(data['LocalDateTime'])
-        #     return data
-        # else:
-        #     return None
+        """
+        # see get_annotations_by_result around line 850
 
         q = self.read._session.query(TimeSeriesResultValues)
         if series_id:
-            q=q.filter_by(ResultID=series_id)
-        q= q.order_by(TimeSeriesResultValues.ValueDateTime)
+            q = q.filter_by(ResultID=series_id)
+        q = q.order_by(TimeSeriesResultValues.ValueDateTime)
         query = q.statement.compile(dialect=self._session_factory.engine.dialect)
         data = pd.read_sql_query(sql=query,
                                  con=self._session_factory.engine,
                                  params=query.params)
         data.set_index(data['valuedatetime'], inplace=True)
+
+        # if series_id:
+        #     anno = self.get_annotations_by_result(series_id)
+        # q = pd.merge(data, anno, how="left", on='valueid', indicator=False)
+        # data.applymap(self.merge_annotation_with_timeseries_result)
+
+        # if len(anno):
+        #     # data.valueid.apply(self.merge_annotation_with_timeseries_result)
+        #     # data.applymap(self.merge_annotation_with_timeseries_result)
+        #     self.merge_annotation_with_timeseries_result(data, anno)
+
+
+
         return data
 
+    # df.drop_duplicates(dup_cols, keep='last', inplace=True)
+    # newdf = pd.merge(df, pd.read_sql(query, engine), how='left', on=dup_cols, indicator=True)
+    # newdf = newdf[newdf['_merge'] == 'left_only']
+    # newdf.drop(['_merge'], axis=1, inplace=True)
+    # return df[df['valuedatetime'].isin(newdf['valuedatetime'])]
+
+    # def merge_annotation_with_timeseries_result(self, x):
+    #     print x
+    #
+    #     # return x
+    #     return 123
+    def merge_annotation_with_timeseries_result(self, data, anno):
+
+        print 123
+        return
+
     def get_all_values_df(self):
 
         """
@@ -877,17 +870,20 @@ def get_all_annotations(self):
         return self.read.getAnnotations(type=None)
 
     def get_annotations_by_result(self, resultid):
-        setSchema(self._session_factory.engine)
+        # setSchema(self._session_factory.engine)
 
         # ids = [x[0] for x in self.read._session.query(TimeSeriesResultValues.ValueID)\
         #         .filter(TimeSeriesResultValues.ResultID == resultid).all()]
         # q = self.read._session.query(TimeSeriesResultValueAnnotations)\
         #     .filter(TimeSeriesResultValueAnnotations.ValueID.in_(ids)).all()
 
-        q =self.read._session.query(TimeSeriesResultValueAnnotations.AnnotationID, TimeSeriesResultValueAnnotations.ValueID,
-                                    TimeSeriesResultValues.ResultID, TimeSeriesResultValues.ValueDateTime)\
+        resultid = int(resultid)
+
+        q = self.read._session.query(TimeSeriesResultValueAnnotations.AnnotationID, TimeSeriesResultValueAnnotations.ValueID,
+                                    TimeSeriesResultValues.ResultID, TimeSeriesResultValues.ValueDateTime, Annotations.AnnotationCode)\
                             .filter(TimeSeriesResultValues.ResultID == resultid)\
-                            .filter(TimeSeriesResultValueAnnotations.ValueID == TimeSeriesResultValues.ValueID)
+                            .filter(TimeSeriesResultValueAnnotations.ValueID == TimeSeriesResultValues.ValueID)\
+                            .filter(Annotations.AnnotationID==TimeSeriesResultValueAnnotations.AnnotationID)
 
         query = q.statement.compile(dialect=self._session_factory.engine.dialect)
         data = pd.read_sql_query(sql=query, con=self._session_factory.engine,