begin adding code for filling gaps

Author: Stephanie Reeder

ODMTools.py

@@ -20,7 +20,7 @@
 
 import pyodbc
 import pymysql
-import psycopg2
+# import psycopg2
 
 tool = LoggerTool()
 logger = tool.setupLogger('main',  'odmtools.log', 'a', logging.INFO)

odmtools/controller/logicEditTools.py

@@ -50,14 +50,20 @@ def filter_date(self, endDate, startDate):
         else:
             return "Cannot filter: %s" % (self._edit_error)
 
+    def fill_gap(self, value, time_period):
+        self._edit_service.fill_gap(value, time_period)
+        self.refresh_selection()
+        if self._record:
+            self._script("edit_service.fill_gap(%s, '%s')\n" % (value, time_period), 'black')
+            Publisher.sendMessage("scroll")
+
     def data_gaps(self, value, time_period):
         self._edit_service.data_gaps(value, time_period)
         self.refresh_selection()
         if self._record:
             self._script("edit_service.data_gaps(%s, '%s')\n" % (value, time_period), 'black')
             Publisher.sendMessage("scroll")
 
-
     def value_change_threshold(self, value, operator):
         self._edit_service.change_value_threshold(value, operator)
         self.refresh_selection()

odmtools/gui/mnuRibbon.py

@@ -36,9 +36,9 @@
  wxID_RIBBONEDITSCRIPTSAVE, wxID_RIBBONVIEWPLOT, wxID_RIBBONVIEWTABLE, wxID_RIBBONVIEWSERIES, wxID_RIBBONVIEWCONSOLE,
  wxID_RIBBONVIEWSCRIPT, wxID_RIBBONPLOTBLANKBTN, wxID_FileMenu, wxID_STARTDPDATE, wxID_ENDDPDATE, wxID_FRAME1SPINCTRL1,
  wxID_RIBBONEDITFILTER, wxID_RIBBONEDITRECORD, wxID_RIBBONEDITLINFILTER, wxID_RIBBONPLOTDATEAPPLY,
- wxID_RIBBONEDITRESETFILTER, wxID_RIBBONRECORDNEW, wxID_RIBBONRECORDOPEN, wxID_RIBBONRECORDSAVE] = [wx.NewId() for
+ wxID_RIBBONEDITRESETFILTER, wxID_RIBBONRECORDNEW, wxID_RIBBONRECORDOPEN, wxID_RIBBONRECORDSAVE, wxID_GAPFILL] = [wx.NewId() for
                                                                                                       _init_ctrls in
-                                                                                                      range(46)]
+                                                                                                      range(47)]
 
 ## #################################
 ## Build Menu and Toolbar 
@@ -155,6 +155,7 @@ def _init_ctrls(self, prnt):
         self.edit_bar.AddSimpleButton(wxID_RIBBONEDITFLAG, "Flag", flag.GetBitmap(), "")
         self.edit_bar.AddSimpleButton(wxID_RIBBONEDITADDPOINT, "Add Point", add.GetBitmap(), "")
         self.edit_bar.AddSimpleButton(wxID_RIBBONEDITDELPOINT, "Delete Point", delete.GetBitmap(), "")
+        self.edit_bar.AddSimpleButton(wxID_GAPFILL, "Fill Gap", add.GetBitmap(), "")
         #self.edit_bar.AddSimpleButton(wxID_RIBBONEDITRECORD, "Record", bitmap=record.GetBitmap(), help_string="",kind=0x4)
 
         self.edit_bar.EnableButton(wxID_RIBBONEDITFILTER, False)
@@ -250,6 +251,7 @@ def bindEvents(self):
         self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onEditFlag, id=wxID_RIBBONEDITFLAG)
         self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onEditAddPoint, id=wxID_RIBBONEDITADDPOINT)
         self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onEditDelPoint, id=wxID_RIBBONEDITDELPOINT)
+        self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onGapFill, id = wxID_GAPFILL)
 
         self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onRecordNew, id=wxID_RIBBONRECORDNEW)
         self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onRecordOpen, id=wxID_RIBBONRECORDOPEN)
@@ -466,7 +468,12 @@ def onEditAddPoint(self, event):
         addPoint.ShowModal()
 
         event.Skip()
-
+    # ###################################
+    #   Gap fill
+    # ###################################
+    def onGapFill(self, event):
+        self.parent.getRecordService().fill_gap(30, 'minute')
+        event.Skip()
     # ###################################
     #   Delete Point
     # ###################################

odmtools/odmdata/memory_database.py

@@ -201,7 +201,7 @@ def initEditValues(self, seriesID):
         """
         if not self.editLoaded:
             logger.debug("Load series from db")
-
+            self.series = self.series_service.get_series_by_id(seriesID)
             self.df = self.series_service.get_values_by_series(seriesID)
             self.editLoaded = True
 

odmtools/odmservices/edit_service.py

@@ -86,6 +86,7 @@ def _test_filter_previous(self):
 
         return df
 
+
     def datetime2dataframe(self, datetime_list):
         """ Converts datetime_list to a pandas Dataframe
 
@@ -141,19 +142,47 @@ def filter_date(self, before, after):
         if before and after:
             self.filtered_dataframe = df[(df.index < before) & (df.index > after)]
 
+    def fill_gap(self, gap, period):
+
+        df = self.memDB.getDataValuesDF()
+        gaps= self.find_gaps(df, gap, period)
+        points = []
+        series= self.memDB.series
+        timegap = np.timedelta64(gap, self.time_units[period])
+
+        for g in gaps.iterrows():
+            row= g[0]
+            e = row["datetime"]
+            s= row["prevdate"]
+            print (s)
+            print (e)
+
+            # for each gap time period in the larger gap ( until datetime = prev value)
+            while s != e:
+                s= s + timegap
+                points.append(('-9999', None, s, series.begin_date_time_utc, s, None, None, u'nc', None, None, series.site_id, series.variable_id, series.method_id, series.source_id, series.quality_control_level_id))
+                #add a row points.append() [('-9999', None, datetime.datetime(2007, 9, 28, 0, 10), '-7', datetime.datetime(2007, 9, 28, 7, 10), None, None, u'nc', None, None, 1, 4, 2, 1, 0)
+                #('-9999', None, DATE, series.begin_date_time_utc, UTCDATE, None, None, u'nc', None, None, series.site_id, series.variable_id, series.method_id, series.source_id, series.quality_control_level_id
+
+
+        self.add_points(points)
+
+    time_units = {
+        'second': 's',
+        'minute': 'm',
+        'hour': 'h',
+        'day': 'D',
+        'week': 'W',
+        'month': 'M',
+        'year': 'Y'
+    }
+
     # Data Gaps
-    def data_gaps(self, value, time_period):
-        df = self._test_filter_previous()
 
-        time_units = {
-            'second': 's',
-            'minute': 'm',
-            'hour': 'h',
-            'day': 'D',
-            'week': 'W',
-            'month': 'M',
-            'year': 'Y'
-        }
+
+    def find_gaps(self, df, value, time_period):
+
+
 
         # make a copy of the dataframe in order to modify it to be in the form we need to determine data gaps
         copy_df = df
@@ -165,19 +194,29 @@ def data_gaps(self, value, time_period):
             value = int(value)
 
         # create a bool column indicating which rows meet condition
-        filtered_results = copy_df['datetime'].diff() >= np.timedelta64(value, time_units[time_period])
+        filtered_results = copy_df['datetime'].diff() > np.timedelta64(value, self.time_units[time_period])
 
         # filter on rows that passed previous condition
-        copy_df = copy_df[filtered_results]
+        return copy_df[filtered_results]
+
+
 
+
+    def data_gaps(self, value, time_period):
+        df = self._test_filter_previous()
+        copy_df = self.find_gaps(df, value, time_period)
+        print (copy_df)
         # merge values and remove duplicates. this hack allows for both values to be marked when selecting data gaps
         newdf = pd.concat([copy_df['datetime'], copy_df['dateprev']], join='inner')
-        self.filtered_dataframe = df[df.index.isin(newdf.drop_duplicates().dropna())]
 
         # clean up
         del copy_df
-        del filtered_results
-        del newdf
+
+
+        self.filtered_dataframe= df[df.index.isin(newdf.drop_duplicates().dropna())]
+
+
+
 
     def change_value_threshold(self, value, operator):