merging

Author: stephanie

ODMTools.py

@@ -22,6 +22,7 @@
 import pymysql
 #import psycopg2
 
+
 tool = LoggerTool()
 logger = tool.setupLogger('main',  'odmtools.log', 'a', logging.INFO)
 wx.Log.SetLogLevel(0)

doc/wxFormBuilder/frmFillGaps.fbp

@@ -0,0 +1,26 @@
+"""Subclass of dlgFill, which is generated by wxFormBuilder."""
+
+from odmtools.view import clsGapFill
+
+
+# Implementing dlgFill
+class frmGapFill(clsGapFill.dlgFill):
+    def __init__(self, parent, record_service):
+        self.record_service = record_service
+        clsGapFill.dlgFill.__init__(self, parent)
+
+    # Handlers for dlgFill events.
+    def onOKBtn(self, event):
+        #TODO add validation
+        gapvalue= self.txtGap.Value
+        gaptime = self.cbGap.Value
+        fillvalue = self.txtFill.Value
+        filltime= self.cbFill.Value
+
+        self.record_service.fill_gap(gap=[gapvalue, gaptime], fill=[fillvalue, filltime])
+        self.Close()
+
+    def OnCancelBtn(self, event):
+
+        self.Close()
+        self.Destroy()

odmtools/controller/frmGapFill.py

@@ -50,14 +50,26 @@ def filter_date(self, endDate, startDate):
         else:
             return "Cannot filter: %s" % (self._edit_error)
 
+
+    def fill_gap(self, gap, fill):
+        self._edit_service.fill_gap(gap , fill)
+
+        self.refresh_edit()
+        if self._record:
+            self._script("edit_service.fill_gap(gap = %s, fill= %s)\n" % (gap, fill), 'black')
+            Publisher.sendMessage("scroll")
+
+
+
+
+
     def data_gaps(self, value, time_period):
         self._edit_service.data_gaps(value, time_period)
         self.refresh_selection()
         if self._record:
             self._script("edit_service.data_gaps(%s, '%s')\n" % (value, time_period), 'black')
             Publisher.sendMessage("scroll")
 
-
     def value_change_threshold(self, value, operator):
         self._edit_service.change_value_threshold(value, operator)
         self.refresh_selection()

odmtools/controller/logicEditTools.py

@@ -14,6 +14,7 @@
 from frmFlagValues import frmFlagValues
 from odmtools.controller.frmLinearDrift import frmLinearDrift
 from odmtools.controller.frmAbout import frmAbout
+from odmtools.controller.frmGapFill import frmGapFill
 import wizSave
 from odmtools.common.icons import *
 import pandas as pd
@@ -36,9 +37,9 @@
  wxID_RIBBONEDITSCRIPTSAVE, wxID_RIBBONVIEWPLOT, wxID_RIBBONVIEWTABLE, wxID_RIBBONVIEWSERIES, wxID_RIBBONVIEWCONSOLE,
  wxID_RIBBONVIEWSCRIPT, wxID_RIBBONPLOTBLANKBTN, wxID_FileMenu, wxID_STARTDPDATE, wxID_ENDDPDATE, wxID_FRAME1SPINCTRL1,
  wxID_RIBBONEDITFILTER, wxID_RIBBONEDITRECORD, wxID_RIBBONEDITLINFILTER, wxID_RIBBONPLOTDATEAPPLY,
- wxID_RIBBONEDITRESETFILTER, wxID_RIBBONRECORDNEW, wxID_RIBBONRECORDOPEN, wxID_RIBBONRECORDSAVE] = [wx.NewId() for
+ wxID_RIBBONEDITRESETFILTER, wxID_RIBBONRECORDNEW, wxID_RIBBONRECORDOPEN, wxID_RIBBONRECORDSAVE, wxID_GAPFILL] = [wx.NewId() for
                                                                                                       _init_ctrls in
-                                                                                                      range(46)]
+                                                                                                      range(47)]
 
 ## #################################
 ## Build Menu and Toolbar 
@@ -155,6 +156,7 @@ def _init_ctrls(self, prnt):
         self.edit_bar.AddSimpleButton(wxID_RIBBONEDITFLAG, "Flag", flag.GetBitmap(), "")
         self.edit_bar.AddSimpleButton(wxID_RIBBONEDITADDPOINT, "Add Point", add.GetBitmap(), "")
         self.edit_bar.AddSimpleButton(wxID_RIBBONEDITDELPOINT, "Delete Point", delete.GetBitmap(), "")
+        self.edit_bar.AddSimpleButton(wxID_GAPFILL, "Fill Gap", add.GetBitmap(), "")
         #self.edit_bar.AddSimpleButton(wxID_RIBBONEDITRECORD, "Record", bitmap=record.GetBitmap(), help_string="",kind=0x4)
 
         self.edit_bar.EnableButton(wxID_RIBBONEDITFILTER, False)
@@ -250,6 +252,7 @@ def bindEvents(self):
         self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onEditFlag, id=wxID_RIBBONEDITFLAG)
         self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onEditAddPoint, id=wxID_RIBBONEDITADDPOINT)
         self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onEditDelPoint, id=wxID_RIBBONEDITDELPOINT)
+        self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onGapFill, id = wxID_GAPFILL)
 
         self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onRecordNew, id=wxID_RIBBONRECORDNEW)
         self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onRecordOpen, id=wxID_RIBBONRECORDOPEN)
@@ -466,7 +469,23 @@ def onEditAddPoint(self, event):
         addPoint.ShowModal()
 
         event.Skip()
+    # ###################################
+    #   Gap fill
+    # ###################################
+    def onGapFill(self, event):
+
+        gap_fill = frmGapFill(self.parent, self.parent.getRecordService())
+
+        if gap_fill.Show() == wx.OK:
+            gap_fill.Destroy()
+
+
 
+
+
+
+
+        event.Skip()
     # ###################################
     #   Delete Point
     # ###################################

odmtools/gui/mnuRibbon.py

@@ -136,7 +136,9 @@ def updatePlot(self):
                 #self.prob.append(
                 #prop = oneSeries.Probability.plot(column="DataValue", ax=self.plots)
                 #todo FutureWarning: order is deprecated, use sort_values(...)
+                #xValues = oneSeries.Probability.xAxis.order().values
                 xValues = oneSeries.Probability.xAxis.order().values
+                # yValues = oneSeries.Probability.yAxis.order().values
                 yValues = oneSeries.Probability.yAxis.order().values
 
                 ax = self.plots.plot(xValues, yValues, 'bs', color=oneSeries.color,

odmtools/gui/plotProbability.py

@@ -201,7 +201,7 @@ def initEditValues(self, seriesID):
         """
         if not self.editLoaded:
             logger.debug("Load series from db")
-
+            self.series = self.series_service.get_series_by_id(seriesID)
             self.df = self.series_service.get_values_by_series(seriesID)
             self.editLoaded = True
 

odmtools/odmdata/memory_database.py

@@ -17,6 +17,21 @@
 # logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG)
 logger =logging.getLogger('main')
 
+class time(object):
+    time_units = {
+        'second': 's',
+        'minute': 'm',
+        'hour': 'h',
+        'day': 'D',
+        'week': 'W',
+        'month': 'M',
+        'year': 'Y'
+    }
+
+    def __init__(self, value, time_period):
+        self.value = value
+        self.time_period = time_period
+
 class EditService():
     # Mutual exclusion: cursor, or connection_string
     def __init__(self, series_id, connection=None, connection_string="", debug=False):
@@ -86,6 +101,7 @@ def _test_filter_previous(self):
 
         return df
 
+
     def datetime2dataframe(self, datetime_list):
         """ Converts datetime_list to a pandas Dataframe
 
@@ -141,19 +157,51 @@ def filter_date(self, before, after):
         if before and after:
             self.filtered_dataframe = df[(df.index < before) & (df.index > after)]
 
+    def fill_gap(self, gap, fill):
+
+        df = self.memDB.getDataValuesDF()
+        gaps= self.find_gaps(df, gap[0], gap[1])
+        points = []
+        series= self.memDB.series
+        timegap = np.timedelta64(fill[0], self.time_units[fill[1]])
+
+        #if gaps is not of type dataframe- put it in a dataframe
+        #if not isinstance(gaps, pd.DataFrame
+        for g in gaps.iterrows():
+            row = g[1]
+            e = row.datetime
+            s = row.dateprev
+
+            #prime the loop
+            s = s + timegap
+            # for each gap time period in the larger gap ( until datetime = prev value)
+            while s < e:
+                utc_offset = (series.begin_date_time-series.begin_date_time_utc).total_seconds()/3600
+                points.append((-9999, None, s, utc_offset, s, None, None, u'nc', None, None, series.site_id, series.variable_id, series.method_id, series.source_id, series.quality_control_level_id))
+                #('-9999', None, DATE, series.begin_date_time_utc, UTCDATE, None, None, u'nc', None, None,
+                #       series.site_id, series.variable_id, series.method_id, series.source_id,
+                #       series.quality_control_level_id
+
+                s = s + timegap
+        #print points
+        self.add_points(points)
+
+    time_units = {
+        'second': 's',
+        'minute': 'm',
+        'hour': 'h',
+        'day': 'D',
+        'week': 'W',
+        'month': 'M',
+        'year': 'Y'
+    }
+
     # Data Gaps
-    def data_gaps(self, value, time_period):
-        df = self._test_filter_previous()
 
-        time_units = {
-            'second': 's',
-            'minute': 'm',
-            'hour': 'h',
-            'day': 'D',
-            'week': 'W',
-            'month': 'M',
-            'year': 'Y'
-        }
+
+    def find_gaps(self, df, value, time_period):
+
+
 
         # make a copy of the dataframe in order to modify it to be in the form we need to determine data gaps
         copy_df = df
@@ -165,19 +213,29 @@ def data_gaps(self, value, time_period):
             value = int(value)
 
         # create a bool column indicating which rows meet condition
-        filtered_results = copy_df['datetime'].diff() >= np.timedelta64(value, time_units[time_period])
+        filtered_results = copy_df['datetime'].diff() > np.timedelta64(value, self.time_units[time_period])
 
         # filter on rows that passed previous condition
-        copy_df = copy_df[filtered_results]
+        return copy_df[filtered_results]
+
+
 
+
+    def data_gaps(self, value, time_period):
+        df = self._test_filter_previous()
+        copy_df = self.find_gaps(df, value, time_period)
+        print (copy_df)
         # merge values and remove duplicates. this hack allows for both values to be marked when selecting data gaps
         newdf = pd.concat([copy_df['datetime'], copy_df['dateprev']], join='inner')
-        self.filtered_dataframe = df[df.index.isin(newdf.drop_duplicates().dropna())]
 
         # clean up
         del copy_df
-        del filtered_results
-        del newdf
+
+
+        self.filtered_dataframe= df[df.index.isin(newdf.drop_duplicates().dropna())]
+
+
+
 
     def change_value_threshold(self, value, operator):
 
@@ -310,7 +368,6 @@ def change_value(self, value, operator):
 
     def add_points(self, points):
         # todo: add the ability to send in multiple datetimes to a single 'point'
-
         self.memDB.addPoints(points)
 
         self._populate_series()

odmtools/odmservices/edit_service.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*- 
+
+###########################################################################
+## Python code generated with wxFormBuilder (version Jun 17 2015)
+## http://www.wxformbuilder.org/
+##
+## PLEASE DO "NOT" EDIT THIS FILE!
+###########################################################################
+
+import wx
+import wx.xrc
+
+
+###########################################################################
+## Class dlgFill
+###########################################################################
+
+class dlgFill(wx.Dialog):
+    def __init__(self, parent):
+        wx.Dialog.__init__(self, parent, id=wx.ID_ANY, title=wx.EmptyString, pos=wx.DefaultPosition,
+                           size=wx.Size(315, 217), style=wx.DEFAULT_DIALOG_STYLE)
+
+        self.SetSizeHintsSz(wx.Size(315, 217), wx.Size(315, 217))
+
+        bsForm = wx.BoxSizer(wx.VERTICAL)
+
+        self.lblInstructions = wx.StaticText(self, wx.ID_ANY,
+                                             u"This function fills any gaps greater than the gap duration with a no-data value at the fill frequency.",
+                                             wx.DefaultPosition, wx.DefaultSize, 0)
+        self.lblInstructions.Wrap(300)
+        bsForm.Add(self.lblInstructions, 0, wx.ALL, 5)
+
+        fgSizer1 = wx.FlexGridSizer(0, 3, 0, 0)
+        fgSizer1.SetFlexibleDirection(wx.BOTH)
+        fgSizer1.SetNonFlexibleGrowMode(wx.FLEX_GROWMODE_SPECIFIED)
+
+        self.lblGap = wx.StaticText(self, wx.ID_ANY, u"Gap Duration:", wx.DefaultPosition, wx.DefaultSize, 0)
+        self.lblGap.Wrap(-1)
+        fgSizer1.Add(self.lblGap, 0, wx.ALL, 5)
+
+        self.txtGap = wx.TextCtrl(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, 0)
+        fgSizer1.Add(self.txtGap, 0, wx.ALL, 5)
+
+        cbGapChoices = [u"second", u"minute", u"hour", u"days", u"week", u"month", u"day", u"year"]
+        self.cbGap = wx.ComboBox(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, cbGapChoices, 0)
+        self.cbGap.SetSelection(1)
+        fgSizer1.Add(self.cbGap, 1, wx.ALL, 5)
+
+        self.lblFill = wx.StaticText(self, wx.ID_ANY, u"Fill Frequency:", wx.DefaultPosition, wx.DefaultSize, 0)
+        self.lblFill.Wrap(-1)
+        fgSizer1.Add(self.lblFill, 0, wx.ALL, 5)
+
+        self.txtFill = wx.TextCtrl(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, 0)
+        fgSizer1.Add(self.txtFill, 0, wx.ALL, 5)
+
+        cbFillChoices = [u"second", u"minute", u"hour", u"day", u"week", u"month", u"year"]
+        self.cbFill = wx.ComboBox(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, cbFillChoices, 0)
+        self.cbFill.SetSelection(1)
+        fgSizer1.Add(self.cbFill, 1, wx.ALL, 5)
+
+        bsForm.Add(fgSizer1, 1, wx.EXPAND, 5)
+
+        m_sdbSizer1 = wx.StdDialogButtonSizer()
+        self.m_sdbSizer1OK = wx.Button(self, wx.ID_OK)
+        m_sdbSizer1.AddButton(self.m_sdbSizer1OK)
+        self.m_sdbSizer1Cancel = wx.Button(self, wx.ID_CANCEL)
+        m_sdbSizer1.AddButton(self.m_sdbSizer1Cancel)
+        m_sdbSizer1.Realize();
+
+        bsForm.Add(m_sdbSizer1, 1, wx.EXPAND, 5)
+
+        self.SetSizer(bsForm)
+        self.Layout()
+
+        self.Centre(wx.BOTH)
+
+        # Connect Events
+        self.m_sdbSizer1Cancel.Bind(wx.EVT_BUTTON, self.OnCancelBtn)
+        self.m_sdbSizer1OK.Bind(wx.EVT_BUTTON, self.onOKBtn)
+
+    def __del__(self):
+        pass
+
+    # Virtual event handlers, overide them in your derived class
+    def OnCancelBtn(self, event):
+        event.Skip()
+
+    def onOKBtn(self, event):
+        event.Skip()
+
+