Merge pull request #245 from AllenInstitute/fix/legacy_metadata

legacy file support

Author: neuromusic

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.0
+current_version = 0.4.0
 commit = True
 tag = True
 

setup.py

@@ -4,7 +4,7 @@
 
 setuptools.setup(
     name="visual-behavior",
-    version="0.3.0",
+    version="0.4.0",
     author="Justin Kiggins",
     author_email="justink@alleninstitute.org",
     description="analysis package for visual behavior",

tests/conftest.py

@@ -181,23 +181,6 @@ def exemplar_extended_trials_fixture():
 @pytest.fixture(scope="module")
 def trials_df_fixture():
     trials = pd.read_pickle(os.path.join(TESTING_RES_DIR, "trials.pkl"))
-    trials["scheduled_change_time"] = trials.apply(
-        lambda row: row["scheduled_change_time"] - row["starttime"],
-        axis=1
-    )  # change scheduled_change_time from time relative to experiment start to time relative to trial start
-    del trials['stim_on_frames']
-    del trials['publish_time']
-    trials['endframe'] = trials['startframe'].shift(periods=-1)
-    trials.at[trials.index[-1],'endframe']=94157
-
-    def nan_to_empty_string(val):
-        if pd.isnull(val):
-            return ''
-        return val
-    trials["change_image_category"] = trials["change_image_category"].apply(nan_to_empty_string)  # use empty string instead of NoneType
-    trials["change_image_name"] = trials["change_image_name"].apply(nan_to_empty_string)  # use empty string instead of NoneType
-    trials["initial_image_category"] = trials["initial_image_category"].apply(nan_to_empty_string)  # use empty string instead of NoneType
-    trials["initial_image_name"] = trials["initial_image_name"].apply(nan_to_empty_string)  # use empty string instead of NoneType
     return trials
 
 

tests/res/trials.pkl

@@ -26,6 +26,9 @@ def test_load_time(behavioral_session_output_fixture):
 
 def test_load_trials(behavioral_session_output_fixture, trials_df_fixture):
     trials = foraging.load_trials(behavioral_session_output_fixture)
+
+    trials_df_fixture['change_frame'] += 1
+
     pd.testing.assert_frame_equal(
         trials,
         trials_df_fixture,
@@ -96,6 +99,13 @@ def test_load_running_speed(behavioral_session_output_fixture):
             #     3: -100.3045892838318,
             #     4: -218.06477955362016
             # },
+            'frame': {
+                0: 0,
+                1: 1,
+                2: 2,
+                3: 3,
+                4: 4
+            },
             'speed': {
                 0: 0.0,
                 1: 0.14556417360329282,
@@ -111,10 +121,6 @@ def test_load_running_speed(behavioral_session_output_fixture):
                 4: 0.2000794094055891
             }
         },
-        columns=[
-            u'speed',
-            u'time',
-        ]
     )
 
     pd.testing.assert_frame_equal(

tests/translator/foraging/test_foraging_translator.py

@@ -283,10 +283,11 @@ def test_get_licks(monkeypatch, foraging2_data_fixture):
     (
         {},
         pd.DataFrame(data={
-            "time": np.array([0.0, 0.016, 0.032, 0.048, 0.064, ]),
-            "speed (cm/s)": np.array([0, 0, 0, 0, 0, ]),
-            "acceleration (cm/s^2)": np.array([0, 0, 0, 0, 0, ]),
-            "jerk (cm/s^3)": np.array([0, 0, 0, 0, 0, ]),
+            "time": [0.0, 0.016, 0.032, 0.048, 0.064, ],
+            "speed": [0, 0, 0, 0, 0, ],
+            "frame": [0, 1, 2, 3, 4, ]
+            # "acceleration (cm/s^2)": np.array([0, 0, 0, 0, 0, ]),
+            # "jerk (cm/s^3)": np.array([0, 0, 0, 0, 0, ]),
         }),
     ),
 ])

tests/translator/foraging2/test_foraging2_extract.py

@@ -1,5 +1,70 @@
-from visual_behavior.translator import schemas
+
+from visual_behavior.schemas.core import MetadataSchema, StimulusSchema, \
+    RunningSchema, LickSchema, RewardSchema, TrialSchema
+from visual_behavior.schemas.extended_trials import ExtendedTrialSchema
+from visual_behavior.translator import foraging2, foraging
+from visual_behavior.translator.core import create_extended_dataframe
+from visual_behavior.uuid_utils import create_session_uuid
 
 
 """test the schemas vs the outputs here
 """
+
+def _test_core_data_schemas(core_data):
+
+    # metadata
+
+    # core dataframes
+    dataframe_schemas = (
+        (StimulusSchema, core_data['visual_stimuli']),
+        (RunningSchema, core_data['running']),
+        (LickSchema, core_data['licks']),
+        (RewardSchema, core_data['rewards']),
+        (TrialSchema, core_data['trials']),
+    )
+
+    for Schema, data in dataframe_schemas:
+        errors = Schema(many=True).validate(data.to_dict(orient='records'))
+
+        for row, row_errors in errors.items():
+            assert len(row_errors)==0, (Schema, data, row_errors)
+
+    extended_trials = create_extended_dataframe(
+        trials=core_data['trials'],
+        metadata=core_data['metadata'],
+        licks=core_data['licks'],
+        time=core_data['time'],
+    )
+
+    errors = ExtendedTrialSchema(many=True).validate(
+        extended_trials.to_dict(orient='records')
+    )
+
+    for row, row_errors in errors.items():
+        assert len(row_errors)==0, row_errors
+
+    errors = MetadataSchema().validate(core_data['metadata'])
+    assert len(errors)==0, errors.keys()
+
+
+def test_foraging2_translator_schema(foraging2_data_stage4_2018_05_10):
+    core_data = foraging2.data_to_change_detection_core(
+        foraging2_data_stage4_2018_05_10
+    )
+    core_data['metadata']['behavior_session_uuid'] = create_session_uuid(
+        core_data['metadata']['mouseid'],
+        core_data['metadata']['startdatetime'],
+    )
+    _test_core_data_schemas(core_data)
+
+def test_foraging_translator_schema(behavioral_session_output_fixture):
+    core_data = foraging.data_to_change_detection_core(
+        behavioral_session_output_fixture
+    )
+
+    core_data['metadata']['behavior_session_uuid'] = create_session_uuid(
+        str(core_data['metadata']['mouseid']),
+        core_data['metadata']['startdatetime'],
+    )
+
+    _test_core_data_schemas(core_data)

tests/translator/test_translator_functional.py

@@ -1 +1 @@
-__version__ = "0.3.0"
+__version__ = "0.4.0"

visual_behavior/__init__.py

@@ -20,10 +20,10 @@ class RewardSchema(TimeSeriesSchema):
     """ schema for water reward presentations
 
     """
-    volume = fields.Float(
-        description='Volume of water dispensed on this reward presentation in mL',
-        required=True,
-    )
+    # volume = fields.Float(
+    #     description='Volume of water dispensed on this reward presentation in mL',
+    #     required=True,
+    # )
     # lickspout = fields.Int(
     #     description='The water line on which this reward was dispensed',
     #     required=True,
@@ -208,13 +208,13 @@ class StimulusSchema(TimeSeriesSchema):
         required=True,
         allow_none=True,
     )
-    contrast = fields.Float(
-        description='The contrast of a grating stimulus',
+    orientation = fields.Float(
+        description='The orientation of a grating stimulus',
         required=True,
         allow_none=True,
     )
-    orientation = fields.Float(
-        description='The orientation of a grating stimulus',
+    end_frame = fields.Int(
+        description='The last frame of this stimulus, non-inclusive',
         required=True,
     )
 
@@ -298,6 +298,54 @@ class MetadataSchema(Schema):
         description='total number of stimulus frames',
         required=True,
     )
+    auto_reward_vol = fields.Float(
+        description='volume provided during autoreward trials',
+        required=True,
+    )
+    max_session_duration = fields.Float(
+        description='maximum duration in minutes of a session',
+        required=True,
+    )
+    min_no_lick_time = fields.Float(
+        description='minimum time where there should be no licks before the start of a trial',
+        required=True,
+    )
+    free_reward_trials = fields.Int(
+        description='number of free reward trials to start the session',
+        required=True,
+    )
+    abort_on_early_response = fields.Bool(
+        description='if True, abort trials on early responses',
+        required=True,
+    )
+    even_sampling_enabled = fields.Bool(
+        description='if True, images should be sample evenly from the change matrix',
+        required=True,
+    )
+    failure_repeats = fields.Int(
+        description='maximum number of times to repeat parameters after a false alarm',
+        required=True,
+    )
+    initial_blank_duration = fields.Float(
+        description='duration of grey screen at start of each trial, in seconds',
+        required=True,
+    )
+    catch_frequency = fields.Float(
+        description='fraction of trials that should be catch trials',
+        required=True,
+    )
+    warm_up_trials = fields.Int(
+        description='number of warm up trials at start of session',
+        required=True,
+    )
+    stimulus_window = fields.Float(
+        description='start and stop times of stimulus window',
+        required=True,
+    )
+    volume_limit = fields.Float(
+        description='maximum volume of water to deliver in a session, in mL',
+        required=True,
+    )
 
 
 # class ChangeDetectionSessionCoreSchema(Schema):

visual_behavior/schemas/core.py

@@ -2,6 +2,7 @@
 import pandas as pd
 import numpy as np
 from scipy.signal import medfilt
+from .extract import get_end_time
 from ...utilities import calc_deriv, rad_to_dist, local_time
 
 warnings.warn(
@@ -81,6 +82,25 @@ def load_metadata(data):
         timezone='America/Los_Angeles',
     )
 
+    metadata['auto_reward_vol'] = 0.05  # hard coded
+    metadata['max_session_duration'] = 60.0  # hard coded
+    metadata['min_no_lick_time'] = data['minimum_no_lick_time']
+    metadata['abort_on_early_response'] = data['ignore_false_alarms'] == False
+    metadata['even_sampling_enabled'] = data['image_category_sampling_mode'] == 'even_sampling'
+    metadata['failure_repeats'] = data['max_number_trial_repeats']
+    metadata['catch_frequency'] = data['catch_frequency']
+    metadata['volume_limit'] = data['volumelimit']
+    metadata['initial_blank_duration'] = data['initial_blank']
+    metadata['warm_up_trials'] = data['warmup_trials']
+    metadata['stimulus_window'] = data['trial_duration'] - data['delta_minimum']
+
+    block_length = data['lick_detect_training_block_length']
+
+    try:
+        metadata['free_reward_trials'] = block_length[0]
+    except TypeError:
+        metadata['free_reward_trials'] = block_length
+
     return metadata
 
 
@@ -97,6 +117,9 @@ def load_trials(data, time=None):
 
     """
 
+    if time is None:
+        time = load_time(data)
+
     columns = (
         'auto_rewarded',
         'change_contrast',
@@ -164,7 +187,8 @@ def stringify(x):
     for col in forced_string:
         trials[col] = trials[col].map(stringify)
 
-    trials['change_frame'] = trials['change_frame'].map(lambda x: int(x) if np.isfinite(x) else None)
+    trials['change_frame'] = trials['change_frame'].map(lambda x: int(x) if np.isfinite(x) else None) + 1
+
     trials["change_image_category"] = trials["change_image_category"].apply(lambda x: x if x else '')  # use empty string instead of NoneType
     trials["change_image_name"] = trials["change_image_name"].apply(lambda x: x if x else '')  # use empty string instead of NoneType
     trials["initial_image_category"] = trials["initial_image_category"].apply(lambda x: x if x else '')  # use empty string instead of NoneType
@@ -175,7 +199,10 @@ def stringify(x):
 
     # add endframe column as startframe of last frame. Last endframe is last frame of session
     trials['endframe'] = trials['startframe'].shift(periods=-1)
-    trials.at[trials.index[-1], 'endframe'] = len(load_time(data)) - 1
+    trials.at[trials.index[-1], 'endframe'] = len(time) - 1
+
+    trials['endtime'] = get_end_time(trials, time)
+    trials['trial_length'] = trials['endtime'] - trials['starttime']
 
     return trials
 
@@ -272,6 +299,7 @@ def load_running_speed(data, smooth=False, time=None):
 
     running_speed = pd.DataFrame({
         'time': time,
+        'frame': range(len(time)),
         'speed': speed,
         # 'acceleration (cm/s^2)': accel,
         # 'jerk (cm/s^3)': jerk,
@@ -329,15 +357,18 @@ def load_visual_stimuli(data, time=None):
     stimdf = pd.DataFrame(data['stimuluslog'])
 
     stimdf = find_ends(stimdf)
-    stimdf['end'] = stimdf['frames_to_end'] + stimdf['frame']
+    stimdf['end_frame'] = stimdf['frames_to_end'] + stimdf['frame']
+
+    stimdf['frame'] += 1
+    stimdf['end_frame'] += 1
 
     def find_time(fr):
         try:
             return time[fr]
         except IndexError:
             return np.nan
 
-    stimdf['end_time'] = stimdf['end'].map(find_time)
+    stimdf['end_time'] = stimdf['end_frame'].map(find_time)
     stimdf['duration'] = stimdf['end_time'] - stimdf['time']
 
     onset_mask = (
@@ -348,13 +379,13 @@ def find_time(fr):
     ) > 0
 
     if pd.isnull(stimdf['image_name']).any() == False:  # 'image_category' in stimdf.columns:
-        cols = ['frame', 'time', 'duration', 'image_category', 'image_name']
+        cols = ['frame', 'end_frame', 'time', 'duration', 'image_category', 'image_name']
         stimuli = stimdf[onset_mask][cols]
         stimuli['orientation'] = None
         stimuli['contrast'] = None
 
     elif 'ori' in stimdf.columns:
-        cols = ['frame', 'time', 'duration', 'ori', 'contrast']
+        cols = ['frame', 'end_frame', 'time', 'duration', 'ori', 'contrast']
         stimuli = stimdf[onset_mask][cols]
         stimuli.rename(columns={'ori': 'orientation'}, inplace=True)
         stimuli['image_category'] = None