Merge pull request #281 from AllenInstitute/fix/qc

neuromusic · web-flow · commit 1762f4def4f3 · 2018-07-17T21:06:42.000-07:00
fixes metadata on legacy loading
diff --git a/setup.cfg b/setup.cfg
@@ -1,15 +1,15 @@
 [bumpversion]
-current_version = 0.4.0
+current_version = 0.4.1
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<n>\d+))?
-serialize =
+serialize = 
 	{major}.{minor}.{patch}.{release}{n}
 	{major}.{minor}.{patch}
 
 [bumpversion:part:release]
 optional_value = gamma
-values =
+values = 
 	dev
 	a
 	b
@@ -37,3 +37,4 @@ exclude = git/*, notebooks/*, tests/*, scripts/*
 [coverage:run]
 omit = tests/*, visual_behavior/cohorts/*, visual_behavior/inscopix/*, scripts/*,
 	notebooks/*, circleci/*, .git/*, .tox/*, .cache/*, visual_behavior/pizza/*
+
diff --git a/setup.py b/setup.py
@@ -4,7 +4,7 @@
 
 setuptools.setup(
     name="visual-behavior",
-    version="0.4.0",
+    version="0.4.1",
     author="Justin Kiggins",
     author_email="justink@alleninstitute.org",
     description="analysis package for visual behavior",
diff --git a/tests/translator/foraging2/test_foraging2_extract.py b/tests/translator/foraging2/test_foraging2_extract.py
@@ -690,3 +690,6 @@ def test_get_max_session_duration(foraging2_data_stage4_2018_05_10):
 
 def test_get_abort_on_early_response(foraging2_data_stage4_2018_05_10):
     assert extract.get_abort_on_early_response(foraging2_data_stage4_2018_05_10) == True
+
+def test_get_periodic_flash(foraging2_data_stage4_2018_05_10):
+    assert extract.get_periodic_flash(foraging2_data_stage4_2018_05_10) == (0.25, 0.5)
diff --git a/tests/translator/foraging2/test_foraging2_translator.py b/tests/translator/foraging2/test_foraging2_translator.py
@@ -141,6 +141,7 @@ def test_data_to_metadata(monkeypatch, foraging2_data_fixture):
             "behavior_session_uuid": '',
             "abort_on_early_response": True,
             "auto_reward_delay": 0.0,
+            "periodic_flash": (0.25, 0.5),
         }
 
 
diff --git a/tests/translator/test_translator_functional.py b/tests/translator/test_translator_functional.py
@@ -5,6 +5,7 @@
 from visual_behavior.translator import foraging2, foraging
 from visual_behavior.translator.core import create_extended_dataframe
 from visual_behavior.uuid_utils import create_session_uuid
+from visual_behavior.validation.qc import generate_qc_report
 
 
 """test the schemas vs the outputs here
@@ -56,6 +57,8 @@ def test_foraging2_translator_schema(foraging2_data_stage4_2018_05_10):
         core_data['metadata']['startdatetime'],
     )
     _test_core_data_schemas(core_data)
+    generate_qc_report(core_data)
+
 
 def test_foraging_translator_schema(behavioral_session_output_fixture):
     core_data = foraging.data_to_change_detection_core(
@@ -68,3 +71,4 @@ def test_foraging_translator_schema(behavioral_session_output_fixture):
     )
 
     _test_core_data_schemas(core_data)
+    generate_qc_report(core_data)
diff --git a/visual_behavior/__init__.py b/visual_behavior/__init__.py
@@ -1 +1 @@
-__version__ = "0.4.0"
+__version__ = "0.4.1"
diff --git a/visual_behavior/schemas/core.py b/visual_behavior/schemas/core.py
@@ -346,6 +346,16 @@ class MetadataSchema(Schema):
         description='maximum volume of water to deliver in a session, in mL',
         required=True,
     )
+    auto_reward_delay = fields.Float(
+        description='delay between change time and reward on autoreward trials',
+        required=True,
+    )
+    periodic_flash = fields.List(
+        fields.Float,
+        description='duration of flash and grey screen',
+        required=True,
+        allow_none=True,
+    )
 
 
 # class ChangeDetectionSessionCoreSchema(Schema):
diff --git a/visual_behavior/translator/foraging/__init__.py b/visual_behavior/translator/foraging/__init__.py
@@ -93,6 +93,10 @@ def load_metadata(data):
     metadata['initial_blank_duration'] = data['initial_blank']
     metadata['warm_up_trials'] = data['warmup_trials']
     metadata['stimulus_window'] = data['trial_duration'] - data['delta_minimum']
+    metadata['auto_reward_delay'] = 0.0  # hard coded
+
+    blank_dur = np.mean(data['blank_duration_range'])
+    metadata['periodic_flash'] = (data['stim_duration'], blank_dur) if blank_dur > 0 else None
 
     block_length = data['lick_detect_training_block_length']
 
diff --git a/visual_behavior/translator/foraging2/__init__.py b/visual_behavior/translator/foraging2/__init__.py
@@ -13,7 +13,7 @@
     get_stimulus_window, get_volume_limit, get_failure_repeats, \
     get_catch_frequency, get_free_reward_trials, get_min_no_lick_time, \
     get_max_session_duration, get_abort_on_early_response, get_session_id, \
-    get_even_sampling, get_auto_reward_delay
+    get_even_sampling, get_auto_reward_delay, get_periodic_flash
 
 
 from .extract_stimuli import get_visual_stimuli
@@ -164,6 +164,7 @@ def data_to_metadata(data):
         "initial_blank_duration": get_initial_blank_duration(data),
         "even_sampling_enabled": get_even_sampling(data),
         "behavior_session_uuid": behavior_session_uuid,
+        "periodic_flash": get_periodic_flash(data),
     }
 
 
diff --git a/visual_behavior/translator/foraging2/extract.py b/visual_behavior/translator/foraging2/extract.py
@@ -1434,3 +1434,7 @@ def get_abort_on_early_response(data):
         abort on early response boolean or None if not found
     """
     return data["items"]["behavior"]["config"]["DoC"]["abort_on_early_response"]
+
+
+def get_periodic_flash(data):
+    return tuple(data['items']['behavior']['config']['DoC']['periodic_flash'])
diff --git a/visual_behavior/validation/extended_trials.py b/visual_behavior/validation/extended_trials.py
@@ -168,31 +168,27 @@ def count_stimuli_per_trial(trials, visual_stimuli):
         elif all(pd.isnull(visual_stimuli.orientation)) == False:
             col_to_check = 'orientation'
 
-        # get the index of the first stimulus that started on or before the trial start
-        start_stim = visual_stimuli[
-            (visual_stimuli.frame <= trial.startframe + 1) &
-            (visual_stimuli.end_frame >= trial.startframe + 1)
-        ].index[-1]
-
-        # get the index of the last stimulus that ended on or before the trial end
-        end_stim = visual_stimuli[
-            (visual_stimuli.frame <= trial.endframe)
-        ].index[-1]
-
-        # get all unique stimuli on this trial
-        stimuli = np.unique(visual_stimuli.loc[start_stim:end_stim][col_to_check])
-        # add to array
-        stimuli_per_trial[idx] = len(stimuli)
+        try:
+            # get the index of the first stimulus that started on or before the trial start
+            start_stim = visual_stimuli[
+                (visual_stimuli.frame <= trial.startframe + 1) &
+                (visual_stimuli.end_frame >= trial.startframe + 1)
+            ].index[-1]
+
+            # get the index of the last stimulus that ended on or before the trial end
+            end_stim = visual_stimuli[
+                (visual_stimuli.frame <= trial.endframe)
+            ].index[-1]
+
+            # get all unique stimuli on this trial
+            stimuli = np.unique(visual_stimuli.loc[start_stim:end_stim][col_to_check])
+            # add to array
+            stimuli_per_trial[idx] = len(stimuli)
+        except IndexError:
+            stimuli_per_trial[idx] = 0
     return stimuli_per_trial
 
 
-def fix_periodic_flash(pf):
-    '''
-    deal with core_data['metadata']['params']['periodic_flash']='None'
-    '''
-    return None if pf == 'None' else pf
-
-
 # test functions
 def validate_autoreward_volume(trials, auto_reward_volume):
     '''Each dispense of water for autorewards should be `auto_reward_vol`.'''
@@ -859,7 +855,7 @@ def validate_change_frame_at_flash_onset(trials, visual_stimuli, periodic_flash)
     '''
     if `periodic_flash` is not null, changes should always coincide with a stimulus onset
     '''
-    if fix_periodic_flash(periodic_flash) is None:
+    if periodic_flash is None:
         return True
     else:
         # get all non-null change frames
@@ -877,7 +873,7 @@ def validate_initial_blank(trials, visual_stimuli, initial_blank, periodic_flash
     '''
 
     # this test doesn't make sense for static stimuli with no initial blank. just return True
-    if fix_periodic_flash(periodic_flash) is None and initial_blank == 0:
+    if periodic_flash is None and initial_blank == 0:
         return True
     else:
         # preallocate array
diff --git a/visual_behavior/validation/qc.py b/visual_behavior/validation/qc.py
@@ -45,7 +45,7 @@ def define_validation_functions(core_data):
     STIMULUS_WINDOW = core_data['metadata']['stimulus_window']
     VOLUME_LIMIT = core_data['metadata']['volume_limit']
 
-    PERIODIC_FLASH = et.fix_periodic_flash(core_data['metadata']['params']['periodic_flash'])
+    PERIODIC_FLASH = core_data['metadata']['periodic_flash']
 
     validation_functions = {
         # et.validate_schema

Original file line number	Diff line number	Diff line change
`@@ -141,6 +141,7 @@ def test_data_to_metadata(monkeypatch, foraging2_data_fixture):`
`141`	`141`	`"behavior_session_uuid": '',`
`142`	`142`	`"abort_on_early_response": True,`
`143`	`143`	`"auto_reward_delay": 0.0,`
	`144`	`+ "periodic_flash": (0.25, 0.5),`
`144`	`145`	`}`
`145`	`146`
`146`	`147`
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "0.4.0"`
	`1`	`+__version__ = "0.4.1"`