diff --git a/analytics_data_api/insights_snowflake/mappers/course_summaries.py b/analytics_data_api/insights_snowflake/mappers/course_summaries.py new file mode 100644 index 00000000..3fc38c70 --- /dev/null +++ b/analytics_data_api/insights_snowflake/mappers/course_summaries.py @@ -0,0 +1,117 @@ +"""Map Snowflake course summary rows into the existing API response shape.""" + +from itertools import groupby + +from analytics_data_api.constants import enrollment_modes + +COUNT_FIELDS = ('count', 'cumulative_count', 'count_change_7_days', 'passing_users') +SUMMARY_META_FIELDS = ( + 'catalog_course_title', + 'catalog_course', + 'start_time', + 'end_time', + 'pacing_type', + 'availability', +) + + +def _row_value(row, name): + """Return a row value from dictionary rows produced by the Snowflake client.""" + if name in row: + return row[name] + return row[name.upper()] + + +def _count_value(row, name): + """Return an integer count, treating missing nullable Snowflake counts as zero.""" + return int(_row_value(row, name) or 0) + + +def _base_course_summary(course_id): + """Return the default course summary shape used by the existing API.""" + summary = { + 'course_id': course_id, + 'created': None, + 'enrollment_modes': {}, + } + summary.update({field: 0 for field in COUNT_FIELDS}) + summary['enrollment_modes'].update({ + mode: { + count_field: 0 for count_field in COUNT_FIELDS + } for mode in enrollment_modes.ALL + }) + return summary + + +def _programs_by_course(program_rows): + """Return program IDs grouped by course ID.""" + programs = {} + for row in program_rows or []: + programs.setdefault(_row_value(row, 'course_id'), []).append(_row_value(row, 'program_id')) + return programs + + +def _recent_counts_by_course(recent_rows): + """Return recent enrollment counts keyed by course ID.""" + return { + _row_value(row, 'course_id'): _count_value(row, 'count') + for row in recent_rows or [] + } + + +def _postprocess_course_summary(summary, exclude=None): + """Apply existing course summary response compatibility rules.""" + modes = summary['enrollment_modes'] + prof_no_id_mode = modes.pop(enrollment_modes.PROFESSIONAL_NO_ID, {}) + prof_mode = modes[enrollment_modes.PROFESSIONAL] + for count_key in COUNT_FIELDS: + prof_mode[count_key] = prof_mode.get(count_key, 0) + prof_no_id_mode.pop(count_key, 0) + + if summary['availability'] == 'Starting Soon': + summary['availability'] = 'Upcoming' + + for field in exclude or []: + for mode in summary['enrollment_modes']: + summary['enrollment_modes'][mode].pop(field, None) + + return summary + + +def map_course_summary_rows(summary_rows, program_rows=None, recent_rows=None, exclude=None): + """Group course summary rows into one API item per course.""" + rows = sorted( + summary_rows or [], + key=lambda row: ( + _row_value(row, 'course_id') or '', + _row_value(row, 'enrollment_mode') or '', + ), + ) + programs = _programs_by_course(program_rows) if program_rows is not None else None + recent_counts = _recent_counts_by_course(recent_rows) if recent_rows is not None else None + summaries = [] + + for course_id, group in groupby(rows, lambda row: _row_value(row, 'course_id')): + summary = _base_course_summary(course_id) + + for row in group: + for field in SUMMARY_META_FIELDS: + summary[field] = _row_value(row, field) + + mode = _row_value(row, 'enrollment_mode') + summary['enrollment_modes'][mode] = {field: _count_value(row, field) for field in COUNT_FIELDS} + created = _row_value(row, 'created') + summary['created'] = max(created, summary['created']) if summary['created'] else created + summary.update({ + field: summary[field] + _count_value(row, field) + for field in COUNT_FIELDS + }) + + if recent_counts is not None: + summary['recent_count_change'] = summary['count'] - recent_counts.get(course_id, 0) + + if programs is not None: + summary['programs'] = programs.get(course_id, []) + + summaries.append(_postprocess_course_summary(summary, exclude=exclude)) + + return summaries diff --git a/analytics_data_api/insights_snowflake/mappers/programs.py b/analytics_data_api/insights_snowflake/mappers/programs.py new file mode 100644 index 00000000..07f984ed --- /dev/null +++ b/analytics_data_api/insights_snowflake/mappers/programs.py @@ -0,0 +1,42 @@ +"""Map Snowflake program metadata rows into the existing API response shape.""" + +from itertools import groupby + + +def _row_value(row, name): + """Return a row value from dictionary rows produced by the Snowflake client.""" + if name in row: + return row[name] + return row[name.upper()] + + +def map_program_metadata_rows(rows): + """Group program metadata rows into one API item per program.""" + rows = sorted( + rows or [], + key=lambda row: ( + _row_value(row, 'program_id') or '', + _row_value(row, 'course_id') or '', + ), + ) + programs = [] + + for program_id, group in groupby(rows, lambda row: _row_value(row, 'program_id')): + item = { + 'program_id': program_id, + 'program_type': '', + 'program_title': '', + 'created': None, + 'course_ids': [], + } + + for row in group: + item['program_type'] = _row_value(row, 'program_type') + item['program_title'] = _row_value(row, 'program_title') + item['course_ids'].append(_row_value(row, 'course_id')) + created = _row_value(row, 'created') + item['created'] = max(created, item['created']) if item['created'] else created + + programs.append(item) + + return programs diff --git a/analytics_data_api/insights_snowflake/queries/course_summaries.py b/analytics_data_api/insights_snowflake/queries/course_summaries.py new file mode 100644 index 00000000..096cd5d4 --- /dev/null +++ b/analytics_data_api/insights_snowflake/queries/course_summaries.py @@ -0,0 +1,100 @@ +"""Snowflake queries for course summary metadata.""" + +import datetime + +from analytics_data_api.insights_snowflake.client import fetch_all, get_qualified_table_name + +COURSE_META_SUMMARY_ENROLLMENT_TABLE = 'COURSE_META_SUMMARY_ENROLLMENT' +COURSE_PROGRAM_METADATA_TABLE = 'COURSE_PROGRAM_METADATA' +COURSE_ENROLLMENT_DAILY_TABLE = 'COURSE_ENROLLMENT_DAILY' + + +def _date_value(value): + """Return a date value for date-filtered Snowflake course summary queries.""" + if isinstance(value, datetime.datetime): + return value.date() + return value + + +def _in_filter(column_name, param_prefix, values, prefix='WHERE'): + """Return a parameterized Snowflake IN filter for controlled columns.""" + if not values: + return '', {} + + params = {} + placeholders = [] + for index, value in enumerate(values): + param_name = '{}_{}'.format(param_prefix, index) + params[param_name] = value + placeholders.append('%({})s'.format(param_name)) + + return '{} {} IN ({})'.format(prefix, column_name, ', '.join(placeholders)), params + + +def get_course_summary_rows(course_ids=None): + """Return Snowflake rows for course summary enrollment metadata.""" + table_name = get_qualified_table_name(COURSE_META_SUMMARY_ENROLLMENT_TABLE) + where_clause, params = _in_filter('course_id', 'course_id', course_ids) + sql = """ +SELECT + course_id, + catalog_course_title, + catalog_course, + start_time, + end_time, + pacing_type, + availability, + enrollment_mode, + "COUNT" AS count, + cumulative_count, + count_change_7_days, + passing_users, + created +FROM {table_name} +{where_clause} +ORDER BY course_id, enrollment_mode +""".format(table_name=table_name, where_clause=where_clause) + + return fetch_all(sql, params) + + +def get_course_summary_program_rows(course_ids=None): + """Return Snowflake program metadata rows for course summaries.""" + table_name = get_qualified_table_name(COURSE_PROGRAM_METADATA_TABLE) + where_clause, params = _in_filter('course_id', 'course_id', course_ids) + sql = """ +SELECT + course_id, + program_id, + program_type, + program_title, + created +FROM {table_name} +{where_clause} +ORDER BY course_id, program_id +""".format(table_name=table_name, where_clause=where_clause) + + return fetch_all(sql, params) + + +def get_course_recent_enrollment_rows(course_ids=None, recent_date=None): + """Return Snowflake course enrollment rows for the requested recent date.""" + table_name = get_qualified_table_name(COURSE_ENROLLMENT_DAILY_TABLE) + course_filter, course_params = _in_filter('course_id', 'course_id', course_ids, prefix='AND') + params = { + 'recent_date': _date_value(recent_date), + } + params.update(course_params) + sql = """ +SELECT + course_id, + "DATE" AS date, + "COUNT" AS count, + created +FROM {table_name} +WHERE "DATE" = %(recent_date)s +{course_filter} +ORDER BY course_id +""".format(table_name=table_name, course_filter=course_filter) + + return fetch_all(sql, params) diff --git a/analytics_data_api/insights_snowflake/queries/programs.py b/analytics_data_api/insights_snowflake/queries/programs.py new file mode 100644 index 00000000..0ba42e66 --- /dev/null +++ b/analytics_data_api/insights_snowflake/queries/programs.py @@ -0,0 +1,39 @@ +"""Snowflake queries for program metadata.""" + +from analytics_data_api.insights_snowflake.client import fetch_all, get_qualified_table_name + +COURSE_PROGRAM_METADATA_TABLE = 'COURSE_PROGRAM_METADATA' + + +def _in_filter(column_name, param_prefix, values): + """Return a parameterized Snowflake IN filter for controlled columns.""" + if not values: + return '', {} + + params = {} + placeholders = [] + for index, value in enumerate(values): + param_name = '{}_{}'.format(param_prefix, index) + params[param_name] = value + placeholders.append('%({})s'.format(param_name)) + + return 'WHERE {} IN ({})'.format(column_name, ', '.join(placeholders)), params + + +def get_program_metadata_rows(program_ids=None): + """Return Snowflake rows for course program metadata.""" + table_name = get_qualified_table_name(COURSE_PROGRAM_METADATA_TABLE) + where_clause, params = _in_filter('program_id', 'program_id', program_ids) + sql = """ +SELECT + program_id, + program_type, + program_title, + course_id, + created +FROM {table_name} +{where_clause} +ORDER BY program_id, course_id +""".format(table_name=table_name, where_clause=where_clause) + + return fetch_all(sql, params) diff --git a/analytics_data_api/insights_snowflake/service.py b/analytics_data_api/insights_snowflake/service.py index 4c07dd98..fadae102 100644 --- a/analytics_data_api/insights_snowflake/service.py +++ b/analytics_data_api/insights_snowflake/service.py @@ -1,6 +1,7 @@ """Service functions for Snowflake-backed Insights endpoints.""" from analytics_data_api.insights_snowflake.mappers.activity import map_course_activity_weekly_rows +from analytics_data_api.insights_snowflake.mappers.course_summaries import map_course_summary_rows from analytics_data_api.insights_snowflake.mappers.enrollment import ( map_course_enrollment_daily_rows, map_course_enrollment_education_rows, @@ -8,7 +9,13 @@ map_course_enrollment_location_rows, map_course_enrollment_mode_rows, ) +from analytics_data_api.insights_snowflake.mappers.programs import map_program_metadata_rows from analytics_data_api.insights_snowflake.queries.activity import get_course_activity_weekly_rows +from analytics_data_api.insights_snowflake.queries.course_summaries import ( + get_course_recent_enrollment_rows, + get_course_summary_program_rows, + get_course_summary_rows, +) from analytics_data_api.insights_snowflake.queries.enrollment import ( get_course_enrollment_daily_rows, get_course_enrollment_education_rows, @@ -16,6 +23,7 @@ get_course_enrollment_location_rows, get_course_enrollment_mode_rows, ) +from analytics_data_api.insights_snowflake.queries.programs import get_program_metadata_rows def get_course_activity_weekly(course_id, start_date=None, end_date=None): @@ -52,3 +60,26 @@ def get_course_enrollment_location(course_id, start_date=None, end_date=None): """Return course enrollment location counts in the existing API response shape.""" rows = get_course_enrollment_location_rows(course_id, start_date=start_date, end_date=end_date) return map_course_enrollment_location_rows(rows) + + +def get_program_metadata(program_ids=None): + """Return program metadata in the existing API response shape.""" + rows = get_program_metadata_rows(program_ids=program_ids) + return map_program_metadata_rows(rows) + + +def get_course_summaries(course_ids=None, include_programs=False, recent_date=None, exclude=None): + """Return course summaries in the existing API response shape.""" + summary_rows = get_course_summary_rows(course_ids=course_ids) + program_rows = get_course_summary_program_rows(course_ids=course_ids) if include_programs else None + recent_rows = get_course_recent_enrollment_rows( + course_ids=course_ids, + recent_date=recent_date, + ) if recent_date else None + + return map_course_summary_rows( + summary_rows, + program_rows=program_rows, + recent_rows=recent_rows, + exclude=exclude, + ) diff --git a/analytics_data_api/tests/test_insights_snowflake.py b/analytics_data_api/tests/test_insights_snowflake.py index 21d36ee2..d101ec3e 100644 --- a/analytics_data_api/tests/test_insights_snowflake.py +++ b/analytics_data_api/tests/test_insights_snowflake.py @@ -9,6 +9,7 @@ from analytics_data_api.constants import country, enrollment_modes, genders from analytics_data_api.insights_snowflake.client import fetch_all, get_qualified_table_name from analytics_data_api.insights_snowflake.mappers.activity import map_course_activity_weekly_rows +from analytics_data_api.insights_snowflake.mappers.course_summaries import map_course_summary_rows from analytics_data_api.insights_snowflake.mappers.enrollment import ( map_course_enrollment_daily_rows, map_course_enrollment_education_rows, @@ -16,7 +17,19 @@ map_course_enrollment_location_rows, map_course_enrollment_mode_rows, ) +from analytics_data_api.insights_snowflake.mappers.programs import map_program_metadata_rows from analytics_data_api.insights_snowflake.queries.activity import get_course_activity_weekly_rows +from analytics_data_api.insights_snowflake.queries.course_summaries import ( + COURSE_ENROLLMENT_DAILY_TABLE, + COURSE_META_SUMMARY_ENROLLMENT_TABLE, +) +from analytics_data_api.insights_snowflake.queries.course_summaries import \ + COURSE_PROGRAM_METADATA_TABLE as COURSE_SUMMARY_PROGRAM_METADATA_TABLE +from analytics_data_api.insights_snowflake.queries.course_summaries import ( + get_course_recent_enrollment_rows, + get_course_summary_program_rows, + get_course_summary_rows, +) from analytics_data_api.insights_snowflake.queries.enrollment import ( COURSE_ENROLLMENT_EDUCATION_LEVEL_CURRENT_TABLE, COURSE_ENROLLMENT_GENDER_DAILY_TABLE, @@ -27,6 +40,10 @@ get_course_enrollment_location_rows, get_course_enrollment_mode_rows, ) +from analytics_data_api.insights_snowflake.queries.programs import ( + COURSE_PROGRAM_METADATA_TABLE, + get_program_metadata_rows, +) from analytics_data_api.insights_snowflake.response_headers import ( DATA_SOURCE_HEADER, DATA_SOURCE_SNOWFLAKE, @@ -39,6 +56,8 @@ get_course_enrollment_gender, get_course_enrollment_location, get_course_enrollment_mode, + get_course_summaries, + get_program_metadata, ) from analytics_data_api.insights_snowflake.toggles import ( COURSE_ACTIVITY_SNOWFLAKE_FLAG, @@ -223,6 +242,121 @@ def test_enrollment_query_functions_use_expected_tables(self, mock_get_table_nam self.assertEqual(params, {'course_id': course_id}) +class InsightsSnowflakeCourseSummaryQueryTests(SimpleTestCase): + """Cover course summary query construction with mocked Snowflake execution.""" + + @patch('analytics_data_api.insights_snowflake.queries.course_summaries.fetch_all') + @patch( + 'analytics_data_api.insights_snowflake.queries.course_summaries.get_qualified_table_name', + Mock(return_value='PROD.INSIGHTS.COURSE_META_SUMMARY_ENROLLMENT') + ) + def test_get_course_summary_rows_uses_expected_table_without_ids(self, mock_fetch_all): + mock_fetch_all.return_value = [{'course_id': 'course-v1:edX+DemoX+Demo_Course'}] + + rows = get_course_summary_rows() + + self.assertEqual(rows, [{'course_id': 'course-v1:edX+DemoX+Demo_Course'}]) + sql, params = mock_fetch_all.call_args[0] + self.assertIn('FROM PROD.INSIGHTS.COURSE_META_SUMMARY_ENROLLMENT', sql) + self.assertNotIn('WHERE course_id IN', sql) + self.assertEqual(params, {}) + + @patch('analytics_data_api.insights_snowflake.queries.course_summaries.fetch_all') + @patch('analytics_data_api.insights_snowflake.queries.course_summaries.get_qualified_table_name') + def test_course_summary_query_functions_use_expected_tables(self, mock_get_table_name, _mock_fetch_all): + mock_get_table_name.return_value = 'PROD.INSIGHTS.COURSE_SUMMARY_TABLE' + course_ids = ['course-v1:edX+DemoX+Demo_Course', 'course-v1:edX+DemoX+Demo_2014'] + + query_functions = [ + (get_course_summary_rows, COURSE_META_SUMMARY_ENROLLMENT_TABLE), + (get_course_summary_program_rows, COURSE_SUMMARY_PROGRAM_METADATA_TABLE), + ] + + for query_function, table in query_functions: + mock_get_table_name.reset_mock() + _mock_fetch_all.reset_mock() + + query_function(course_ids=course_ids) + + mock_get_table_name.assert_called_once_with(table) + sql, params = _mock_fetch_all.call_args[0] + self.assertIn('course_id IN (%(course_id_0)s, %(course_id_1)s)', sql) + self.assertEqual(params, { + 'course_id_0': course_ids[0], + 'course_id_1': course_ids[1], + }) + + @patch('analytics_data_api.insights_snowflake.queries.course_summaries.fetch_all') + @patch( + 'analytics_data_api.insights_snowflake.queries.course_summaries.get_qualified_table_name', + Mock(return_value='PROD.INSIGHTS.COURSE_ENROLLMENT_DAILY') + ) + def test_get_course_recent_enrollment_rows_accepts_datetime_recent_date(self, mock_fetch_all): + recent_date = datetime.datetime(2014, 1, 1, tzinfo=datetime.timezone.utc) + + get_course_recent_enrollment_rows( + course_ids=['course-v1:edX+DemoX+Demo_Course'], + recent_date=recent_date, + ) + + sql, params = mock_fetch_all.call_args[0] + self.assertIn('FROM PROD.INSIGHTS.COURSE_ENROLLMENT_DAILY', sql) + self.assertIn('AND course_id IN (%(course_id_0)s)', sql) + self.assertEqual(params, { + 'recent_date': recent_date.date(), + 'course_id_0': 'course-v1:edX+DemoX+Demo_Course', + }) + + @patch('analytics_data_api.insights_snowflake.queries.course_summaries.fetch_all') + @patch('analytics_data_api.insights_snowflake.queries.course_summaries.get_qualified_table_name') + def test_get_course_recent_enrollment_rows_uses_expected_table(self, mock_get_table_name, _mock_fetch_all): + mock_get_table_name.return_value = 'PROD.INSIGHTS.COURSE_ENROLLMENT_DAILY' + recent_date = datetime.date(2014, 1, 1) + + get_course_recent_enrollment_rows(recent_date=recent_date) + + mock_get_table_name.assert_called_once_with(COURSE_ENROLLMENT_DAILY_TABLE) + sql, params = _mock_fetch_all.call_args[0] + self.assertNotIn('AND course_id IN', sql) + self.assertEqual(params, {'recent_date': recent_date}) + + +class InsightsSnowflakeProgramQueryTests(SimpleTestCase): + """Cover program metadata query construction with mocked Snowflake execution.""" + + @patch('analytics_data_api.insights_snowflake.queries.programs.fetch_all') + @patch( + 'analytics_data_api.insights_snowflake.queries.programs.get_qualified_table_name', + Mock(return_value='PROD.INSIGHTS.COURSE_PROGRAM_METADATA') + ) + def test_get_program_metadata_rows_uses_expected_table_without_ids(self, mock_fetch_all): + mock_fetch_all.return_value = [{'program_id': 'program-1'}] + + rows = get_program_metadata_rows() + + self.assertEqual(rows, [{'program_id': 'program-1'}]) + sql, params = mock_fetch_all.call_args[0] + self.assertIn('FROM PROD.INSIGHTS.COURSE_PROGRAM_METADATA', sql) + self.assertNotIn('WHERE program_id IN', sql) + self.assertEqual(params, {}) + + @patch('analytics_data_api.insights_snowflake.queries.programs.fetch_all') + @patch('analytics_data_api.insights_snowflake.queries.programs.get_qualified_table_name') + def test_get_program_metadata_rows_filters_program_ids(self, mock_get_table_name, _mock_fetch_all): + mock_get_table_name.return_value = 'PROD.INSIGHTS.COURSE_PROGRAM_METADATA' + program_ids = ['program-1', 'program-2'] + + get_program_metadata_rows(program_ids=program_ids) + + mock_get_table_name.assert_called_once_with(COURSE_PROGRAM_METADATA_TABLE) + sql, params = _mock_fetch_all.call_args[0] + self.assertIn('WHERE program_id IN (%(program_id_0)s, %(program_id_1)s)', sql) + self.assertEqual(params, { + 'program_id_0': program_ids[0], + 'program_id_1': program_ids[1], + }) + + class InsightsSnowflakeActivityMapperTests(SimpleTestCase): """Cover Snowflake activity row mapping into the existing API shape.""" @@ -456,6 +590,188 @@ def test_map_course_enrollment_location_rows_groups_unsorted_dates(self): self.assertEqual(mapped_rows[1].count, 4) +class InsightsSnowflakeProgramMapperTests(SimpleTestCase): + """Cover Snowflake program rows mapping into the existing API shape.""" + + def test_map_program_metadata_rows_groups_courses_and_accepts_uppercase_keys(self): + created = datetime.datetime(2014, 1, 2, tzinfo=datetime.timezone.utc) + later_created = datetime.datetime(2014, 1, 3, tzinfo=datetime.timezone.utc) + rows = [ + { + 'PROGRAM_ID': 'program-1', + 'PROGRAM_TYPE': 'Demo', + 'PROGRAM_TITLE': 'Test', + 'COURSE_ID': 'course-v1:edX+DemoX+Demo_2014', + 'CREATED': created, + }, + { + 'PROGRAM_ID': 'program-1', + 'PROGRAM_TYPE': 'Demo', + 'PROGRAM_TITLE': 'Test', + 'COURSE_ID': 'course-v1:edX+DemoX+Demo_Course', + 'CREATED': later_created, + }, + ] + + self.assertEqual(map_program_metadata_rows(rows), [{ + 'program_id': 'program-1', + 'program_type': 'Demo', + 'program_title': 'Test', + 'created': later_created, + 'course_ids': [ + 'course-v1:edX+DemoX+Demo_2014', + 'course-v1:edX+DemoX+Demo_Course', + ], + }]) + + def test_map_program_metadata_rows_handles_null_sort_values(self): + created = datetime.datetime(2014, 1, 2, tzinfo=datetime.timezone.utc) + rows = [ + { + 'program_id': 'program-1', + 'program_type': 'Demo', + 'program_title': 'Test', + 'course_id': 'course-v1:edX+DemoX+Demo_Course', + 'created': created, + }, + { + 'program_id': 'program-1', + 'program_type': 'Demo', + 'program_title': 'Test', + 'course_id': None, + 'created': created, + }, + ] + + self.assertEqual(map_program_metadata_rows(rows), [{ + 'program_id': 'program-1', + 'program_type': 'Demo', + 'program_title': 'Test', + 'created': created, + 'course_ids': [ + None, + 'course-v1:edX+DemoX+Demo_Course', + ], + }]) + + +class InsightsSnowflakeCourseSummaryMapperTests(SimpleTestCase): + """Cover Snowflake course summary rows mapping into the existing API shape.""" + + def test_map_course_summary_rows_merges_modes_programs_and_recent_counts(self): + course_id = 'course-v1:edX+DemoX+Demo_Course' + start_time = datetime.datetime(2016, 10, 11, tzinfo=datetime.timezone.utc) + end_time = datetime.datetime(2016, 12, 18, tzinfo=datetime.timezone.utc) + created = datetime.datetime(2014, 1, 2, tzinfo=datetime.timezone.utc) + later_created = datetime.datetime(2014, 1, 3, tzinfo=datetime.timezone.utc) + summary_rows = [ + { + 'COURSE_ID': course_id, + 'CATALOG_COURSE_TITLE': 'Title', + 'CATALOG_COURSE': 'Catalog', + 'START_TIME': start_time, + 'END_TIME': end_time, + 'PACING_TYPE': 'instructor', + 'AVAILABILITY': 'Starting Soon', + 'ENROLLMENT_MODE': enrollment_modes.PROFESSIONAL_NO_ID, + 'COUNT': 3, + 'CUMULATIVE_COUNT': 7, + 'COUNT_CHANGE_7_DAYS': 1, + 'PASSING_USERS': None, + 'CREATED': created, + }, + { + 'COURSE_ID': course_id, + 'CATALOG_COURSE_TITLE': 'Title', + 'CATALOG_COURSE': 'Catalog', + 'START_TIME': start_time, + 'END_TIME': end_time, + 'PACING_TYPE': 'instructor', + 'AVAILABILITY': 'Starting Soon', + 'ENROLLMENT_MODE': enrollment_modes.PROFESSIONAL, + 'COUNT': 4, + 'CUMULATIVE_COUNT': 8, + 'COUNT_CHANGE_7_DAYS': 2, + 'PASSING_USERS': 6, + 'CREATED': later_created, + }, + ] + program_rows = [{ + 'course_id': course_id, + 'program_id': 'program-1', + }] + recent_rows = [{ + 'course_id': course_id, + 'count': 2, + }] + + mapped_rows = map_course_summary_rows( + summary_rows, + program_rows=program_rows, + recent_rows=recent_rows, + exclude=['passing_users'], + ) + + self.assertEqual(len(mapped_rows), 1) + summary = mapped_rows[0] + self.assertEqual(summary['course_id'], course_id) + self.assertEqual(summary['availability'], 'Upcoming') + self.assertEqual(summary['created'], later_created) + self.assertEqual(summary['count'], 7) + self.assertEqual(summary['cumulative_count'], 15) + self.assertEqual(summary['count_change_7_days'], 3) + self.assertEqual(summary['passing_users'], 6) + self.assertEqual(summary['recent_count_change'], 5) + self.assertEqual(summary['programs'], ['program-1']) + self.assertNotIn(enrollment_modes.PROFESSIONAL_NO_ID, summary['enrollment_modes']) + self.assertEqual(summary['enrollment_modes'][enrollment_modes.PROFESSIONAL]['count'], 7) + self.assertNotIn('passing_users', summary['enrollment_modes'][enrollment_modes.PROFESSIONAL]) + + def test_map_course_summary_rows_handles_null_sort_values(self): + course_id = 'course-v1:edX+DemoX+Demo_Course' + start_time = datetime.datetime(2016, 10, 11, tzinfo=datetime.timezone.utc) + end_time = datetime.datetime(2016, 12, 18, tzinfo=datetime.timezone.utc) + created = datetime.datetime(2014, 1, 2, tzinfo=datetime.timezone.utc) + summary_rows = [ + { + 'course_id': course_id, + 'catalog_course_title': 'Title', + 'catalog_course': 'Catalog', + 'start_time': start_time, + 'end_time': end_time, + 'pacing_type': 'instructor', + 'availability': 'Current', + 'enrollment_mode': enrollment_modes.PROFESSIONAL, + 'count': 4, + 'cumulative_count': 8, + 'count_change_7_days': 2, + 'passing_users': 6, + 'created': created, + }, + { + 'course_id': course_id, + 'catalog_course_title': 'Title', + 'catalog_course': 'Catalog', + 'start_time': start_time, + 'end_time': end_time, + 'pacing_type': 'instructor', + 'availability': 'Current', + 'enrollment_mode': None, + 'count': 3, + 'cumulative_count': 7, + 'count_change_7_days': 1, + 'passing_users': 0, + 'created': created, + }, + ] + + mapped_rows = map_course_summary_rows(summary_rows) + + self.assertEqual(len(mapped_rows), 1) + self.assertEqual(mapped_rows[0]['count'], 7) + self.assertEqual(mapped_rows[0]['enrollment_modes'][enrollment_modes.PROFESSIONAL]['count'], 4) + + class InsightsSnowflakeServiceTests(SimpleTestCase): """Cover service orchestration without real Snowflake calls.""" @@ -546,6 +862,84 @@ def test_get_course_enrollment_location_calls_query_and_mapper(self): 'analytics_data_api.insights_snowflake.service.map_course_enrollment_location_rows', ) + @patch('analytics_data_api.insights_snowflake.service.map_program_metadata_rows') + @patch('analytics_data_api.insights_snowflake.service.get_program_metadata_rows') + def test_get_program_metadata_calls_query_and_mapper(self, mock_get_rows, mock_map_rows): + raw_rows = [{'program_id': 'program-1'}] + mapped_rows = [{'program_id': 'program-1', 'course_ids': ['course-v1:edX+DemoX+Demo_Course']}] + mock_get_rows.return_value = raw_rows + mock_map_rows.return_value = mapped_rows + + self.assertEqual(get_program_metadata(program_ids=['program-1']), mapped_rows) + + mock_get_rows.assert_called_once_with(program_ids=['program-1']) + mock_map_rows.assert_called_once_with(raw_rows) + + @patch('analytics_data_api.insights_snowflake.service.map_course_summary_rows') + @patch('analytics_data_api.insights_snowflake.service.get_course_recent_enrollment_rows') + @patch('analytics_data_api.insights_snowflake.service.get_course_summary_program_rows') + @patch('analytics_data_api.insights_snowflake.service.get_course_summary_rows') + def test_get_course_summaries_calls_required_queries_and_mapper( + self, + mock_get_summary_rows, + mock_get_program_rows, + mock_get_recent_rows, + mock_map_rows, + ): + course_ids = ['course-v1:edX+DemoX+Demo_Course'] + recent_date = datetime.date(2014, 1, 1) + summary_rows = [{'course_id': course_ids[0]}] + program_rows = [{'course_id': course_ids[0], 'program_id': 'program-1'}] + recent_rows = [{'course_id': course_ids[0], 'count': 3}] + mapped_rows = [{'course_id': course_ids[0], 'count': 4}] + mock_get_summary_rows.return_value = summary_rows + mock_get_program_rows.return_value = program_rows + mock_get_recent_rows.return_value = recent_rows + mock_map_rows.return_value = mapped_rows + + self.assertEqual( + get_course_summaries( + course_ids=course_ids, + include_programs=True, + recent_date=recent_date, + exclude=['created'], + ), + mapped_rows, + ) + + mock_get_summary_rows.assert_called_once_with(course_ids=course_ids) + mock_get_program_rows.assert_called_once_with(course_ids=course_ids) + mock_get_recent_rows.assert_called_once_with(course_ids=course_ids, recent_date=recent_date) + mock_map_rows.assert_called_once_with( + summary_rows, + program_rows=program_rows, + recent_rows=recent_rows, + exclude=['created'], + ) + + @patch('analytics_data_api.insights_snowflake.service.map_course_summary_rows') + @patch('analytics_data_api.insights_snowflake.service.get_course_recent_enrollment_rows') + @patch('analytics_data_api.insights_snowflake.service.get_course_summary_program_rows') + @patch('analytics_data_api.insights_snowflake.service.get_course_summary_rows') + def test_get_course_summaries_skips_optional_queries( + self, + mock_get_summary_rows, + mock_get_program_rows, + mock_get_recent_rows, + mock_map_rows, + ): + summary_rows = [{'course_id': 'course-v1:edX+DemoX+Demo_Course'}] + mapped_rows = [{'course_id': 'course-v1:edX+DemoX+Demo_Course', 'count': 4}] + mock_get_summary_rows.return_value = summary_rows + mock_map_rows.return_value = mapped_rows + + self.assertEqual(get_course_summaries(), mapped_rows) + + mock_get_summary_rows.assert_called_once_with(course_ids=None) + mock_get_program_rows.assert_not_called() + mock_get_recent_rows.assert_not_called() + mock_map_rows.assert_called_once_with(summary_rows, program_rows=None, recent_rows=None, exclude=None) + class BaseTestView: """Small base class for testing response mixin behavior.""" diff --git a/analytics_data_api/v0/tests/views/test_course_summaries.py b/analytics_data_api/v0/tests/views/test_course_summaries.py index 7cb603c1..bd545fc2 100644 --- a/analytics_data_api/v0/tests/views/test_course_summaries.py +++ b/analytics_data_api/v0/tests/views/test_course_summaries.py @@ -1,4 +1,5 @@ import datetime +from unittest.mock import patch import ddt import pytz @@ -7,6 +8,7 @@ from django_dynamic_fixture import G from analytics_data_api.constants import enrollment_modes +from analytics_data_api.middleware import thread_data from analytics_data_api.tests.test_utils import set_databases from analytics_data_api.v0 import models, serializers from analytics_data_api.v0.tests.views import APIListViewTestMixin, CourseSamples, VerifyCourseIdMixin @@ -15,7 +17,11 @@ @ddt.ddt @set_databases -class CourseSummariesViewTests(VerifyCourseIdMixin, TestCaseWithAuthentication, APIListViewTestMixin): +class CourseSummariesViewTests( # pylint: disable=too-many-public-methods + VerifyCourseIdMixin, + TestCaseWithAuthentication, + APIListViewTestMixin, +): model = models.CourseMetaSummaryEnrollment model_id = 'course_id' ids_param = 'course_ids' @@ -28,11 +34,15 @@ class CourseSummariesViewTests(VerifyCourseIdMixin, TestCaseWithAuthentication, def setUp(self): super().setUp() + if hasattr(thread_data, 'analyticsapi_database'): + del thread_data.analyticsapi_database self.now = timezone.now() self.maxDiff = None def tearDown(self): self.model.objects.all().delete() + if hasattr(thread_data, 'analyticsapi_database'): + del thread_data.analyticsapi_database def create_model(self, model_id, **kwargs): modes = kwargs.get('modes', []) @@ -114,6 +124,17 @@ def expected_result(self, item_id, modes=None, availability='Current', programs= summary['programs'] = [CourseSamples.program_ids[0]] return summary + def snowflake_summary(self, course_id, programs=False, recent_count_change=None): + """Expected Snowflake summary data before DRF serializer formatting.""" + summary = self.expected_result(course_id, programs=programs, recent_count_change=recent_count_change) + summary.update({ + 'start_time': datetime.datetime(2016, 10, 11, tzinfo=pytz.utc), + 'end_time': datetime.datetime(2016, 12, 18, tzinfo=pytz.utc), + }) + summary.pop('start_date') + summary.pop('end_date') + return summary + def all_expected_results(self, # pylint: disable=arguments-differ ids=None, modes=None, @@ -255,3 +276,68 @@ def test_recent_count_change(self): responseBeforeDate = self.validated_request(exclude=self.always_exclude, recent_date=before) self.assertEqual(responseBeforeDate.status_code, 200) self.assertCountEqual(responseBeforeDate.data, expectedBeforeDate) + + def test_get_uses_aurora_when_global_snowflake_flag_disabled(self): + course_id = CourseSamples.course_ids[1] + self.generate_data(ids=[course_id]) + + with patch('analytics_data_api.v0.views.course_summaries.is_insights_snowflake_enabled', return_value=False): + with patch('analytics_data_api.v0.views.course_summaries.get_course_summaries') as mock_get_summaries: + response = self.authenticated_get( + self.path({self.ids_param: [course_id], 'exclude': ['created']}) + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data, [self.expected_result(course_id)]) + self.assertEqual(response['X-Insights-Data-Source'], 'aurora') + mock_get_summaries.assert_not_called() + + def test_get_uses_snowflake_service_when_global_flag_enabled(self): + course_id = CourseSamples.course_ids[1] + snowflake_data = [self.snowflake_summary(course_id)] + + with patch('analytics_data_api.v0.views.course_summaries.is_insights_snowflake_enabled', return_value=True): + with patch( + 'analytics_data_api.v0.views.course_summaries.get_course_summaries', + return_value=snowflake_data, + ) as mock_get_summaries: + response = self.authenticated_get( + self.path({self.ids_param: [course_id], 'exclude': ['created']}).replace('/api/v0/', '/api/v1/') + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data, [self.expected_result(course_id)]) + self.assertEqual(response['X-Insights-Data-Source'], 'snowflake') + mock_get_summaries.assert_called_once_with( + course_ids=[course_id], + include_programs=False, + recent_date=None, + exclude=['programs', 'created'], + ) + + def test_post_uses_snowflake_service_with_programs_and_recent_date(self): + course_id = CourseSamples.course_ids[1] + recent = (datetime.datetime.today() - datetime.timedelta(5)).strftime('%Y-%m-%d') + snowflake_data = [self.snowflake_summary(course_id, programs=True, recent_count_change=5)] + + with patch('analytics_data_api.v0.views.course_summaries.is_insights_snowflake_enabled', return_value=True): + with patch( + 'analytics_data_api.v0.views.course_summaries.get_course_summaries', + return_value=snowflake_data, + ) as mock_get_summaries: + response = self.authenticated_post('/api/v1/course_summaries/', data={ + 'course_ids': [course_id], + 'exclude': ['created'], + 'programs': ['True'], + 'recent_date': [recent], + }) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data, [self.expected_result(course_id, programs=True, recent_count_change=5)]) + self.assertEqual(response['X-Insights-Data-Source'], 'snowflake') + + _args, kwargs = mock_get_summaries.call_args + self.assertEqual(kwargs['course_ids'], [course_id]) + self.assertTrue(kwargs['include_programs']) + self.assertEqual(kwargs['recent_date'].strftime('%Y-%m-%d'), recent) + self.assertEqual(kwargs['exclude'], ['created']) diff --git a/analytics_data_api/v0/tests/views/test_programs.py b/analytics_data_api/v0/tests/views/test_programs.py index 7caa93a2..58b7fc32 100644 --- a/analytics_data_api/v0/tests/views/test_programs.py +++ b/analytics_data_api/v0/tests/views/test_programs.py @@ -1,4 +1,5 @@ import datetime +from unittest.mock import patch import ddt from django_dynamic_fixture import G @@ -106,3 +107,38 @@ def test_all_programs_multi_courses(self, program_ids, course_ids): response = self.validated_request(ids=program_ids, exclude=self.always_exclude) self.assertEqual(response.status_code, 200) self.assertCountEqual(response.data, self.all_expected_results(ids=program_ids, course_ids=course_ids)) + + def test_get_uses_aurora_when_global_snowflake_flag_disabled(self): + program_id = CourseSamples.program_ids[0] + self.generate_data(ids=[program_id]) + + with patch('analytics_data_api.v0.views.programs.is_insights_snowflake_enabled', return_value=False): + with patch('analytics_data_api.v0.views.programs.get_program_metadata') as mock_get_program_metadata: + response = self.authenticated_get(f'/api/v0/programs/?program_ids={program_id}&exclude=created') + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data, [self.expected_result(program_id)]) + self.assertEqual(response['X-Insights-Data-Source'], 'aurora') + mock_get_program_metadata.assert_not_called() + + def test_get_uses_snowflake_service_when_global_flag_enabled(self): + program_id = CourseSamples.program_ids[0] + snowflake_data = [{ + 'program_id': program_id, + 'program_type': 'Demo', + 'program_title': 'Test', + 'created': self.now, + 'course_ids': [self.course_id], + }] + + with patch('analytics_data_api.v0.views.programs.is_insights_snowflake_enabled', return_value=True): + with patch( + 'analytics_data_api.v0.views.programs.get_program_metadata', + return_value=snowflake_data, + ) as mock_get_program_metadata: + response = self.authenticated_get(f'/api/v1/programs/?program_ids={program_id}&exclude=created') + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data, [self.expected_result(program_id)]) + self.assertEqual(response['X-Insights-Data-Source'], 'snowflake') + mock_get_program_metadata.assert_called_once_with(program_ids=[program_id]) diff --git a/analytics_data_api/v0/views/course_summaries.py b/analytics_data_api/v0/views/course_summaries.py index f19aa381..a5290529 100644 --- a/analytics_data_api/v0/views/course_summaries.py +++ b/analytics_data_api/v0/views/course_summaries.py @@ -3,15 +3,18 @@ from itertools import groupby from django.db.models import Q -from django.http import HttpResponseBadRequest +from django.http import Http404, HttpResponseBadRequest from analytics_data_api.constants import enrollment_modes +from analytics_data_api.insights_snowflake.response_headers import InsightsDataSourceResponseMixin +from analytics_data_api.insights_snowflake.service import get_course_summaries +from analytics_data_api.insights_snowflake.toggles import is_insights_snowflake_enabled from analytics_data_api.v0 import models, serializers from analytics_data_api.v0.views import APIListView from analytics_data_api.v0.views.utils import split_query_argument, validate_course_id -class CourseSummariesView(APIListView): +class CourseSummariesView(InsightsDataSourceResponseMixin, APIListView): """ Returns summary information for courses. @@ -243,3 +246,25 @@ def add_recent_count_change(self, field_dict, recents): def get_query(self): return functools_reduce(lambda q, item_id: q | Q(course_id=item_id), self.ids, Q()) + + def get_snowflake_queryset(self): + """Return Snowflake-backed course summary data.""" + fetch_recents = self.recent_date and 'recent_count_change' not in self.exclude + fetch_programs = self.exclude == [] or (self.exclude and 'programs' not in self.exclude) + data = get_course_summaries( + course_ids=self.ids, + include_programs=fetch_programs, + recent_date=self.recent_date if fetch_recents else None, + exclude=self.exclude, + ) + if data: + return data + raise Http404 + + def get_queryset(self): + if is_insights_snowflake_enabled(self.request): + self.set_insights_data_source_snowflake() + return self.get_snowflake_queryset() + + self.set_insights_data_source_aurora() + return super().get_queryset() diff --git a/analytics_data_api/v0/views/programs.py b/analytics_data_api/v0/views/programs.py index b0579a2b..fc8abeed 100644 --- a/analytics_data_api/v0/views/programs.py +++ b/analytics_data_api/v0/views/programs.py @@ -1,12 +1,16 @@ from functools import reduce as functools_reduce from django.db.models import Q +from django.http import Http404 +from analytics_data_api.insights_snowflake.response_headers import InsightsDataSourceResponseMixin +from analytics_data_api.insights_snowflake.service import get_program_metadata +from analytics_data_api.insights_snowflake.toggles import is_insights_snowflake_enabled from analytics_data_api.v0 import models, serializers from analytics_data_api.v0.views import APIListView -class ProgramsView(APIListView): +class ProgramsView(InsightsDataSourceResponseMixin, APIListView): """ Returns metadata information for programs. @@ -63,3 +67,18 @@ def update_field_dict_from_model(self, model, base_field_dict=None, field_list=N def get_query(self): return functools_reduce(lambda q, item_id: q | Q(program_id=item_id), self.ids, Q()) + + def get_snowflake_queryset(self): + """Return Snowflake-backed program metadata.""" + data = get_program_metadata(program_ids=self.ids) + if data: + return data + raise Http404 + + def get_queryset(self): + if is_insights_snowflake_enabled(self.request): + self.set_insights_data_source_snowflake() + return self.get_snowflake_queryset() + + self.set_insights_data_source_aurora() + return super().get_queryset()