Merge branch 'improve-tasks'

This set of changes optimizes exporters, particularly `export.tasks`, to
greatly improve stability, performance, and reliability of export jobs.
Testing in our staging environment shows a 4-5X throughput with the new
tasks.

Note that these are a bit more memory-intensive than the previous tasks,
so you may need to use the new EXPORTER_MAX_RC_CONFIG and
EXPORTER_MAX_DC_CONFIG settings to scale how large job chunks are.

Author: jthomale

README.md

@@ -899,6 +899,19 @@ circumstances. If the variable is not set, the default value is used.
     will match up with a `config.properties` file in
     `<project_root>/solr/solrmarc`. (See "SolrMarc Configuration," below,
     for more information.) Default is `dev_config.properties`.
+    * `EXPORTER_MAX_RC_CONFIG` and `EXPORTER_MAX_DC_CONFIG` &mdash; These two
+    settings allow you to set overrides for the `max_rec_chunk` and
+    `max_del_chunk` attributes of `Exporter` objects. They are totally
+    optional; by default whatever value is set on the class is what will be
+    used, if a specific override is not set. However, depending on how your
+    development, production, staging, and testing environments are set up,
+    you may need (e.g.) your development settings scaled back compared to
+    your staging and production settings. This lets you configure that on
+    an env-specific basis. Do note that the convention used for the settings
+    as in your .env file looks like this:
+
+          EXPORTER_MAX_RC_CONFIG="ItemsToSolr:1000,BibsToSolr:500"
+
 * Production Settings &mdash; These are settings you'll probably only need to
 set in production. If your development environment is very different than
 the default setup, then you may need to set these there as well.

django/sierra/base/managers.py

@@ -1,6 +1,6 @@
-'''
+"""
 Custom Managers for sierra base app models.
-'''
+"""
 from datetime import date, time, datetime
 
 from django.db import models
@@ -10,7 +10,7 @@
 
 
 class CustomFilterManager(models.Manager):
-    '''
+    """
     A generic models.Manager class that provides the ability to set
     custom filters easily. Just create a child class, and then create a
     method with the same name as the filter that you pass to filter_by.
@@ -20,13 +20,13 @@ class CustomFilterManager(models.Manager):
     set. For instance: [{a AND b AND c} OR {d} OR {e AND f}]. Order_by
     should be an array that can be passed as arguments to
     queryset.order_by().
-    '''
+    """
     options = {}
 
     def _apply_filter(self, filter_method):
-        '''
+        """
         Applies the filter_method and returns the filtered queryset.
-        '''
+        """
         filter_params = filter_method()
         filter = filter_params['filter']
         order_by = filter_params['order_by']
@@ -39,29 +39,31 @@ def _apply_filter(self, filter_method):
             set = set.order_by(*order_by)
         return set
 
-    def filter_by(self, filter_method, options={}):
-        '''
+    def filter_by(self, filter_method, options=None):
+        """
         Fetches a set of records based on a filter string and any
         options you specify. Options should be a dictionary.
-        '''
-        self.options = options
+        """
+        self.options = options or {}
         filter_method = getattr(self, filter_method)
         return self._apply_filter(filter_method)
 
 
 class RecordManager(CustomFilterManager):
-    '''
+    """
     Defines some common filters that apply across multiple types of
     records from the Sierra database, such as the base record types
     (item, bib, patron, etc.)
-    '''
+    """
 
     def updated_date_range(self):
-        '''
+        """
         Filter by a date range for last_updated. Options should contain
-        date_range_from and date_range_to, each of which are simply date
-        objects.
-        '''
+        date_range_from and date_range_to, each of which are simply
+        date objects. Options *may* contain `is_deletion`, which is a
+        boolean that indicates whether or not this requires "last
+        deleted" rather than "last updated".
+        """
         options = self.options
         date_from = datetime.combine(options['date_range_from'], time(0, 0))
         date_from = tz.make_aware(date_from, tz.get_default_timezone())
@@ -74,25 +76,25 @@ def updated_date_range(self):
             prefix = 'record_metadata__'
         else:
             prefix = ''
-        filter = [
-            {
-                '{}record_last_updated_gmt__gte'.format(prefix): date_from,
-                '{}record_last_updated_gmt__lte'.format(prefix): date_to
-            },
-            {
+        if options.get('is_deletion', False):
+            filter = [{
                 '{}deletion_date_gmt__gte'.format(prefix): date_from,
                 '{}deletion_date_gmt__lte'.format(prefix): date_to,
-            }
-        ]
-
-        order_by = ['{}record_last_updated_gmt'.format(prefix)]
+            }]
+            order_by = ['{}deletion_date_gmt'.format(prefix)]
+        else:
+            filter = [{
+                '{}record_last_updated_gmt__gte'.format(prefix): date_from,
+                '{}record_last_updated_gmt__lte'.format(prefix): date_to
+            }]
+            order_by = ['{}record_last_updated_gmt'.format(prefix)]
         return {'filter': filter, 'order_by': order_by}
 
     def record_range(self):
-        '''
+        """
         Filter by a III record number range. Options should contain
         record_range_from and record_range_to.
-        '''
+        """
         options = self.options
         record_from = options['record_range_from']
         record_to = options['record_range_to']
@@ -109,30 +111,31 @@ def record_range(self):
         return {'filter': filter, 'order_by': order_by}
 
     def last_export(self):
-        '''
+        """
         Filter by a latest updated datetime in options['latest_time'].
-        '''
+        """
         options = self.options
         latest_time = options['latest_time']
         if self.model._meta.object_name != 'RecordMetadata':
             prefix = 'record_metadata__'
         else:
             prefix = ''
-        filter = [
-            {
-                '{}record_last_updated_gmt__gte'.format(prefix): latest_time
-            },
-            {
+        if options.get('is_deletion', False):
+            filter = [{
                 '{}deletion_date_gmt__gte'.format(prefix): latest_time
-            }
-        ]
-        order_by = ['{}record_last_updated_gmt'.format(prefix)]
+            }]
+            order_by = ['{}record_last_updated_gmt'.format(prefix)]
+        else:
+            filter = [{
+                '{}record_last_updated_gmt__gte'.format(prefix): latest_time
+            }]
+            order_by = ['{}deletion_date_gmt'.format(prefix)]
         return {'filter': filter, 'order_by': order_by}
 
     def full_export(self):
-        '''
+        """
         No filter.
-        '''
+        """
         if self.model._meta.object_name != 'RecordMetadata':
             prefix = 'record_metadata__'
         else:
@@ -142,9 +145,9 @@ def full_export(self):
         return {'filter': filter, 'order_by': order_by}
 
     def location(self):
-        '''
+        """
         Filters item records by location (code).
-        '''
+        """
         options = self.options
         location_code = self.options['location_code']
         if self.model._meta.object_name == 'RecordMetadata':

django/sierra/export/admin.py

@@ -6,13 +6,13 @@
 
 from .models import ExportType, ExportFilter, ExportInstance, Status
 from .forms.modelforms import ExportForm
-from .tasks import trigger_export
+from .tasks import export_dispatch
 
 def process_export_form(request):
-    '''
+    """
     Takes a request object and validates/parses/processes POSTed form
     data from an ExportForm. Returns the validated form object.
-    '''
+    """
     form = ExportForm(request.POST)
     post = request.POST
     if form.is_valid():
@@ -21,7 +21,7 @@ def process_export_form(request):
         export_instance.user = request.user
         export_instance.filter_params = params
         export_instance.timestamp = tz.now()
-        export_instance.status = Status.objects.get(pk='in_progress')
+        export_instance.status = Status.objects.get(pk='waiting')
         export_instance.save()
     return form
 
@@ -68,8 +68,8 @@ def add_view(self, request, form_url='', extra_content=None):
                 export_type = data['export_type'].pk
                 del data['export_filter']
                 del data['export_type']
-                trigger_export(form.instance, export_filter,
-                               export_type, data)
+                export_dispatch(form.instance.pk, export_filter, export_type,
+                                data)
                 reverse_url = 'admin:{}_{}_change'.format(
                     self.model._meta.app_label,
                     self.model._meta.module_name
@@ -91,4 +91,4 @@ class StatusAdmin(admin.ModelAdmin):
 admin.site.register(ExportType, ExportTypeAdmin)
 admin.site.register(ExportFilter, ExportFilterAdmin)
 admin.site.register(ExportInstance, ExportInstanceAdmin)
-admin.site.register(Status, StatusAdmin)
+admin.site.register(Status, StatusAdmin)