Move Solrmarc conversion into custom Solr backend

As we're working on additional indexes fed by Solrmarc (for Blacklight),
it started making more sense to move the Solrmarc conversion process
into a lower-level class so that we can have more uniformity at the
index and exporter levels. This makes that change. It also means we can
remove some cruft from the solr/solrmarc project directory.

Author: jthomale

django/sierra/base/search_indexes.py

@@ -14,6 +14,7 @@
 
 from haystack import indexes, constants, utils, exceptions
 
+from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
 
 from export import sierra2marc as s2m
@@ -200,7 +201,20 @@ def full_prepare(self, obj):
             return self.prepared_data
 
 
-class BibIndex(CustomQuerySetIndex, indexes.Indexable):
+class SolrmarcIndex(CustomQuerySetIndex):
+    """
+    Extends `CustomQuerySetIndex` with a few class attributes for the
+    SolrmarcIndexBackend, for indexing data via Solrmarc. See the
+    `sierra.solr_backend.SolrmarcIndexBackend` docstring for more info.
+    """
+
+    s2marc_class = s2m.S2MarcBatch
+    index_properties = None
+    config_file = None
+    temp_filedir = None
+
+
+class BibIndex(SolrmarcIndex, indexes.Indexable):
     """
     WARNING: This is a total hack to force Haystack to register our
     BibRecord model as being indexed by Haystack. This is the only way

django/sierra/export/basic_exporters.py

@@ -8,19 +8,13 @@
 """
 from __future__ import unicode_literals
 import logging
-import re
-import subprocess
-import os
-from collections import OrderedDict
 
 from django.conf import settings
 
 from base import models as sierra_models
 from base import search_indexes as indexes
-from export import models as export_models
 from export.exporter import (Exporter, ToSolrExporter, MetadataToSolrExporter,
                              CompoundMixin, AttachedRecordExporter)
-from export.sierra2marc import S2MarcError, S2MarcBatch
 from utils import helpers, redisobjs, solr
 
 # set up logger, for debugging
@@ -350,7 +344,13 @@ def final_callback(self, vals=None, status='success'):
 
 class BibsDownloadMarc(Exporter):
     """
-    Defines processes that convert Sierra bib records to MARC.
+    This exporter is now deprecated--please do not use.
+
+    Previously this defined processes that convert Sierra bib records
+    to MARC, but now that is handled through a custom Solr backend for
+    Haystack.
+
+    `BibsDownloadMarc` will be removed in the version 1.5.
     """
     max_rec_chunk = 2000
     parallel = False
@@ -369,6 +369,15 @@ class BibsDownloadMarc(Exporter):
     ]
     select_related = ['record_metadata', 'record_metadata__record_type']
 
+    def _warn(self):
+        msg = ('The `BibsDownloadMarc` exporter is deprecated and will be '
+               'removed in version 1.5.')
+        self.log('Warning', msg)
+
+    def __init__(self, *args, **kwargs):
+        super(BibsDownloadMarc, self).__init__(*args, **kwargs)
+        self._warn()
+
     def export_records(self, records):
         batch = S2MarcBatch(records)
         out_recs = batch.to_marc()
@@ -382,17 +391,15 @@ def export_records(self, records):
         return { 'marcfile': filename }
 
 
-class BibsToSolr(CompoundMixin, ToSolrExporter):
+class BibsToSolr(ToSolrExporter):
     """
     Defines processes that export Sierra/MARC bibs out to Solr.
     """
     Index = ToSolrExporter.Index
-    Child = CompoundMixin.Child
     index_config = (
         Index('Bibs', indexes.BibIndex, SOLR_CONNS['BibsToSolr:BIBS']),
         Index('MARC', indexes.MarcIndex, SOLR_CONNS['BibsToSolr:MARC'])
     )
-    children_config = (Child('BibsDownloadMarc'),)
     model = sierra_models.BibRecord
     deletion_filter = [
         {
@@ -401,57 +408,19 @@ class BibsToSolr(CompoundMixin, ToSolrExporter):
         }
     ]
     max_rec_chunk = 2000
-
-    @property
-    def prefetch_related(self):
-        return self.children['BibsDownloadMarc'].prefetch_related
-
-    @property
-    def select_related(self):
-        return self.children['BibsDownloadMarc'].select_related
-
-    def export_records(self, records):
-        cmd = 'bash'
-        index_script = settings.SOLRMARC_COMMAND
-        config_file = settings.SOLRMARC_CONFIG_FILE
-        filedir = settings.MEDIA_ROOT
-        if filedir[-1] != '/':
-            filedir = '{}/'.format(filedir)
-        bib_converter = self.children['BibsDownloadMarc']
-        converter_vals = bib_converter.export_records(records)
-        filename = converter_vals['marcfile']
-        filepath = '{}{}'.format(filedir, filename)
-        try:
-            output = subprocess.check_output([cmd, index_script, config_file,
-                                             filepath], 
-                                             stderr=subprocess.STDOUT,
-                                             shell=False,
-                                             universal_newlines=True)
-            output = output.decode('unicode-escape')
-        except subprocess.CalledProcessError as e:
-            error_lines = e.output.split("\n")
-            for line in error_lines:
-                self.log('Error', line)
-            self.log('Error', 'Solrmarc process did not run successfully.')
-        else:
-            error_lines = output.split("\n")
-            del(error_lines[-1])
-            if error_lines:
-                for line in error_lines:
-                    line = re.sub(r'^\s+', '', line)
-                    if re.match(r'^WARN', line):
-                        self.log('Warning', line)
-                    elif re.match(r'^ERROR', line):
-                        self.log('Warning', line)
-
-            # if all went well, we now try to index the MARC record
-            try:
-                self.indexes['MARC'].do_update(records)
-            except Exception as e:
-                self.log_error(e)
-
-        # delete the file when we're done so we don't take up space
-        os.remove(filepath)
+    prefetch_related = [
+        'record_metadata__varfield_set',
+        'bibrecorditemrecordlink_set',
+        'bibrecorditemrecordlink_set__item_record',
+        'bibrecorditemrecordlink_set__item_record__record_metadata',
+        'bibrecorditemrecordlink_set__item_record__record_metadata'
+            '__record_type',
+        'bibrecordproperty_set',
+        'bibrecordproperty_set__material__materialpropertyname_set',
+        'bibrecordproperty_set__material__materialpropertyname_set'
+            '__iii_language',
+    ]
+    select_related = ['record_metadata', 'record_metadata__record_type']
 
 
 class ItemsBibsToSolr(AttachedRecordExporter):

django/sierra/export/exporter.py

@@ -400,14 +400,20 @@ def indexes(self):
             self._indexes = type(self).spawn_indexes(self.export_type)
         return self._indexes
 
+    def handle_error(self, obj_str, error):
+        if obj_str == 'ERROR':
+            raise error
+        obj_info = '' if obj_str == 'WARNING' else '{} '.format(obj_str)
+        msg = '{} update skipped due to error: {}'.format(obj_info, error)
+        self.log('Warning', msg)
+
     def export_records(self, records):
         for index in self.indexes.values():
             index.do_update(records)
 
         for index in self.indexes.values():
             for obj_str, e in index.last_batch_errors:
-                msg = '{} update skipped due to error: {}'.format(obj_str, e)
-                self.log('Warning', msg)
+                self.handle_error(obj_str, e)
 
     def delete_records(self, records):
         for index in self.indexes.values():

django/sierra/sierra/settings/base.py

@@ -291,15 +291,15 @@ def raise_setting_error(setting):
         'TIMEOUT': 60 * 20,
     },
     'bibdata': {
-        'ENGINE': 'sierra.solr_backend.CustomSolrEngine',
+        'ENGINE': 'sierra.solr_backend.SolrmarcEngine',
         'URL': solr_bibdata_url,
         'TIMEOUT': 60 * 20,
     },
     'marc': {
         'ENGINE': 'sierra.solr_backend.CustomSolrEngine',
         'URL': solr_marc_url,
         'TIMEOUT': 60 * 20,
-    },
+    }
 }
 
 # HAYSTACK_LIMIT_TO_REGISTERED_MODELS, set to False to allow Haystack

django/sierra/sierra/solr_backend.py

@@ -14,8 +14,13 @@
 2. Overrides the SolrSearchBackend.clear() method so that the Solr
 index optimization isn't triggered if commit is false.
 """
+import subprocess
+import os
+import shlex
+import re
 
 from django.apps import apps
+from django.conf import settings
 
 from haystack.backends import solr_backend, BaseEngine
 from haystack.models import SearchResult
@@ -155,4 +160,110 @@ def clear(self, models=[], commit=True):
 
 class CustomSolrEngine(BaseEngine):
     backend = CustomSolrSearchBackend
-    query = solr_backend.SolrSearchQuery
+    query = solr_backend.SolrSearchQuery
+
+
+class SolrmarcIndexBackend(CustomSolrSearchBackend):
+    """
+    This is a custom Solr backend class for Haystack(ish) indexes that
+    implements doing index updates via Solrmarc. All of the code here
+    is derived from the code that was part of the `BibsDownloadMarc`
+    `BibsToSolr` exporters (in `export.basic_exporters`). As we're
+    working on additional indexes fed by Solrmarc (for Blacklight),
+    it started to make more sense to move that into a lower-level
+    class for more uniformity at the index and exporter levels.
+
+    How to use this class? In Django settings, use the SolrmarcEngine
+    class in your HAYSTACK_CONNECTIONS definition. Ensure that you've
+    created the applicable Solr core and that you have an
+    index.properties file in the solr/solrmarc project directory for
+    that index. (By default you should name it <core>_index.properties,
+    where <core> is the name of the Solr core.) Your haystack index
+    class should be a `base.search_indexes.CustomQuerySetIndex` or
+    `SolrmarcIndex` class. There are a few class attributes you can add
+    to the index class to help further define how the SolrMarc process
+    works--without them, sensible defaults are used.
+
+    `s2marc_class` -- The S2MarcBatch (see `export.sierra2marc`) or
+    equivalent/derived class that does the batch conversion of Sierra
+    data (via the Django ORM models) to MARC records and saves them to
+    the filesystem so that Solrmarc can index them. Default is
+    S2MarcBatch.
+
+    `index_properties` -- The filename for the index.properties file
+    that converts the MARC files to Solr fields. As mentioned above,
+    the default is '<core>_index.propertes' -- where <core> is the name
+    of the Solr core for that index.
+
+    `config_file` -- The filename for the Solrmarc config.properties
+    file that defines a bunch of settings used by Solrmarc. Default is
+    the SOLRMARC_CONFIG_FILE Django setting.
+
+    `temp_filepath` -- The filesystem location where the temporary MARC
+    file that gets loaded into Solrmarc is stored. Default is the
+    MEDIA_ROOT Django setting.
+    """
+
+    class IndexError(Exception):
+        pass
+
+    def log_error(self, index, obj_str, err):
+        err = err if isinstance(err, Exception) else self.IndexError(err)
+        index.last_batch_errors.append((obj_str, err))
+
+    def _records_to_marcfile(self, index, records):
+        batch = index.s2marc_class(records)
+        out_recs = batch.to_marc()
+        try:
+            filename = batch.to_file(out_recs, append=False)
+        except IOError as e:
+            raise IOError('Error writing to output file: {}'.format(e))
+        for e in batch.errors:
+            self.log_error(index, e.id, e.msg)
+        return filename
+
+    def _formulate_solrmarc_cmd(self, index, rec_filepath, commit):
+        def_ip = '{}_index.properties'.format(self.get_core_name())
+        index_properties = getattr(index, 'index_properties', None) or def_ip
+        def_config = settings.SOLRMARC_CONFIG_FILE
+        config_file = getattr(index, 'config_file', None) or def_config
+        commit_str = 'true' if commit else 'false'
+        jarfile = ('{}/../../solr/solrmarc/StanfordSearchWorksSolrMarc.jar'
+                   ''.format(settings.PROJECT_DIR))
+        return ('java -Xmx1g -Dsolr.hosturl="{}" '
+                '-Dsolrmarc.indexing.properties="{}" '
+                '-Dsolr.commit_at_end="{}" '
+                '-jar "{}" {} {}'
+                ''.format(self.conn.url, index_properties, commit_str, jarfile,
+                          config_file, rec_filepath))
+
+    def get_core_name(self):
+        return self.conn.url.split('/')[-1]
+
+    def update(self, index, records, commit=False):
+        filedir = getattr(index, 'temp_filedir', None) or settings.MEDIA_ROOT
+        if not filedir.endswith('/'):
+            filedir = '{}/'.format(filedir)
+        rec_filename = self._records_to_marcfile(index, records)
+        rec_filepath = '{}{}'.format(filedir, rec_filename)
+        cmd = self._formulate_solrmarc_cmd(index, rec_filepath, commit)
+        call_options = {'stderr': subprocess.STDOUT, 'shell': False,
+                        'universal_newlines': True}
+        try:
+            result = subprocess.check_output(shlex.split(cmd), **call_options)
+            output = result.decode('unicode-escape')
+        except subprocess.CalledProcessError as e:
+            msg = ('Solrmarc process did not run successfully: {}'
+                       ''.format(e.output))
+            self.log_error(index, 'ERROR', msg)
+        else:
+            for line in output.split("\n")[:-1]:
+                line = re.sub(r'^\s+', '', line)
+                if re.match(r'^(WARN|ERROR)', line):
+                    self.log_error(index, 'WARNING', line)
+        os.remove(rec_filepath)
+
+
+class SolrmarcEngine(BaseEngine):
+    backend = SolrmarcIndexBackend
+    query = solr_backend.SolrSearchQuery