Merge pull request #38 from unt-libraries/add-basic-exporter-tests

Add basic exporter tests

Author: jason-ellis

django/sierra/base/fixtures/eresfixtures.json

@@ -188,7 +188,7 @@ class ControlField(ReadOnlyModel):
     p42 = models.CharField(max_length=1, blank=True)
     p43 = models.CharField(max_length=1, blank=True)
     occ_num = models.IntegerField(null=True, blank=True)
-    remainder = models.CharField(max_length=100, blank=True)
+    remainder = models.CharField(max_length=100, null=True, blank=True)
 
     def get_tag(self):
         return '{:03}'.format(self.control_num)
@@ -797,7 +797,7 @@ class BibRecord(MainRecordTypeModel):
     index_change_count = models.IntegerField(null=True, blank=True)
     is_on_course_reserve = models.NullBooleanField(null=True, blank=True)
     is_right_result_exact = models.NullBooleanField(null=True, blank=True)
-    allocation_rule_code = models.CharField(max_length=1, blank=True)
+    allocation_rule_code = models.CharField(max_length=1, null=True, blank=True)
     skip_num = models.IntegerField(null=True, blank=True)
     cataloging_date_gmt = models.DateTimeField(null=True, blank=True)
     marc_type_code = models.CharField(max_length=1, blank=True)
@@ -1197,7 +1197,7 @@ class HoldingRecord(MainRecordTypeModel):
                                        null=True,
                                        blank=True)
     is_inherit_loc = models.NullBooleanField(null=True, blank=True)
-    allocation_rule_code = models.CharField(max_length=1, blank=True)
+    allocation_rule_code = models.CharField(max_length=1, null=True, blank=True)
     accounting_unit = models.ForeignKey('AccountingUnit',
                                         db_column='accounting_unit_code_num',
                                         to_field='code_num',

django/sierra/base/fixtures/makefixtures/eresources.json

@@ -0,0 +1,164 @@
+"""
+Contains all shared pytest fixtures
+"""
+
+import pytest
+import pysolr
+from model_mommy import mommy
+
+from django.conf import settings
+
+import utils
+from export import models as em
+from base import models as bm
+
+
+# Base app-related fixtures
+
+@pytest.fixture
+def sierra_records_by_recnum_range():
+    """
+    Return Sierra records by record number range.
+
+    This is a pytest fixture that returns a set of objects based on the
+    provided start and (optional) end record number. Uses the
+    appropriate model based on the type of record, derived from the
+    first character of the record numbers (e.g., b is BibRecord).
+
+    Metadata about the filter used is added to an `info` attribute on
+    the object queryset that is returned.
+    """
+    def _sierra_records_by_recnum_range(start, end=None):
+        rectype = start[0]
+        filter_options = {'record_range_from': start, 
+                          'record_range_to': end or start}
+        modelname = bm.RecordMetadata.record_type_models[rectype]
+        model = getattr(bm, modelname)
+        recset = model.objects.filter_by('record_range', filter_options)
+        recset.info = {
+            'filter_method': 'filter_by',
+            'filter_args': ['record_range', filter_options],
+            'filter_kwargs': {},
+            'export_filter_type': 'record_range',
+            'export_filter_options': filter_options,
+        }
+        return recset
+    return _sierra_records_by_recnum_range
+
+
+@pytest.fixture
+def sierra_full_object_set():
+    """
+    Return a full set of objects from a Sierra (base) model.
+
+    This is a pytest fixture that returns a full set of model objects
+    based on the type of model provided.
+
+    Metadata about the filter used is added to an `info` attribute on
+    the object queryset that is returned.
+    """
+    def _sierra_full_object_set(model_name):
+        recset = getattr(bm, model_name).objects.all()
+        recset.info = {
+            'filter_method': 'all',
+            'filter_args': [],
+            'filter_kwargs': {},
+            'export_filter_type': 'full_export',
+            'export_filter_options': {}
+        }
+        return recset
+    return _sierra_full_object_set
+
+
+# Export app-related fixtures
+
+
+@pytest.fixture
+def export_type():
+    def _export_type(code):
+        return em.ExportType.objects.get(code=code)
+    return _export_type
+
+
+@pytest.fixture
+def export_filter():
+    def _export_filter(code):
+        return em.ExportFilter.objects.get(code=code)
+    return _export_filter
+
+
+@pytest.fixture
+def status():
+    def _status(code):
+        return em.Status.objects.get(code=code)
+    return _status
+
+
+@pytest.fixture
+def new_export_instance(export_type, export_filter, status):
+    def _new_export_instance(et_code, ef_code, st_code):
+        return mommy.make(
+            em.ExportInstance,
+            status=status(st_code),
+            export_type=export_type(et_code),
+            export_filter=export_filter(ef_code),
+            errors=0,
+            warnings=0
+        )
+    return _new_export_instance
+
+
+@pytest.fixture
+def new_exporter(new_export_instance):
+    def _new_exporter(et_code, ef_code, st_code, options={}):
+        instance = new_export_instance(et_code, ef_code, st_code)
+        exp_class = utils.load_class(instance.export_type.path)
+        return exp_class(instance.pk, ef_code, et_code, options)
+    yield _new_exporter
+    em.ExportInstance.objects.all().delete()
+
+
+@pytest.fixture
+def get_records():
+    def _get_records(exporter):
+        return exporter.get_records()
+    return _get_records
+
+
+@pytest.fixture
+def export_records():
+    def _export_records(exporter, records):
+        exporter.export_records(records)
+        exporter.final_callback()
+    return _export_records
+
+
+@pytest.fixture
+def delete_records():
+    def _delete_records(exporter, records):
+        exporter.delete_records(records)
+        exporter.final_callback()
+    return _delete_records
+
+
+# Utils-related fixtures
+
+@pytest.fixture
+def solr_conn():
+    url = 'http://{}:{}/solr'.format(settings.SOLR_HOST, settings.SOLR_PORT)
+    def _solr_conn(core_name):
+        connection = pysolr.Solr('{}/{}'.format(url, core_name))
+        _solr_conn.connections.append(connection)
+        return connection
+    _solr_conn.connections = []
+    yield _solr_conn
+    for conn in _solr_conn.connections:
+        conn.delete(q='*:*')
+
+
+@pytest.fixture
+def solr_search():
+    def _solr_search(conn, options):
+        results = conn.search(**options)
+        return [r for r in results]
+    return _solr_search

django/sierra/base/models.py

@@ -0,0 +1,95 @@
+"""
+Tests classes derived from `export.exporter.Exporter`.
+"""
+
+import pytest
+
+
+# FIXTURES AND TEST DATA
+# Fixtures used in the below tests can be found in
+# django/sierra/base/tests/conftest.py:
+#    sierra_records_by_recnum_range, sierra_full_object_set,
+#    new_exporter, export_records, delete_records, solr_conn,
+#    solr_search
+
+pytestmark = pytest.mark.django_db
+
+
+@pytest.fixture
+def solr_exporter_test_params(sierra_records_by_recnum_range,
+                              sierra_full_object_set):
+    bib_set = sierra_records_by_recnum_range('b4371446')
+    eres_set = sierra_records_by_recnum_range('e1001249')
+    item_set = sierra_records_by_recnum_range('i4264281')
+    itype_set = sierra_full_object_set('ItypeProperty')
+    istatus_set = sierra_full_object_set('ItemStatusProperty')
+    location_set = sierra_full_object_set('Location')
+
+    return {
+        'BibsToSolr': {
+            'record_set': bib_set,
+            'cores': ['bibdata', 'marc'],
+            'try_delete': True
+        },
+        'EResourcesToSolr': {
+            'record_set': eres_set,
+            'cores': ['haystack'],
+            'try_delete': True
+        },
+        'ItemsToSolr': {
+            'record_set': item_set,
+            'cores': ['haystack'],
+            'try_delete': True
+        },
+        'ItemStatusesToSolr': {
+            'record_set': istatus_set,
+            'cores': ['haystack'],
+            'try_delete': False
+        },
+        'ItypesToSolr': {
+            'record_set': itype_set,
+            'cores': ['haystack'],
+            'try_delete': False
+        },
+        'LocationsToSolr': {
+            'record_set': location_set,
+            'cores': ['haystack'],
+            'try_delete': False
+        },
+    }
+
+
+# TESTS
+
+@pytest.mark.parametrize('etype_code', [
+    'BibsToSolr',
+    'EResourcesToSolr',
+    'ItemsToSolr',
+    'ItemStatusesToSolr',
+    'ItypesToSolr',
+    'LocationsToSolr'])
+def test_exports_to_solr(etype_code, solr_exporter_test_params, new_exporter,
+                         export_records, delete_records, solr_conn,
+                         solr_search):
+    """
+    For Exporter classes that load data into Solr, blah
+    """
+    record_set = solr_exporter_test_params[etype_code]['record_set']
+    cores = solr_exporter_test_params[etype_code]['cores']
+    try_delete = solr_exporter_test_params[etype_code]['try_delete']
+    load_exporter = new_exporter(etype_code, 'full_export', 'waiting')
+    conns = {c: solr_conn(c) for c in cores}
+    pre_results = {c: solr_search(conns[c], {'q': '*'}) for c in cores}
+    export_records(load_exporter, record_set)
+    load_results = {c: solr_search(conns[c], {'q': '*'}) for c in cores}
+    del_results = {}
+    if try_delete:
+        del_exporter = new_exporter(etype_code, 'full_export', 'waiting')
+        delete_records(del_exporter, record_set)
+        del_results = {c: solr_search(conns[c], {'q': '*'}) for c in cores}
+
+    for core in cores:
+        assert len(pre_results[core]) == 0
+        assert len(load_results[core]) > 0
+        if try_delete:
+            assert len(del_results[core]) == 0

django/sierra/conftest.py

@@ -385,7 +385,8 @@ def raise_setting_error(setting):
 ]
 
 # The path (relative or absolute) to the command that runs SolrMarc.
-SOLRMARC_COMMAND = '../../solr/solrmarc/indexfile.sh'
+SOLRMARC_COMMAND = get_env_variable('SOLRMARC_COMMAND',
+                                    '../../solr/solrmarc/indexfile.sh')
 # The name of the properties file to use when running SolrMarc.
 SOLRMARC_CONFIG_FILE = get_env_variable('SOLRMARC_CONFIG_FILE',
                                         'dev_config.properties')

django/sierra/export/tests/__init__.py

@@ -173,6 +173,7 @@ services:
       - ./docker_data/test/media:/project/media
     environment:
       - SOLRMARC_CONFIG_FILE=${TEST_SOLRMARC_CONFIG_FILE:-test_config.properties}
+      - SOLRMARC_COMMAND=/project/catalog-api/solr/solrmarc/indexfile.sh
     working_dir: /project/catalog-api/
     networks:
       - testing