fix(transformers): more import problems...

Author: dv-ar

emm/indexing/__init__.py

@@ -61,10 +61,9 @@
 
 # Optional Sentence Transformer support 
 try:
-    import sentence_transformers
     from emm.indexing.pandas_sentence_transformer import PandasSentenceTransformerIndexer
     __all__.extend([
         "PandasSentenceTransformerIndexer"
     ])
 except ImportError:
-    pass
+    pass  # Transformer features unavailable

emm/models/sentence_transformer/__init__.py

@@ -7,10 +7,12 @@
 from __future__ import annotations
 
 from emm.models.sentence_transformer.base import BaseSentenceTransformerComponent
-from emm.models.sentence_transformer.tuning import TuningConfig, SentenceTransformerTuner
 
-__all__ = [
-    "BaseSentenceTransformerComponent",
-    "TuningConfig",
-    "SentenceTransformerTuner"
-] 
+__all__ = ["BaseSentenceTransformerComponent"]
+
+# Move optional imports inside try block
+try:
+    from emm.models.sentence_transformer.tuning import TuningConfig, SentenceTransformerTuner
+    __all__ += ["TuningConfig", "SentenceTransformerTuner"]
+except ImportError:
+    pass 

emm/pipeline/pandas_entity_matching.py

@@ -36,7 +36,14 @@
 from emm.indexing.pandas_cos_sim_matcher import PandasCosSimIndexer
 from emm.indexing.pandas_naive_indexer import PandasNaiveIndexer
 from emm.indexing.pandas_sni import PandasSortedNeighbourhoodIndexer
-from emm.indexing.pandas_sentence_transformer import PandasSentenceTransformerIndexer
+
+# Make sentence transformer import optional
+try:
+    from emm.indexing.pandas_sentence_transformer import PandasSentenceTransformerIndexer
+    _HAS_SENTENCE_TRANSFORMER = True
+except ImportError:
+    _HAS_SENTENCE_TRANSFORMER = False
+
 from emm.loggers import Timer
 from emm.loggers.logger import logger
 from emm.parameters import DEFAULT_CARRY_ON_COLS, MODEL_PARAMS

pyproject.toml

@@ -37,27 +37,30 @@ dependencies = [
 dynamic = ["version"]
 
 [project.optional-dependencies]
-spark = [
-  # In NumPy 1.24.0, np.bool has been removed.
-  # https://issues.apache.org/jira/browse/SPARK-41718
-  # 3.4 is needed for python 3.11
-  # https://github.com/apache/spark/pull/38987
-  "pyspark>=3.1; python_version < '3.11'",
-  "numpy<1.24.0",
+transformers = [
+    "sentence-transformers>=2.2.0",
+    "torch>=1.11.0"
+]
+
+tuning = [
+    "lightning",
+    "wandb",
+    "sentence-transformers>=2.2.0",  # Include transformer deps since tuning requires it
+    "torch>=1.11.0"
 ]
+
 dev = [
-  "pre-commit",
-  "gitpython",
-  "nbconvert",
-  "jupyter_client>=5.2.3",
-  "ipykernel>=5.1.3",
-  "matplotlib",
-  "pygments",
-  "pandoc",
-  "pympler",
-  "sentence-transformers>=2.2.0",
-  "torch>=1.11.0"
+    "pre-commit",
+    "gitpython",
+    "nbconvert",
+    "jupyter_client>=5.2.3",
+    "ipykernel>=5.1.3",
+    "matplotlib",
+    "pygments",
+    "pandoc",
+    "pympler"
 ]
+
 test = [
   "pytest",
   "pytest-ordering",
@@ -97,15 +100,6 @@ doc = [
   "myst_parser"
 ]
 
-transformers = [
-  "sentence-transformers",
-]
-
-tuning = [
-  "lightning",
-  "wandb"
-]
-
 [tool.pytest.ini_options]
 filterwarnings = [
   # DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.