Remove the need for a picker id during classification (#764)

Inserting the picker id into the ParticleClassificationGroup table in ispyb creates a race condition which we have previously worked around. Given the picker id is optional we should just not do this. Particularly needed for doppio live.

Author: stephen-riggs

src/murfey/server/feedback.py

@@ -354,7 +354,6 @@ def _release_2d_hold(message: dict, _db):
                 "nr_classes": default_spa_parameters.nr_classes_2d,
                 "do_icebreaker_jobs": default_spa_parameters.do_icebreaker_jobs,
                 "class2d_fraction_of_classes_to_remove": default_spa_parameters.fraction_of_classes_to_remove_2d,
-                "picker_id": feedback_params.picker_ispyb_id,
                 "class_uuids": _2d_class_murfey_ids(
                     first_class2d.particles_file, message["program_id"], _db
                 ),
@@ -432,7 +431,6 @@ def _release_3d_hold(message: dict, _db):
                 "mask_diameter": relion_params.mask_diameter or 0,
                 "do_initial_model": False if feedback_params.initial_model else True,
                 "initial_model_file": feedback_params.initial_model,
-                "picker_id": feedback_params.picker_ispyb_id,
                 "class_uuids": _3d_class_murfey_ids(
                     class3d_params.particles_file, _app_id(pj_id, _db), _db
                 ),
@@ -518,7 +516,6 @@ def _release_refine_hold(message: dict, _db):
                 "symmetry": relion_params.symmetry,
                 "node_creator_queue": machine_config.node_creator_queue,
                 "nr_iter": default_spa_parameters.nr_iter_3d,
-                "picker_id": feedback_params.picker_ispyb_id,
                 "refined_class_uuid": _refine_murfey_id(
                     refine_dir=refine_params.refine_dir,
                     tag=refine_params.tag,
@@ -590,13 +587,6 @@ def _register_incomplete_2d_batch(message: dict, _db):
     feedback_params.hold_class2d = True
     relion_options = dict(relion_params)
     other_options = dict(feedback_params)
-    if other_options["picker_ispyb_id"] is None:
-        logger.info("No ISPyB particle picker ID yet")
-        feedback_params.hold_class2d = False
-        _db.add(feedback_params)
-        _db.commit()
-        _db.expunge(feedback_params)
-        return
     _db.add(feedback_params)
     _db.commit()
     _db.expunge(feedback_params)
@@ -630,7 +620,6 @@ def _register_incomplete_2d_batch(message: dict, _db):
             "batch_is_complete": False,
             "particle_diameter": relion_options["particle_diameter"],
             "combine_star_job_number": -1,
-            "picker_id": other_options["picker_ispyb_id"],
             "nr_iter": default_spa_parameters.nr_iter_2d,
             "batch_size": default_spa_parameters.batch_size_2d,
             "nr_classes": default_spa_parameters.nr_classes_2d,
@@ -694,7 +683,7 @@ def _register_complete_2d_batch(message: dict, _db):
     ).one()
     _db.expunge(relion_params)
     _db.expunge(feedback_params)
-    if feedback_params.hold_class2d or feedback_params.picker_ispyb_id is None:
+    if feedback_params.hold_class2d:
         feedback_params.rerun_class2d = True
         _db.add(feedback_params)
         _db.commit()
@@ -792,7 +781,6 @@ def _register_complete_2d_batch(message: dict, _db):
                 "mask_diameter": relion_params.mask_diameter or 0,
                 "combine_star_job_number": feedback_params.star_combination_job,
                 "autoselect_min_score": 0,
-                "picker_id": feedback_params.picker_ispyb_id,
                 "class_uuids": class_uuids,
                 "class2d_grp_uuid": class2d_grp_uuid,
                 "nr_iter": default_spa_parameters.nr_iter_2d,
@@ -867,7 +855,6 @@ def _register_complete_2d_batch(message: dict, _db):
                 "mask_diameter": relion_params.mask_diameter or 0,
                 "combine_star_job_number": feedback_params.star_combination_job,
                 "autoselect_min_score": feedback_params.class_selection_score or 0,
-                "picker_id": feedback_params.picker_ispyb_id,
                 "class_uuids": class_uuids,
                 "class2d_grp_uuid": class2d_grp_uuid,
                 "nr_iter": default_spa_parameters.nr_iter_2d,
@@ -958,7 +945,6 @@ def _flush_class2d(
                 "mask_diameter": relion_params.mask_diameter or 0,
                 "combine_star_job_number": feedback_params.star_combination_job,
                 "autoselect_min_score": feedback_params.class_selection_score or 0,
-                "picker_id": feedback_params.picker_ispyb_id,
                 "class_uuids": _2d_class_murfey_ids(
                     saved_message.particles_file, _app_id(pj_id, _db), _db
                 ),
@@ -1008,16 +994,6 @@ def _register_class_selection(message: dict, _db):
     ).one()
     _db.expunge(feedback_params)
 
-    if feedback_params.picker_ispyb_id is None:
-        selection_stash = db.SelectionStash(
-            pj_id=pj_id,
-            class_selection_score=message["class_selection_score"] or 0,
-        )
-        _db.add(selection_stash)
-        _db.commit()
-        _db.close()
-        return
-
     feedback_params.class_selection_score = message.get("class_selection_score") or 0
     feedback_params.hold_class2d = False
     _flush_class2d(
@@ -1285,7 +1261,6 @@ def _register_3d_batch(message: dict, _db):
                 "particle_diameter": relion_options["particle_diameter"],
                 "mask_diameter": relion_options["mask_diameter"] or 0,
                 "do_initial_model": True,
-                "picker_id": other_options["picker_ispyb_id"],
                 "class_uuids": {i + 1: m for i, m in enumerate(class_uuids)},
                 "class3d_grp_uuid": class3d_grp_uuid,
                 "nr_iter": default_spa_parameters.nr_iter_3d,
@@ -1326,7 +1301,6 @@ def _register_3d_batch(message: dict, _db):
                 "mask_diameter": relion_options["mask_diameter"] or 0,
                 "do_initial_model": False,
                 "initial_model_file": other_options["initial_model"],
-                "picker_id": other_options["picker_ispyb_id"],
                 "class_uuids": _3d_class_murfey_ids(
                     class3d_params.particles_file, _app_id(pj_id, _db), _db
                 ),
@@ -1535,7 +1509,6 @@ def _register_refinement(message: dict, _db):
             db.ClassificationFeedbackParameters.pj_id == pj_id_params
         )
     ).one()
-    other_options = dict(feedback_params)
 
     if feedback_params.hold_refine:
         # If waiting then save the message
@@ -1627,7 +1600,6 @@ def _register_refinement(message: dict, _db):
                 "symmetry": relion_options["symmetry"],
                 "node_creator_queue": machine_config.node_creator_queue,
                 "nr_iter": default_spa_parameters.nr_iter_3d,
-                "picker_id": other_options["picker_ispyb_id"],
                 "refined_class_uuid": _refine_murfey_id(
                     refine_dir=refine_params.refine_dir,
                     tag=refine_params.tag,
@@ -1762,7 +1734,6 @@ def _register_bfactors(message: dict, _db):
                 "particle_diameter": relion_options["particle_diameter"],
                 "mask_diameter": relion_options["mask_diameter"] or 0,
                 "node_creator_queue": machine_config.node_creator_queue,
-                "picker_id": feedback_params.picker_ispyb_id,
                 "refined_grp_uuid": bfactor_params.refined_grp_uuid,
                 "refined_class_uuid": bfactor_params.refined_class_uuid,
                 "session_id": message["session_id"],

src/murfey/util/db.py

@@ -339,9 +339,6 @@ class ProcessingJob(SQLModel, table=True):  # type: ignore
     auto_proc_programs: List["AutoProcProgram"] = Relationship(
         back_populates="processing_job", sa_relationship_kwargs={"cascade": "delete"}
     )
-    selection_stash: List["SelectionStash"] = Relationship(
-        back_populates="processing_job", sa_relationship_kwargs={"cascade": "delete"}
-    )
     particle_sizes: List["ParticleSizes"] = Relationship(
         back_populates="processing_job", sa_relationship_kwargs={"cascade": "delete"}
     )
@@ -393,15 +390,6 @@ class PreprocessStash(SQLModel, table=True):  # type: ignore
     foil_hole: Optional["FoilHole"] = Relationship(back_populates="preprocess_stashes")
 
 
-class SelectionStash(SQLModel, table=True):  # type: ignore
-    id: Optional[int] = Field(default=None, primary_key=True)
-    class_selection_score: float
-    pj_id: int = Field(foreign_key="processingjob.processingJobId")
-    processing_job: Optional[ProcessingJob] = Relationship(
-        back_populates="selection_stash"
-    )
-
-
 class TomographyProcessingParameters(SQLModel, table=True):  # type: ignore
     dcg_id: int = Field(
         primary_key=True, foreign_key="datacollectiongroup.dataCollectionGroupId"
@@ -717,7 +705,6 @@ class ClassificationFeedbackParameters(SQLModel, table=True):  # type: ignore
     initial_model: str
     next_job: int
     picker_murfey_id: Optional[int] = Field(default=None, foreign_key="murfeyledger.id")
-    picker_ispyb_id: Optional[int] = None
     processing_job: Optional[ProcessingJob] = Relationship(
         back_populates="classification_feedback_parameters"
     )

src/murfey/workflows/spa/picking.py

@@ -9,9 +9,7 @@
 from murfey.server import _transport_object
 from murfey.server.feedback import (
     _app_id,
-    _flush_class2d,
     _pj_id,
-    _register_class_selection,
 )
 from murfey.util.config import get_machine_config
 from murfey.util.db import (
@@ -24,7 +22,6 @@
     NotificationValue,
     ParticleSizes,
     ProcessingJob,
-    SelectionStash,
     Session as MurfeySession,
     SPARelionParameters,
 )
@@ -84,40 +81,8 @@ def _register_picked_particles_use_diameter(message: dict, _db: Session):
             select(SPARelionParameters).where(SPARelionParameters.pj_id == pj_id)
         ).one()
         relion_options = dict(relion_params)
-        feedback_params = _db.exec(
-            select(ClassificationFeedbackParameters).where(
-                ClassificationFeedbackParameters.pj_id == pj_id
-            )
-        ).one()
-
         particle_diameter = relion_params.particle_diameter
 
-        if feedback_params.picker_ispyb_id is None:
-            if not _transport_object:
-                feedback_params.picker_ispyb_id = 1000
-            else:
-                assert feedback_params.picker_murfey_id is not None
-                feedback_params.picker_ispyb_id = _transport_object.do_buffer_lookup(
-                    message["program_id"], feedback_params.picker_murfey_id
-                )
-                if feedback_params.picker_ispyb_id is not None:
-                    _flush_class2d(message["session_id"], message["program_id"], _db)
-            _db.add(feedback_params)
-            _db.commit()
-            selection_stash = _db.exec(
-                select(SelectionStash).where(SelectionStash.pj_id == pj_id)
-            ).all()
-            for s in selection_stash:
-                _register_class_selection(
-                    {
-                        "session_id": s.session_id,
-                        "class_selection_score": s.class_selection_score or 0,
-                    },
-                    _db=_db,
-                )
-                _db.delete(s)
-                _db.commit()
-
         if not particle_diameter:
             # If the diameter has not been calculated then find it
             picking_db = _db.exec(
@@ -256,34 +221,6 @@ def _register_picked_particles_use_boxsize(message: dict, _db: Session):
     relion_params = _db.exec(
         select(SPARelionParameters).where(SPARelionParameters.pj_id == pj_id)
     ).one()
-    feedback_params = _db.exec(
-        select(ClassificationFeedbackParameters).where(
-            ClassificationFeedbackParameters.pj_id == pj_id
-        )
-    ).one()
-
-    if feedback_params.picker_ispyb_id is None and _transport_object:
-        assert feedback_params.picker_murfey_id is not None
-        feedback_params.picker_ispyb_id = _transport_object.do_buffer_lookup(
-            message["program_id"], feedback_params.picker_murfey_id
-        )
-        if feedback_params.picker_ispyb_id is not None:
-            _flush_class2d(message["session_id"], message["program_id"], _db)
-        _db.add(feedback_params)
-        _db.commit()
-        selection_stash = _db.exec(
-            select(SelectionStash).where(SelectionStash.pj_id == pj_id)
-        ).all()
-        for s in selection_stash:
-            _register_class_selection(
-                {
-                    "session_id": s.session_id,
-                    "class_selection_score": s.class_selection_score or 0,
-                },
-                _db=_db,
-            )
-            _db.delete(s)
-            _db.commit()
 
     # Send the message to extraction with the box sizes
     zocalo_message: dict = {

src/murfey/workflows/tomo/picking.py

@@ -135,7 +135,6 @@ def _register_picked_tomogram_use_diameter(message: dict, murfey_db: Session):
                         "autoproc_program_id": _app_id(pj_id, murfey_db),
                         "batch_size": default_tomo_parameters.batch_size_2d,
                         "nr_classes": default_tomo_parameters.nr_classes_2d,
-                        "picker_id": None,
                         "class2d_grp_uuid": class2d_grp_uuid,
                         "class_uuids": class_uuids,
                         "next_job": feedback_params.next_job,
@@ -177,7 +176,6 @@ def _register_picked_tomogram_use_diameter(message: dict, murfey_db: Session):
                     "autoproc_program_id": _app_id(pj_id, murfey_db),
                     "batch_size": default_tomo_parameters.batch_size_2d,
                     "nr_classes": default_tomo_parameters.nr_classes_2d,
-                    "picker_id": None,
                     "class2d_grp_uuid": class2d_grp_uuid,
                     "class_uuids": class_uuids,
                     "next_job": feedback_params.next_job,

tests/workflows/tomo/test_tomo_picking.py

@@ -222,7 +222,6 @@ def test_picked_tomogram_run_class2d_with_diameter(
                 "autoproc_program_id": 0,
                 "batch_size": 5000,
                 "nr_classes": 5,
-                "picker_id": None,
                 "class2d_grp_uuid": 6,
                 "class_uuids": {str(i): i for i in range(1, 6)},
                 "next_job": 9,
@@ -312,7 +311,6 @@ def test_picked_tomogram_run_class2d_estimate_diameter(
                 "autoproc_program_id": 0,
                 "batch_size": 5000,
                 "nr_classes": 5,
-                "picker_id": None,
                 "class2d_grp_uuid": 12,
                 "class_uuids": {str(i): i + 6 for i in range(1, 6)},
                 "next_job": 9,
@@ -336,7 +334,6 @@ def test_picked_tomogram_run_class2d_estimate_diameter(
                 "autoproc_program_id": 0,
                 "batch_size": 5000,
                 "nr_classes": 5,
-                "picker_id": None,
                 "class2d_grp_uuid": 18,
                 "class_uuids": {str(i): i + 12 for i in range(1, 6)},
                 "next_job": 11,