Skip to content

Commit ba1c83c

Browse files
d-j-hattond-j-hatton
authored
Remove the need for an ISPyB instance (#379)
ISPyB may not be in use at other facilities. This removes the requirement for a connection to an ISPyB database. The option to manually name visits is provided and data collection IDs etc. are autoincremented within the Murfey database when there is no ISPyB to interact with.
1 parent b7114c6 commit ba1c83c

9 files changed

Lines changed: 408 additions & 234 deletions

File tree

‎src/murfey/client/multigrid_control.py‎

Lines changed: 68 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ def _start_rsyncer_multigrid(
133133
remove_files=remove_files,
134134
tag=tag,
135135
limited=limited,
136+
transfer=machine_data.get("data_transfer_enabled", True),
136137
)
137138
self.ws.send(json.dumps({"message": "refresh"}))
138139

@@ -173,6 +174,7 @@ def _start_rsyncer(
173174
remove_files: bool = False,
174175
tag: str = "",
175176
limited: bool = False,
177+
transfer: bool = True,
176178
):
177179
log.info(f"starting rsyncer: {source}")
178180
if self._environment:
@@ -189,47 +191,48 @@ def _start_rsyncer(
189191
log.warning(
190192
f"Gain reference file {self._environment.gain_ref} was not successfully transferred to {visit_path}/processing"
191193
)
192-
self.rsync_processes[source] = RSyncer(
193-
source,
194-
basepath_remote=Path(destination),
195-
server_url=self._environment.url,
196-
stop_callback=self._rsyncer_stopped,
197-
do_transfer=self.do_transfer,
198-
remove_files=remove_files,
199-
)
194+
if transfer:
195+
self.rsync_processes[source] = RSyncer(
196+
source,
197+
basepath_remote=Path(destination),
198+
server_url=self._environment.url,
199+
stop_callback=self._rsyncer_stopped,
200+
do_transfer=self.do_transfer,
201+
remove_files=remove_files,
202+
)
200203

201-
def rsync_result(update: RSyncerUpdate):
202-
if not update.base_path:
203-
raise ValueError("No base path from rsyncer update")
204-
if not self.rsync_processes.get(update.base_path):
205-
raise ValueError("TUI rsync process does not exist")
206-
if update.outcome is TransferResult.SUCCESS:
207-
# log.info(
208-
# f"File {str(update.file_path)!r} successfully transferred ({update.file_size} bytes)"
209-
# )
210-
pass
211-
else:
212-
log.warning(f"Failed to transfer file {str(update.file_path)!r}")
213-
self.rsync_processes[update.base_path].enqueue(update.file_path)
204+
def rsync_result(update: RSyncerUpdate):
205+
if not update.base_path:
206+
raise ValueError("No base path from rsyncer update")
207+
if not self.rsync_processes.get(update.base_path):
208+
raise ValueError("TUI rsync process does not exist")
209+
if update.outcome is TransferResult.SUCCESS:
210+
# log.info(
211+
# f"File {str(update.file_path)!r} successfully transferred ({update.file_size} bytes)"
212+
# )
213+
pass
214+
else:
215+
log.warning(f"Failed to transfer file {str(update.file_path)!r}")
216+
self.rsync_processes[update.base_path].enqueue(update.file_path)
214217

215-
self.rsync_processes[source].subscribe(rsync_result)
216-
self.rsync_processes[source].subscribe(
217-
partial(
218-
self._increment_transferred_files,
219-
destination=destination,
220-
source=str(source),
221-
),
222-
secondary=True,
223-
)
224-
url = f"{str(self._environment.url.geturl())}/sessions/{str(self._environment.murfey_session)}/rsyncer"
225-
rsyncer_data = {
226-
"source": str(source),
227-
"destination": destination,
228-
"session_id": self.session_id,
229-
"transferring": self.do_transfer or self._environment.demo,
230-
"tag": tag,
231-
}
232-
requests.post(url, json=rsyncer_data)
218+
self.rsync_processes[source].subscribe(rsync_result)
219+
self.rsync_processes[source].subscribe(
220+
partial(
221+
self._increment_transferred_files,
222+
destination=destination,
223+
source=str(source),
224+
),
225+
secondary=True,
226+
)
227+
url = f"{str(self._environment.url.geturl())}/sessions/{str(self._environment.murfey_session)}/rsyncer"
228+
rsyncer_data = {
229+
"source": str(source),
230+
"destination": destination,
231+
"session_id": self.session_id,
232+
"transferring": self.do_transfer or self._environment.demo,
233+
"tag": tag,
234+
}
235+
requests.post(url, json=rsyncer_data)
233236
self._environment.watchers[source] = DirWatcher(source, settling_time=30)
234237

235238
if not self.analysers.get(source) and analyse:
@@ -254,15 +257,36 @@ def rsync_result(update: RSyncerUpdate):
254257
else:
255258
self.analysers[source].subscribe(self._data_collection_form)
256259
self.analysers[source].start()
257-
self.rsync_processes[source].subscribe(self.analysers[source].enqueue)
260+
if transfer:
261+
self.rsync_processes[source].subscribe(self.analysers[source].enqueue)
258262

259-
self.rsync_processes[source].start()
263+
if transfer:
264+
self.rsync_processes[source].start()
260265

261266
if self._environment:
262267
if self._environment.watchers.get(source):
263-
self._environment.watchers[source].subscribe(
264-
self.rsync_processes[source].enqueue
265-
)
268+
if transfer:
269+
self._environment.watchers[source].subscribe(
270+
self.rsync_processes[source].enqueue
271+
)
272+
else:
273+
# the watcher and rsyncer don't notify with the same object so conversion required here
274+
def _rsync_update_converter(p: Path) -> None:
275+
self.analysers[source].enqueue(
276+
RSyncerUpdate(
277+
file_path=p,
278+
file_size=0,
279+
outcome=TransferResult.SUCCESS,
280+
transfer_total=0,
281+
queue_size=0,
282+
base_path=source,
283+
)
284+
)
285+
return None
286+
287+
self._environment.watchers[source].subscribe(
288+
_rsync_update_converter
289+
)
266290
self._environment.watchers[source].subscribe(
267291
partial(
268292
self._increment_file_count,

‎src/murfey/client/tui/app.py‎

Lines changed: 83 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
MainScreen,
2626
ProcessingForm,
2727
SessionSelection,
28+
VisitCreation,
2829
VisitSelection,
2930
WaitingScreen,
3031
determine_default_destination,
@@ -192,6 +193,7 @@ def _start_rsyncer_multigrid(
192193
analyse=analyse,
193194
remove_files=remove_files,
194195
limited=limited,
196+
transfer=machine_data.get("data_transfer_enabled", True),
195197
)
196198

197199
def _start_rsyncer(
@@ -203,6 +205,7 @@ def _start_rsyncer(
203205
analyse: bool = True,
204206
remove_files: bool = False,
205207
limited: bool = False,
208+
transfer: bool = True,
206209
):
207210
log.info(f"starting rsyncer: {source}")
208211
if self._environment:
@@ -219,55 +222,57 @@ def _start_rsyncer(
219222
log.warning(
220223
f"Gain reference file {self._environment.gain_ref} was not successfully transferred to {visit_path}/processing"
221224
)
222-
self.rsync_processes[source] = RSyncer(
223-
source,
224-
basepath_remote=Path(destination),
225-
server_url=self._url,
226-
# local=self._environment.demo,
227-
status_bar=self._statusbar,
228-
do_transfer=self._do_transfer,
229-
required_substrings_for_removal=self._data_substrings,
230-
remove_files=remove_files,
231-
)
225+
if transfer:
226+
self.rsync_processes[source] = RSyncer(
227+
source,
228+
basepath_remote=Path(destination),
229+
server_url=self._url,
230+
# local=self._environment.demo,
231+
status_bar=self._statusbar,
232+
do_transfer=self._do_transfer,
233+
required_substrings_for_removal=self._data_substrings,
234+
remove_files=remove_files,
235+
)
232236

233-
def rsync_result(update: RSyncerUpdate):
234-
if not update.base_path:
235-
raise ValueError("No base path from rsyncer update")
236-
if not self.rsync_processes.get(update.base_path):
237-
raise ValueError("TUI rsync process does not exist")
238-
if update.outcome is TransferResult.SUCCESS:
239-
log.debug(
240-
f"Succesfully transferred file {str(update.file_path)!r} ({update.file_size} bytes)"
237+
def rsync_result(update: RSyncerUpdate):
238+
if not update.base_path:
239+
raise ValueError("No base path from rsyncer update")
240+
if not self.rsync_processes.get(update.base_path):
241+
raise ValueError("TUI rsync process does not exist")
242+
if update.outcome is TransferResult.SUCCESS:
243+
log.debug(
244+
f"Succesfully transferred file {str(update.file_path)!r} ({update.file_size} bytes)"
245+
)
246+
# pass
247+
else:
248+
log.warning(f"Failed to transfer file {str(update.file_path)!r}")
249+
self.rsync_processes[update.base_path].enqueue(update.file_path)
250+
251+
self.rsync_processes[source].subscribe(rsync_result)
252+
self.rsync_processes[source].subscribe(
253+
partial(
254+
self._increment_transferred_files_prometheus,
255+
destination=destination,
256+
source=str(source),
241257
)
242-
# pass
243-
else:
244-
log.warning(f"Failed to transfer file {str(update.file_path)!r}")
245-
self.rsync_processes[update.base_path].enqueue(update.file_path)
246-
247-
self.rsync_processes[source].subscribe(rsync_result)
248-
self.rsync_processes[source].subscribe(
249-
partial(
250-
self._increment_transferred_files_prometheus,
251-
destination=destination,
252-
source=str(source),
253258
)
254-
)
255-
self.rsync_processes[source].subscribe(
256-
partial(
257-
self._increment_transferred_files,
258-
destination=destination,
259-
source=str(source),
260-
),
261-
secondary=True,
262-
)
263-
url = f"{str(self._url.geturl())}/sessions/{str(self._environment.murfey_session)}/rsyncer"
264-
rsyncer_data = {
265-
"source": str(source),
266-
"destination": destination,
267-
"session_id": self._environment.murfey_session,
268-
"transferring": self._do_transfer,
269-
}
270-
requests.post(url, json=rsyncer_data)
259+
self.rsync_processes[source].subscribe(
260+
partial(
261+
self._increment_transferred_files,
262+
destination=destination,
263+
source=str(source),
264+
),
265+
secondary=True,
266+
)
267+
url = f"{str(self._url.geturl())}/sessions/{str(self._environment.murfey_session)}/rsyncer"
268+
rsyncer_data = {
269+
"source": str(source),
270+
"destination": destination,
271+
"session_id": self._environment.murfey_session,
272+
"transferring": self._do_transfer,
273+
}
274+
requests.post(url, json=rsyncer_data)
275+
271276
self._environment.watchers[source] = DirWatcher(source, settling_time=30)
272277

273278
if not self.analysers.get(source) and analyse:
@@ -295,15 +300,36 @@ def rsync_result(update: RSyncerUpdate):
295300
else:
296301
self.analysers[source].subscribe(self._data_collection_form)
297302
self.analysers[source].start()
298-
self.rsync_processes[source].subscribe(self.analysers[source].enqueue)
303+
if transfer:
304+
self.rsync_processes[source].subscribe(self.analysers[source].enqueue)
299305

300-
self.rsync_processes[source].start()
306+
if transfer:
307+
self.rsync_processes[source].start()
301308

302309
if self._environment:
303310
if self._environment.watchers.get(source):
304-
self._environment.watchers[source].subscribe(
305-
self.rsync_processes[source].enqueue
306-
)
311+
if transfer:
312+
self._environment.watchers[source].subscribe(
313+
self.rsync_processes[source].enqueue
314+
)
315+
else:
316+
317+
def _rsync_update_converter(p: Path) -> None:
318+
self.analysers[source].enqueue(
319+
RSyncerUpdate(
320+
file_path=p,
321+
file_size=0,
322+
outcome=TransferResult.SUCCESS,
323+
transfer_total=0,
324+
queue_size=0,
325+
base_path=source,
326+
)
327+
)
328+
return None
329+
330+
self._environment.watchers[source].subscribe(
331+
_rsync_update_converter
332+
)
307333
self._environment.watchers[source].subscribe(
308334
partial(
309335
self._increment_file_count,
@@ -657,8 +683,12 @@ async def on_mount(self) -> None:
657683
exisiting_sessions = requests.get(
658684
f"{self._environment.url.geturl()}/sessions"
659685
).json()
660-
self.install_screen(VisitSelection(self.visits), "visit-select-screen")
661-
self.push_screen("visit-select-screen")
686+
if self.visits:
687+
self.install_screen(VisitSelection(self.visits), "visit-select-screen")
688+
self.push_screen("visit-select-screen")
689+
else:
690+
self.install_screen(VisitCreation(), "visit-creation-screen")
691+
self.push_screen("visit-creation-screen")
662692
if exisiting_sessions:
663693
self.install_screen(
664694
SessionSelection(

‎src/murfey/client/tui/controller.css‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,12 @@ SessionSelection {
4646
border: hidden;
4747
}
4848

49+
VisitCreation {
50+
layout: grid;
51+
grid-size: 2;
52+
border: hidden;
53+
}
54+
4955
VisitSelection {
5056
layout: grid;
5157
grid-size: 4;
@@ -235,6 +241,22 @@ RadioSet {
235241
background: darkslateblue;
236242
}
237243

244+
.btn-visit-create {
245+
width: 100%;
246+
height: 90fr;
247+
column-span: 2;
248+
background: teal;
249+
border: solid black;
250+
}
251+
252+
.btn-visit-create:hover {
253+
background: purple;
254+
}
255+
256+
.btn-visit-create:focus {
257+
background: darkslateblue;
258+
}
259+
238260
.btn-session {
239261
width: 100%;
240262
height: 20%;
@@ -468,6 +490,16 @@ RadioSet {
468490
background: black;
469491
}
470492

493+
.input-visit-name {
494+
width: 100%;
495+
height: 100%;
496+
column-span: 2;
497+
row-span: 1;
498+
content-align: left middle;
499+
text-style: bold;
500+
background: blueviolet;
501+
}
502+
471503
#log_book {
472504
width: 100%;
473505
height: 100%;

0 commit comments

Comments
 (0)