- cmd = f'{rsync} {bundle.code_file} {username}@{machine}:{bundle.code_file}'
- logger.debug(f"Copying work to {worker} via {cmd}")
- exec_utils.run_silently(cmd)
-
- # Before we do more work, make sure it's still viable.
- if self.check_if_cancelled(bundle):
- return self.post_launch_work(bundle)
-
- # Fucking Apple has a python3 binary in /usr/sbin that is not
- # the one we want and is protected by the OS so make sure that
- # /usr/local/bin is early in the path.
- cmd = (f'{ssh} {bundle.username}@{bundle.machine} '
- f'"export PATH=/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/home/scott/bin:/home/scott/.local/bin; /home/scott/lib/python_modules/remote_worker.py'
+ try:
+ cmd = f'{RSYNC} {bundle.code_file} {username}@{machine}:{bundle.code_file}'
+ start_ts = time.time()
+ logger.info(f"{uuid}/{fname}: Copying work to {worker} via {cmd}.")
+ run_silently(cmd)
+ xfer_latency = time.time() - start_ts
+ logger.info(f"{uuid}/{fname}: Copying done in {xfer_latency:.1f}s.")
+ except Exception as e:
+ logger.exception(e)
+ logger.error(
+ f'{uuid}/{fname}: failed to send instructions to worker machine?!?'
+ )
+ assert bundle.worker is not None
+ self.status.record_release_worker(
+ bundle.worker,
+ bundle.uuid,
+ True,
+ )
+ self.release_worker(bundle.worker)
+ self.adjust_task_count(-1)
+ if is_original:
+ # Weird. We tried to copy the code to the worker and it failed...
+ # And we're the original bundle. We have to retry.
+ return self.emergency_retry_nasty_bundle(bundle)
+ else:
+ # This is actually expected; we're a backup.
+ # There's a race condition where someone else
+ # already finished the work and removed the source
+ # code file before we could copy it. No biggie.
+ return None
+
+ # Kick off the work. Note that if this fails we let
+ # wait_for_process deal with it.
+ self.status.record_processing_began(uuid)
+ cmd = (f'{SSH} {bundle.username}@{bundle.machine} '
+ f'"source py39-venv/bin/activate &&'
+ f' /home/scott/lib/python_modules/remote_worker.py'