machine: Optional[str]
"""The remote machine running this bundle or None if none (yet)"""
- hostname: str
+ controller: str
"""The controller machine"""
code_file: str
total_finished = len(self.finished_bundle_timings)
total_in_flight = self.total_in_flight()
ret = f'\n\n{underline()}Remote Executor Pool Status{reset()}: '
- qall = None
+ qall_median = None
+ qall_p95 = None
if len(self.finished_bundle_timings) > 1:
qall_median = self.finished_bundle_timings.get_median()
qall_p95 = self.finished_bundle_timings.get_percentile(95)
if details is not None:
details.slower_than_local_p95 = False
- if qall is not None:
- if sec > qall[1]:
+ if qall_p95 is not None:
+ if sec > qall_p95:
ret += f'{bg("red")}>∀p95{reset()} '
if details is not None:
details.slower_than_global_p95 = True
self.adjust_task_count(+1)
uuid = bundle.uuid
- hostname = bundle.hostname
+ controller = bundle.controller
avoid_machine = override_avoid_machine
is_original = bundle.src_bundle is None
return None
# Send input code / data to worker machine if it's not local.
- if hostname not in machine:
+ if controller not in machine:
try:
cmd = (
f'{SCP} {bundle.code_file} {username}@{machine}:{bundle.code_file}'
bundle.end_ts = time.time()
if not was_cancelled:
assert bundle.machine is not None
- if bundle.hostname not in bundle.machine:
+ if bundle.controller not in bundle.machine:
cmd = f'{SCP} {username}@{machine}:{result_file} {result_file} 2>/dev/null'
logger.info(
"%s: Fetching results back from %s@%s via %s",
worker=None,
username=None,
machine=None,
- hostname=platform.node(),
+ controller=platform.node(),
code_file=code_file,
result_file=result_file,
pid=0,
worker=None,
username=None,
machine=None,
- hostname=src_bundle.hostname,
+ controller=src_bundle.controller,
code_file=src_bundle.code_file,
result_file=src_bundle.result_file,
pid=0,
raise RemoteExecutorException(
f'{bundle}: This bundle can\'t be completed despite several backups and retries',
)
- else:
- logger.error(
- '%s: At least it\'s only a backup; better luck with the others.',
- bundle,
- )
+ logger.error(
+ '%s: At least it\'s only a backup; better luck with the others.',
+ bundle,
+ )
return None
else:
msg = f'>>> Emergency rescheduling {bundle} because of unexected errors (wtf?!) <<<'