Kitchen Kiosk

- %s @ %s ago. +""" + ) + f.write(f'{pageid} @ {age} ago.') + f.write( +"""

None: + logger.info( + 'renderer: Updating internal render statistics page.' + ) + with file_writer.file_writer(constants.render_stats_pagename) as f: + f.write( +''' + + + + + + + + +''') + for n, r in enumerate(renderer_catalog.get_renderers()): + if n % 2 == 0: + style = 'style="margin: 0; padding: 0; background: #c6b0b0;"' + else: + style = 'style="margin: 0; padding: 0; background: #eeeeee;"' + name = r.get_name() + last = last_render.get(name, None) + if last is None: + last = 'never' + else: + last = last.strftime('%Y/%m/%d %I:%M:%S%P') + count = render_counts.get(name, 0) + latency = render_times.get(name, np.array([])) + p25 = p50 = p75 = p90 = p99 = 'N/A' + try: + p25 = np.percentile(latency, 25) + p50 = np.percentile(latency, 50) + p75 = np.percentile(latency, 75) + p90 = np.percentile(latency, 90) + p99 = np.percentile(latency, 99) + f.write( +f''' + + + + + + +''' + ) + except IndexError: + pass + f.write('

Renderer Name	Last Run	Num Invocations	Render Latency
{name}	{last}	{count}	p25={p25:5.2f}, p50={p50:5.2f}, p75={p75:5.2f}, p90={p90:5.2f}, p99={p99:5.2f}

') + + def thread_invoke_renderers() -> None: + render_times: Dict[str, np.array] = {} + render_counts: collections.Counter = collections.Counter() + last_render: Dict[str, datetime] = {} + + # Touch the internal render page now to signal that we're alive. + renderer_update_internal_stats_page(last_render, render_counts, render_times) + + # Main renderer loop while True: - print(f"renderer[{utils.timestamp()}]: invoking all renderers in catalog...") + logger.info( + 'renderer: invoking all overdue renderers in catalog...' + ) for r in renderer_catalog.get_renderers(): + name = r.get_name() now = time.time() + logger.info(f'renderer: Invoking {name}\'s render method.') try: r.render() except Exception as e: - traceback.print_exc() - print( - f"renderer[{utils.timestamp()}] unknown exception in {r.get_name()}, swallowing it." + logger.exception(e) + logger.error( + f'renderer: Unexpected and unhandled exception ({e}) in {name}, swallowing it.' ) + continue + + # Increment the count of render operations per renderer. + render_counts[name] += 1 + + # Keep track of the last time we invoked each renderer. + last_render[name] = datetime_utils.now_pacific() + + # Record how long each render operation takes and warn if very long. delta = time.time() - now + times = render_times.get(name, np.array([])) + times = np.insert(times, 0, delta) + render_times[name] = times if delta > 1.0: - print( - f"renderer[{utils.timestamp()}]: Warning: {r.get_name()}'s rendering took {delta:%5.2f}s." + hdr = 'renderer: ' + logger.warning( +f''' +{hdr} Warning: {name}'s rendering took {delta:5.2f}s. +{hdr} FYI: {name}'s render times: p25={np.percentile(times, 25):5.2f}, p50={np.percentile(times, 50):5.2f}, p75={np.percentile(times, 75):5.2f}, p90={np.percentile(times, 90):5.2f}, p99={np.percentile(times, 99):5.2f} +''' ) - print( - f"renderer[{utils.timestamp()}]: thread having a little break for {constants.render_period_sec}s..." - ) + + # Update a page about internal stats of renderers. + renderer_update_internal_stats_page(last_render, render_counts, render_times) + logger.info('renderer: having a little nap...') time.sleep(constants.render_period_sec) -if __name__ == "__main__": - logging.basicConfig() +@bootstrap.initialize +def main() -> None: + command_queue: Queue = Queue() changer_thread: Optional[Thread] = None renderer_thread: Optional[Thread] = None + janitor_thread: Optional[Thread] = None + hotword_thread: Optional[Thread] = None while True: - if changer_thread is None or not changer_thread.is_alive(): - print( - f"MAIN[{utils.timestamp()}] - (Re?)initializing chooser thread... (wtf?!)" + if hotword_thread is None or not hotword_thread.is_alive(): + if hotword_thread is None: + logger.info('watchdog: Starting up the hotword detector thread...') + else: + logger.warning( + 'watchdog: The hotword detector thread seems to have died; restarting it and hoping for the best.' + ) + keyword_paths = [pvporcupine.KEYWORD_PATHS[x] for x in ["bumblebee"]] + sensitivities = [0.7] * len(keyword_paths) + listener = listen.HotwordListener( + command_queue, + keyword_paths, + sensitivities, ) - changer_thread = Thread(target=thread_change_current, args=()) + hotword_thread = Thread(target=listener.listen_forever, args=()) + hotword_thread.start() + + if changer_thread is None or not changer_thread.is_alive(): + if changer_thread is None: + logger.info('watchdog: Starting up the current page changer thread...') + else: + logger.warning( + 'watchdog: The current page changer thread seems to have died; restarting it and hoping for the best.' + ) + changer_thread = Thread(target=thread_change_current, args=(command_queue,)) changer_thread.start() + if renderer_thread is None or not renderer_thread.is_alive(): - print( - f"MAIN[{utils.timestamp()}] - (Re?)initializing render thread... (wtf?!)" - ) + if renderer_thread is None: + logger.info('watchdog: Starting up the page renderer thread...') + else: + logger.warning( + 'watchdog: The page renderer thread seems to have died; restarting it and hoping for the best.' + ) renderer_thread = Thread(target=thread_invoke_renderers, args=()) renderer_thread.start() - time.sleep(60) + + if janitor_thread is None or not janitor_thread.is_alive(): + if janitor_thread is None: + logger.info('watchdog: Starting up the memory janitor thread...') + else: + logger.warning( + 'watchdog: The memory janitor thread seems to have died; restarting it and hoping for the best.' + ) + janitor_thread = Thread(target=thread_janitor, args=()) + janitor_thread.start() + + # Have a little break and then check to make sure all threads are still alive. + logger.debug('watchdog: having a little nap.') + time.sleep(constants.check_threads_period_sec) + + +if __name__ == "__main__": + main()