|
@@ -462,13 +532,21 @@ def emit_wrapped(f,
|
-
-
+"""
+ )
+ f.write(f'')
+ f.write(
+"""
+
-
+
- %s @ %s ago.
+"""
+ )
+ f.write(f'{pageid} @ {age} ago.')
+ f.write(
+"""
None:
+ logger.info(
+ 'renderer: Updating internal render statistics page.'
+ )
+ with file_writer.file_writer(constants.render_stats_pagename) as f:
+ f.write(
+'''
+
+
+
+ | Renderer Name |
+ Last Run |
+ Num Invocations |
+ Render Latency |
+
+''')
+ for n, r in enumerate(renderer_catalog.get_renderers()):
+ if n % 2 == 0:
+ style = 'style="margin: 0; padding: 0; background: #c6b0b0;"'
+ else:
+ style = 'style="margin: 0; padding: 0; background: #eeeeee;"'
+ name = r.get_name()
+ last = last_render.get(name, None)
+ if last is None:
+ last = 'never'
+ else:
+ last = last.strftime('%Y/%m/%d %I:%M:%S%P')
+ count = render_counts.get(name, 0)
+ latency = render_times.get(name, np.array([]))
+ p25 = p50 = p75 = p90 = p99 = 'N/A'
+ try:
+ p25 = np.percentile(latency, 25)
+ p50 = np.percentile(latency, 50)
+ p75 = np.percentile(latency, 75)
+ p90 = np.percentile(latency, 90)
+ p99 = np.percentile(latency, 99)
+ f.write(
+f'''
+
+ | {name} |
+ {last} |
+ {count} |
+ p25={p25:5.2f}, p50={p50:5.2f}, p75={p75:5.2f}, p90={p90:5.2f}, p99={p99:5.2f} |
+
+'''
+ )
+ except IndexError:
+ pass
+ f.write(' ')
+
+
def thread_invoke_renderers() -> None:
render_times: Dict[str, np.array] = {}
render_counts: collections.Counter = collections.Counter()
last_render: Dict[str, datetime] = {}
+ # Touch the internal render page now to signal that we're alive.
+ renderer_update_internal_stats_page(last_render, render_counts, render_times)
+
+ # Main renderer loop
while True:
- print(f'renderer[{utils.timestamp()}]: invoking all overdue renderers in catalog...')
+ logger.info(
+ 'renderer: invoking all overdue renderers in catalog...'
+ )
for r in renderer_catalog.get_renderers():
name = r.get_name()
now = time.time()
+ logger.info(f'renderer: Invoking {name}\'s render method.')
try:
r.render()
except Exception as e:
- traceback.print_exc(file=sys.stdout)
logger.exception(e)
- print(
- f"renderer[{utils.timestamp()}] Unknown exception ({e}) in {name}, swallowing it."
+ logger.error(
+ f'renderer: Unexpected and unhandled exception ({e}) in {name}, swallowing it.'
)
+ continue
# Increment the count of render operations per renderer.
render_counts[name] += 1
# Keep track of the last time we invoked each renderer.
- last_render[name] = datetime.now(tz=pytz.timezone("US/Pacific"))
+ last_render[name] = datetime_utils.now_pacific()
# Record how long each render operation takes and warn if very long.
delta = time.time() - now
@@ -526,68 +657,22 @@ def thread_invoke_renderers() -> None:
times = np.insert(times, 0, delta)
render_times[name] = times
if delta > 1.0:
- hdr = f'renderer[{utils.timestamp()}]:'
- print(
+ hdr = 'renderer: '
+ logger.warning(
f'''
{hdr} Warning: {name}'s rendering took {delta:5.2f}s.
{hdr} FYI: {name}'s render times: p25={np.percentile(times, 25):5.2f}, p50={np.percentile(times, 50):5.2f}, p75={np.percentile(times, 75):5.2f}, p90={np.percentile(times, 90):5.2f}, p99={np.percentile(times, 99):5.2f}
-''')
-
- # Render a page about internal stats of renderers.
- print(f'renderer[{utils.timestamp()}]: Updating internal statistics page.')
- with file_writer.file_writer(constants.internal_stats_pagename) as f:
- f.write(
-f'''
-
-
- | Renderer Name |
- Last Run |
- Num Invocations |
- Render Latency |
-
-''')
- for n, r in enumerate(renderer_catalog.get_renderers()):
- if n % 2 == 0:
- style = 'style="margin: 0; padding: 0; background: #c6b0b0;"'
- else:
- style = 'style="margin: 0; padding: 0; background: #eeeeee;"'
- name = r.get_name()
- last = last_render.get(name, None)
- if last is None:
- last = 'never'
- else:
- last = last.strftime('%Y/%m/%d %I:%M:%S%P')
- count = render_counts.get(name, 0)
- latency = render_times.get(name, np.array([]))
- p25 = p50 = p75 = p90 = p99 = 'N/A'
- try:
- p25 = np.percentile(latency, 25)
- p50 = np.percentile(latency, 50)
- p75 = np.percentile(latency, 75)
- p90 = np.percentile(latency, 90)
- p99 = np.percentile(latency, 99)
- except IndexError:
- pass
- f.write(
-f'''
-
- | {name} |
- {last} |
- {count} |
- p25={p25:5.2f}, p50={p50:5.2f}, p75={p75:5.2f}, p90={p90:5.2f}, p99={p99:5.2f} |
-
-''')
- f.write(' ')
+'''
+ )
- print(
- f"renderer[{utils.timestamp()}]: " +
- f"thread having a little break for {constants.render_period_sec}s..."
- )
+ # Update a page about internal stats of renderers.
+ renderer_update_internal_stats_page(last_render, render_counts, render_times)
+ logger.info('renderer: having a little nap...')
time.sleep(constants.render_period_sec)
-if __name__ == "__main__":
- logging.basicConfig()
+@bootstrap.initialize
+def main() -> None:
command_queue: Queue = Queue()
changer_thread: Optional[Thread] = None
renderer_thread: Optional[Thread] = None
@@ -595,6 +680,12 @@ if __name__ == "__main__":
hotword_thread: Optional[Thread] = None
while True:
if hotword_thread is None or not hotword_thread.is_alive():
+ if hotword_thread is None:
+ logger.info('watchdog: Starting up the hotword detector thread...')
+ else:
+ logger.warning(
+ 'watchdog: The hotword detector thread seems to have died; restarting it and hoping for the best.'
+ )
keyword_paths = [pvporcupine.KEYWORD_PATHS[x] for x in ["bumblebee"]]
sensitivities = [0.7] * len(keyword_paths)
listener = listen.HotwordListener(
@@ -604,22 +695,41 @@ if __name__ == "__main__":
)
hotword_thread = Thread(target=listener.listen_forever, args=())
hotword_thread.start()
+
if changer_thread is None or not changer_thread.is_alive():
- print(
- f"MAIN[{utils.timestamp()}] - (Re?)initializing chooser thread... (wtf?!)"
- )
+ if changer_thread is None:
+ logger.info('watchdog: Starting up the current page changer thread...')
+ else:
+ logger.warning(
+ 'watchdog: The current page changer thread seems to have died; restarting it and hoping for the best.'
+ )
changer_thread = Thread(target=thread_change_current, args=(command_queue,))
changer_thread.start()
+
if renderer_thread is None or not renderer_thread.is_alive():
- print(
- f"MAIN[{utils.timestamp()}] - (Re?)initializing render thread... (wtf?!)"
- )
+ if renderer_thread is None:
+ logger.info('watchdog: Starting up the page renderer thread...')
+ else:
+ logger.warning(
+ 'watchdog: The page renderer thread seems to have died; restarting it and hoping for the best.'
+ )
renderer_thread = Thread(target=thread_invoke_renderers, args=())
renderer_thread.start()
+
if janitor_thread is None or not janitor_thread.is_alive():
- print(
- f"MAIN[{utils.timestamp()}] - (Re?)initializing janitor thread... (wtf?!)"
- )
+ if janitor_thread is None:
+ logger.info('watchdog: Starting up the memory janitor thread...')
+ else:
+ logger.warning(
+ 'watchdog: The memory janitor thread seems to have died; restarting it and hoping for the best.'
+ )
janitor_thread = Thread(target=thread_janitor, args=())
janitor_thread.start()
- time.sleep(60)
+
+ # Have a little break and then check to make sure all threads are still alive.
+ logger.debug('watchdog: having a little nap.')
+ time.sleep(constants.check_threads_period_sec)
+
+
+if __name__ == "__main__":
+ main()
|