kiosk.py

   1 #!/usr/bin/env python3
   2
   3 import collections
   4 from datetime import datetime
   5 from difflib import SequenceMatcher
   6 import gc
   7 import logging
   8 import os
   9 import re
  10 from threading import Thread
  11 import time
  12 import tracemalloc
  13 from typing import Dict, List, Optional, Tuple
  14 from queue import Queue
  15
  16 import astral  # type: ignore
  17 from astral.sun import sun  # type: ignore
  18 import numpy as np
  19 import pvporcupine
  20 import pytz
  21
  22 from pyutils import (
  23     bootstrap,
  24     config,
  25     logging_utils,
  26 )
  27 from pyutils.datetimes import datetime_utils
  28 from pyutils.files import file_utils
  29
  30 import kiosk_constants
  31 import file_writer
  32 import renderer_catalog
  33 import chooser
  34 import listen
  35 import trigger_catalog
  36
  37
  38 cfg = config.add_commandline_args(
  39     f"Kiosk Server ({__file__})", "A python server that runs a kiosk."
  40 )
  41 logger = logging.getLogger(__name__)
  42
  43
  44 @logging_utils.LoggingContext(logger, prefix="janitor:")
  45 def thread_janitor() -> None:
  46     tracemalloc.start()
  47     tracemalloc_target = 0.0
  48     gc_target = 0.0
  49     gc.enable()
  50
  51     # Main janitor loop; dump the largest pigs and force regular gcs.
  52     while True:
  53         now = time.time()
  54         if now > tracemalloc_target:
  55             tracemalloc_target = now + 30.0
  56             snapshot = tracemalloc.take_snapshot()
  57             snapshot = snapshot.filter_traces(
  58                 (
  59                     tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
  60                     tracemalloc.Filter(False, "<unknown>"),
  61                 )
  62             )
  63             key_type = "lineno"
  64             limit = 10
  65             top_stats = snapshot.statistics(key_type)
  66             logger.info(f"Top {limit} lines")
  67             for index, stat in enumerate(top_stats[:limit], 1):
  68                 frame = stat.traceback[0]
  69
  70                 # replace "/path/to/module/file.py" with "module/file.py"
  71                 filename = os.sep.join(frame.filename.split(os.sep)[-2:])
  72                 logger.info(
  73                     f"#{index}: {filename}:{frame.lineno}: {stat.size / 1024:.1f} KiB"
  74                 )
  75
  76             other = top_stats[limit:]
  77             if other:
  78                 size = sum(stat.size for stat in other)
  79                 logger.info(f"{len(other)} others: {size/1024:.1f} KiB")
  80             total = sum(stat.size for stat in top_stats)
  81             logger.info(f"Total allocated size: {total / 1024:.1f} KiB")
  82         if now > gc_target:
  83             logger.info("Kicking off a manual gc operation now.")
  84             gc.collect()
  85             gc_target = now + 120.0
  86         logger.info("Having a little nap...")
  87         time.sleep(30.0)
  88
  89
  90 def guess_page(command: str, page_chooser: chooser.chooser) -> str:
  91     def normalize_page(page: str) -> str:
  92         logger.debug(f"normalize_page input: {page}")
  93         page = page.replace("(", " ")
  94         page = page.replace("_", " ")
  95         page = page.replace(")", " ")
  96         page = page.replace(".html", "")
  97         page = page.replace("CNNNews", "news")
  98         page = page.replace("CNNTechnology", "technology")
  99         page = page.replace("gocostco", "costco list")
 100         page = page.replace("gohardware", "hardware list")
 101         page = page.replace("gohouse", "house list honey do")
 102         page = page.replace("gcal", "google calendar events")
 103         page = page.replace("mynorthwest", "northwest news")
 104         page = page.replace("myq", "myq garage door status")
 105         page = page.replace("gomenu", "dinner menu")
 106         page = page.replace("gmaps-seattle-unwrapped", "traffic")
 107         page = page.replace("gomenu", "dinner menu")
 108         page = page.replace("WSJNews", "news")
 109         page = page.replace("telma", "telma cabin")
 110         page = page.replace("WSJBusiness", "business news")
 111         page = re.sub(r"[0-9]+", "", page)
 112         logger.debug(f"normalize_page output: {page}")
 113         return page
 114
 115     logger.info(f"No exact match for f{command}; trying to guess...")
 116     best_page = None
 117     best_score = None
 118     for page in page_chooser.get_page_list():
 119         npage = normalize_page(page)
 120         score = SequenceMatcher(None, command, npage).ratio()
 121         if best_score is None or score > best_score:
 122             best_page = page
 123             best_score = score
 124     assert best_page is not None
 125     logger.info(f"Best guess for f{command} => {best_page} (score = {best_score})")
 126     return best_page
 127
 128
 129 def process_command(
 130     command: str, page_history: List[str], page_chooser
 131 ) -> Optional[str]:
 132     command = command.lower()
 133     logger.info(f"Parsing verbal command: {command}")
 134     page = None
 135     if "hold" in command:
 136         page = page_history[0]
 137     elif "down" in command:
 138         os.system(
 139             'xdotool search --onlyvisible "chrom" windowactivate click --repeat 8 5'
 140         )
 141         return None
 142     elif "up" in command:
 143         os.system(
 144             'xdotool search --onlyvisible "chrom" windowactivate click --repeat 8 4'
 145         )
 146         return None
 147     elif "back" in command:
 148         page = page_history[1]
 149     elif "skip" in command:
 150         while True:
 151             (page, _) = page_chooser.choose_next_page()
 152             if page == page_history[0]:
 153                 logger.debug(f"{page} is the same as last time!  Try again.")
 154             else:
 155                 break
 156     elif "internal" in command:
 157         if "render" in command:
 158             page = kiosk_constants.render_stats_pagename
 159         else:
 160             page = kiosk_constants.render_stats_pagename
 161     elif "weather" in command:
 162         if "telma" in command or "cabin" in command:
 163             page = "weather-telma_3_10800.html"
 164         elif "stevens" in command:
 165             page = "weather-stevens_3_10800.html"
 166         else:
 167             page = "weather-home_3_10800.html"
 168     elif "cabin" in command:
 169         if "list" in command:
 170             page = "Cabin-(gocabin)_2_3600.html"
 171         else:
 172             page = "hidden/cabin_driveway.html"
 173     elif "news" in command or "headlines" in command:
 174         page = "cnn-CNNNews_4_25900.html"
 175     elif "clock" in command or "time" in command:
 176         page = "clock_10_none.html"
 177     elif "countdown" in command or "countdowns" in command:
 178         page = "countdown_3_7200.html"
 179     elif "costco" in command:
 180         page = "Costco-(gocostco)_2_3600.html"
 181     elif "calendar" in command or "events" in command:
 182         page = "gcal_3_86400.html"
 183     elif "countdown" in command or "countdowns" in command:
 184         page = "countdown_3_7200.html"
 185     elif "grocery" in command or "groceries" in command:
 186         page = "Grocery-(gogrocery)_2_3600.html"
 187     elif "hardware" in command:
 188         page = "Hardware-(gohardware)_2_3600.html"
 189     elif "garage" in command:
 190         page = "myq_4_300.html"
 191     elif "menu" in command:
 192         page = "Menu-(gomenu)_2_3600.html"
 193     elif "cron" in command or "health" in command:
 194         page = "periodic-health_6_300.html"
 195     elif "photo" in command or "picture" in command:
 196         page = "photo_23_3600.html"
 197     elif "quote" in command or "quotation" in command or "quotes" in command:
 198         page = "quotes_4_10800.html"
 199     elif "stevens" in command:
 200         page = "stevens-conditions_1_86400.html"
 201     elif "stock" in command or "stocks" in command:
 202         page = "stock_3_86400.html"
 203     elif "twitter" in command:
 204         page = "twitter_10_3600.html"
 205     elif "traffic" in command:
 206         page = "gmaps-seattle-unwrapped_5_none.html"
 207     elif "front" in command and "door" in command:
 208         page = "hidden/frontdoor.html"
 209     elif "driveway" in command:
 210         page = "hidden/driveway.html"
 211     elif "backyard" in command:
 212         page = "hidden/backyard.html"
 213     else:
 214         page = guess_page(command, page_chooser)
 215     assert page is not None
 216     logger.info(f"Parsed to {page}")
 217     return page
 218
 219
 220 @logging_utils.LoggingContext(logger, prefix="chooser:")
 221 def thread_change_current(command_queue: Queue) -> None:
 222     page_history = ["", ""]
 223     swap_page_target = 0.0
 224
 225     def filter_news_during_dinnertime(page: str) -> bool:
 226         now = datetime.now(tz=pytz.timezone("US/Pacific"))
 227         is_dinnertime = now.hour >= 18 and now.hour <= 20
 228         return not is_dinnertime or not (
 229             "cnn" in page
 230             or "news" in page
 231             or "mynorthwest" in page
 232             or "seattle" in page
 233             or "wsj" in page
 234         )
 235
 236     def check_for_command() -> Tuple[Optional[str], Optional[str]]:
 237         command = None
 238         try:
 239             command = command_queue.get(block=False)
 240         except Exception:
 241             command = None
 242
 243         if command:
 244             logger.info(f'We got a verbal command ("{command}"), parsing it...')
 245             page = process_command(command, page_history, page_chooser)
 246             if page:
 247                 return page, command
 248         return None, None
 249
 250     def choose_page_randomly() -> Tuple[str, bool]:
 251         while True:
 252             (page, triggered) = page_chooser.choose_next_page()
 253             if triggered:
 254                 logger.info("A trigger is active...")
 255                 break
 256             else:
 257                 if page == page_history[0]:
 258                     logger.debug(f"{page} is the same as last time! Try again.")
 259                 else:
 260                     break
 261         return (page, triggered)
 262
 263     page_chooser = chooser.weighted_random_chooser_with_triggers(
 264         trigger_catalog.get_triggers(), [filter_news_during_dinnertime]
 265     )
 266     current_file = os.path.join(kiosk_constants.pages_dir, "current.shtml")
 267     emergency_file = os.path.join(kiosk_constants.pages_dir, "reload_immediately.html")
 268
 269     # Main chooser loop -- note that this executes a couple of times per second
 270     # to be responsive to triggers.
 271     while True:
 272         now = time.time()
 273         page = None
 274         triggered = False
 275         command = None
 276
 277         # Is there a verbal command to parse?
 278         (page, command) = check_for_command()
 279         if page and command:
 280             triggered = True
 281         else:
 282             # If not, choose the a (new!) page randomly or respond to a trigger.
 283             (page, triggered) = choose_page_randomly()
 284
 285         # By now we have a page and we may be triggered.
 286         if triggered:
 287
 288             # If we are triggered and the page is new, change the
 289             # current page to the new page immediately.
 290             if page != page_history[0]:
 291                 logger.info(
 292                     f"An emergency page reload to {page} is needed at this time."
 293                 )
 294                 swap_page_target = now + kiosk_constants.emergency_refresh_period_sec
 295
 296                 # Set current.shtml to the triggered page.
 297                 try:
 298                     with open(current_file, "w") as f:
 299                         emit(
 300                             f,
 301                             page,
 302                             override_refresh_sec=kiosk_constants.emergency_refresh_period_sec,
 303                             command=command,
 304                         )
 305                     logger.debug(f"Wrote {current_file}.")
 306                 except Exception:
 307                     logger.exception(
 308                         f"Unexpected exception; assuming {page} doesn't exist?!"
 309                     )
 310                     continue
 311
 312                 # Notify XMLHTTP clients that they need to refresh now.
 313                 with open(emergency_file, "w") as f:
 314                     f.write(f"Reload, suckers... you HAVE to see {page}!")
 315                 logger.debug(f"Wrote {emergency_file}...")
 316
 317                 # Fix this hack... maybe read the webserver logs and see if it
 318                 # actually was picked up?
 319                 time.sleep(0.999)
 320                 os.remove(emergency_file)
 321                 logger.debug(f"...and removed {emergency_file}.")
 322                 page_history.insert(0, page)
 323                 page_history = page_history[0:10]
 324
 325         # If we're not triggered, only render a new page if time has expired.
 326         elif now >= swap_page_target:
 327             logger.info(f"Nominal choice of {page} as the next to show.")
 328             swap_page_target = now + kiosk_constants.refresh_period_sec
 329             try:
 330                 with open(current_file, "w") as f:
 331                     emit(f, page)
 332                 logger.debug(f"Wrote {current_file}.")
 333             except Exception:
 334                 logger.exception(
 335                     f"Unexpected exception; assuming {page} doesn't exist?!"
 336                 )
 337                 continue
 338             page_history.insert(0, page)
 339             page_history = page_history[0:10]
 340
 341         time.sleep(0.5)
 342
 343
 344 def emit(f, filename: str, *, override_refresh_sec: int = None, command: str = None):
 345     if "unwrapped" not in filename:
 346         logger.debug(f"Emitting {filename} wrapped.")
 347         emit_wrapped(
 348             f, filename, override_refresh_sec=override_refresh_sec, command=command
 349         )
 350     else:
 351         logger.debug(f"Emitting {filename} raw.")
 352         emit_raw(f, filename)
 353
 354
 355 def emit_raw(f, filename: str):
 356     f.write(f'<!--#include virtual="{filename}"-->')
 357
 358
 359 def emit_wrapped(
 360     f, filename: str, *, override_refresh_sec: int = None, command: str = None
 361 ) -> None:
 362     def pick_background_color() -> str:
 363         now = datetime_utils.now_pacific()
 364         city = astral.LocationInfo("Bellevue", "USA", "US/Pacific", 47.610, -122.201)
 365         s = sun(city.observer, date=now, tzinfo=pytz.timezone("US/Pacific"))
 366         sunrise_mod = datetime_utils.minute_number(
 367             s["sunrise"].hour, s["sunrise"].minute
 368         )
 369         sunset_mod = datetime_utils.minute_number(s["sunset"].hour, s["sunset"].minute)
 370         now_mod = datetime_utils.minute_number(now.hour, now.minute)
 371         if now_mod < sunrise_mod or now_mod > (sunset_mod + 45):
 372             return "E6B8B8"
 373         elif now_mod < (sunrise_mod + 45) or now_mod > (sunset_mod + 120):
 374             return "EECDCD"
 375         else:
 376             return "FFFFFF"
 377
 378     def get_refresh_period() -> float:
 379         if override_refresh_sec is not None:
 380             return float(override_refresh_sec * 1000.0)
 381         now = datetime.now(tz=pytz.timezone("US/Pacific"))
 382         if now.hour < 6:
 383             return float(kiosk_constants.refresh_period_night_sec * 1000.0)
 384         else:
 385             return float(kiosk_constants.refresh_period_sec * 1000.0)
 386
 387     age = file_utils.describe_file_ctime(f"pages/{filename}")
 388     bgcolor = pick_background_color()
 389     if command is None:
 390         pageid = filename
 391     else:
 392         pageid = f'"{command}" -> {filename}'
 393
 394     f.write(
 395         """
 396 <HEAD>
 397   <TITLE>Kitchen Kiosk</TITLE>
 398   <LINK rel="stylesheet" type="text/css" href="style.css">
 399   <meta name="google" content="notranslate">
 400   <SCRIPT TYPE="text/javascript">
 401
 402   // Zoom the 'contents' div to fit without scrollbars and then make
 403   // it visible.
 404   function zoomScreen() {
 405     z = 285;
 406     do {
 407       document.getElementById("content").style.zoom = z+"%";
 408       var body = document.body;
 409       var html = document.documentElement;
 410       var height = Math.max(body.scrollHeight,
 411                             body.offsetHeight,
 412                             html.clientHeight,
 413                             html.scrollHeight,
 414                             html.offsetHeight);
 415       var windowHeight = window.innerHeight;
 416       var width = Math.max(body.scrollWidth,
 417                            body.offsetWidth,
 418                            html.clientWidth,
 419                            html.scrollWidth,
 420                            html.offsetWidth);
 421       var windowWidth = window.innerWidth;
 422       var heightRatio = height / windowHeight;
 423       var widthRatio = width / windowWidth;
 424
 425       if (heightRatio <= 1.0 && widthRatio <= 1.0) {
 426         break;
 427       }
 428       z -= 4;
 429     } while(z >= 70);
 430     document.getElementById("content").style.visibility = "visible";
 431   }
 432
 433   // Load IMG tags with DATA-SRC attributes late.
 434   function lateLoadImages() {
 435     var image = document.getElementsByTagName('img');
 436     for (var i = 0; i < image.length; i++) {
 437       if (image[i].getAttribute('DATA-SRC')) {
 438         image[i].setAttribute('SRC', image[i].getAttribute('DATA-SRC'));
 439       }
 440     }
 441   }
 442 """
 443     )
 444     f.write(
 445         """
 446   // Operate the clock at the top of the page.
 447   function runClock() {
 448     var today = new Date();
 449     var h = today.getHours();
 450     var ampm = h >= 12 ? 'pm' : 'am';
 451     h = h %% 12;
 452     h = h ? h : 12; // the hour '0' should be '12'
 453     var m = maybeAddZero(today.getMinutes());
 454     var colon = ":";
 455     if (today.getSeconds() %% 2 == 0) {
 456       colon = "<FONT STYLE='color: #%s; font-size: 4vmin; font-weight: bold'>:</FONT>";
 457     }
 458     document.getElementById("time").innerHTML = h + colon + m + ampm;
 459     document.getElementById("date").innerHTML = today.toDateString();
 460     var t = setTimeout(function(){runClock()}, 1000);
 461   }
 462 """
 463         % bgcolor
 464     )
 465     f.write(
 466         """
 467   // Helper method for running the clock.
 468   function maybeAddZero(x) {
 469     return (x < 10) ? "0" + x : x;
 470   }
 471
 472   // Do something on page load.
 473   function addLoadEvent(func) {
 474     var oldonload = window.onload;
 475     if (typeof window.onload != 'function') {
 476       window.onload = func;
 477     } else {
 478       window.onload = function() {
 479         if (oldonload) {
 480           oldonload();
 481         }
 482         func();
 483       }
 484     }
 485   }
 486
 487   // Sleep thread helper.
 488   const sleep = (milliseconds) => {
 489     return new Promise(resolve => setTimeout(resolve, milliseconds))
 490   }
 491
 492   var loadedDate = new Date();
 493
 494   addLoadEvent(zoomScreen);
 495   addLoadEvent(runClock);
 496   addLoadEvent(lateLoadImages);
 497 """
 498     )
 499     f.write(
 500         """
 501   // Runs the countdown line at the bottom and is responsible for
 502   // normal page reloads caused by the expiration of a timer.
 503   (function countdown() {
 504     setTimeout(
 505       function() {
 506         var now = new Date();
 507         var deltaMs = now.getTime() - loadedDate.getTime();
 508         var totalMs = %d;
 509         var remainingMs = (totalMs - deltaMs);
 510
 511         if (remainingMs > 0) {
 512           var hr = document.getElementById("countdown");
 513           var width = (remainingMs / (totalMs - 5000)) * 100.0;
 514           if (width <= 100) {
 515             hr.style.visibility = "visible";
 516             hr.style.width = " ".concat(width, "%%");
 517             hr.style.backgroundColor = "maroon";
 518           }
 519         } else {
 520           // Reload unconditionally after 22 sec.
 521           window.location.reload(true);
 522         }
 523
 524         // Brief sleep before doing it all over again.
 525         sleep(50).then(() => {
 526           countdown();
 527         });
 528       }, 50)
 529   })();
 530 """
 531         % get_refresh_period()
 532     )
 533     f.write(
 534         """
 535   function check_reload() {
 536     var xhr = new XMLHttpRequest();
 537     xhr.open('GET',
 538              '%s/reload_immediately.html');
 539     xhr.onload =
 540       function() {
 541         if (xhr.status === 200) {
 542           window.location.reload(true);
 543         }
 544       };
 545     xhr.send();
 546   }
 547
 548   // Periodically checks for emergency reload events.
 549   setInterval(check_reload, 500);
 550   </SCRIPT>
 551 </HEAD>
 552 """
 553         % kiosk_constants.root_url
 554     )
 555     f.write(f'<BODY BGCOLOR="#{bgcolor}">')
 556     f.write(
 557         """
 558     <TABLE style="height:100%; width:100%" BORDER=0>
 559     <TR HEIGHT=30>
 560         <TD ALIGN="left">
 561             <DIV id="date">&nbsp;</DIV>
 562         </TD>
 563         <TD ALIGN="center"><FONT COLOR=#bbbbbb>
 564             <DIV id="info"></DIV></FONT>
 565         </TD>
 566         <TD ALIGN="right">
 567             <DIV id="time">&nbsp;</DIV>
 568         </TD>
 569     </TR>
 570     <TR STYLE="vertical-align:top">
 571         <TD COLSPAN=3>
 572             <DIV ID="content" STYLE="zoom: 1; visibility: hidden;">
 573               <!-- BEGIN main page contents. -->
 574 """
 575     )
 576     f.write(f'<!--#include virtual="{filename}"-->')
 577     f.write(
 578         """
 579             <!-- END main page contents. -->
 580             </DIV>
 581             <BR>
 582             <DIV STYLE="position: absolute; top:1030px; width:99%">
 583             <P ALIGN="right">
 584 """
 585     )
 586     f.write(f"<FONT SIZE=2 COLOR=#bbbbbb>{pageid} @ {age} ago.</FONT>")
 587     f.write(
 588         """
 589             </P>
 590             <HR id="countdown" STYLE="width:0px;
 591                                       text-align:left;
 592                                       margin:0;
 593                                       border:none;
 594                                       border-width:0;
 595                                       height:5px;
 596                                       visibility:hidden;
 597                                       background-color:#ffffff;">
 598             </DIV>
 599         </TD>
 600     </TR>
 601     </TABLE>
 602 </BODY>"""
 603     )
 604
 605
 606 def renderer_update_internal_stats_page(
 607     last_render: Dict[str, datetime],
 608     render_counts: collections.Counter,
 609     render_times: Dict[str, np.array],
 610 ) -> None:
 611     logger.info("Updating internal render statistics page.")
 612     with file_writer.file_writer(kiosk_constants.render_stats_pagename) as f:
 613         f.write(
 614             """
 615 <CENTER>
 616 <TABLE BORDER=0 WIDTH=95%>
 617     <TR>
 618     <TH><B>Renderer Name</B></TH>
 619     <TH><B>Last Run</B></TH>
 620     <TH><B>Num Invocations</B></TH>
 621     <TH><B>Render Latency</B></TH>
 622     </TR>
 623 """
 624         )
 625         for n, r in enumerate(renderer_catalog.get_renderers()):
 626             if n % 2 == 0:
 627                 style = 'style="margin: 0; padding: 0; background: #c6b0b0;"'
 628             else:
 629                 style = 'style="margin: 0; padding: 0; background: #eeeeee;"'
 630             name = r.get_name()
 631             last = last_render.get(name, None)
 632             if last is None:
 633                 last = "never"
 634             else:
 635                 last = last.strftime("%Y/%m/%d %I:%M:%S%P")
 636             count = render_counts.get(name, 0)
 637             latency = render_times.get(name, np.array([]))
 638             p25 = p50 = p75 = p90 = p99 = "N/A"
 639             try:
 640                 p25 = np.percentile(latency, 25)
 641                 p50 = np.percentile(latency, 50)
 642                 p75 = np.percentile(latency, 75)
 643                 p90 = np.percentile(latency, 90)
 644                 p99 = np.percentile(latency, 99)
 645                 f.write(
 646                     f"""
 647     <TR>
 648     <TD {style}>{name}&nbsp;</TD>
 649     <TD {style}>&nbsp;{last}&nbsp;</TD>
 650     <TD {style}><CENTER>&nbsp;{count}&nbsp;</CENTER></TD>
 651     <TD {style}>&nbsp;p25={p25:5.2f}, p50={p50:5.2f}, p75={p75:5.2f}, p90={p90:5.2f}, p99={p99:5.2f}</TD>
 652     </TR>
 653 """
 654                 )
 655             except IndexError:
 656                 pass
 657         f.write("</TABLE>")
 658
 659
 660 @logging_utils.LoggingContext(logger, prefix="renderer:")
 661 def thread_invoke_renderers() -> None:
 662     render_times: Dict[str, np.array] = {}
 663     render_counts: collections.Counter = collections.Counter()
 664     last_render: Dict[str, datetime] = {}
 665
 666     # Touch the internal render page now to signal that we're alive.
 667     renderer_update_internal_stats_page(last_render, render_counts, render_times)
 668
 669     # Main renderer loop
 670     while True:
 671         logger.info("invoking all overdue renderers in catalog...")
 672         for r in renderer_catalog.get_renderers():
 673             name = r.get_name()
 674             now = time.time()
 675             logger.info(f"Invoking {name}'s render method.")
 676             try:
 677                 r.render()
 678             except Exception as e:
 679                 logger.exception(e)
 680                 logger.error(
 681                     f"Unexpected and unhandled exception ({e}) in {name}, swallowing it."
 682                 )
 683                 continue
 684
 685             # Increment the count of render operations per renderer.
 686             render_counts[name] += 1
 687
 688             # Keep track of the last time we invoked each renderer.
 689             last_render[name] = datetime_utils.now_pacific()
 690
 691             # Record how long each render operation takes and warn if very long.
 692             delta = time.time() - now
 693             times = render_times.get(name, np.array([]))
 694             times = np.insert(times, 0, delta)
 695             render_times[name] = times
 696             if delta > 1.0:
 697                 logger.warning(
 698                     f"""
 699 Warning: {name}'s rendering took {delta:5.2f}s.
 700 FYI: {name}'s render times: p25={np.percentile(times, 25):5.2f}, p50={np.percentile(times, 50):5.2f}, p75={np.percentile(times, 75):5.2f}, p90={np.percentile(times, 90):5.2f}, p99={np.percentile(times, 99):5.2f}
 701 """
 702                 )
 703
 704         # Update a page about internal stats of renderers.
 705         renderer_update_internal_stats_page(last_render, render_counts, render_times)
 706         logger.info("Having a little nap...")
 707         time.sleep(kiosk_constants.render_period_sec)
 708
 709
 710 @bootstrap.initialize
 711 def main() -> None:
 712     command_queue: Queue = Queue()
 713     changer_thread: Optional[Thread] = None
 714     renderer_thread: Optional[Thread] = None
 715     janitor_thread: Optional[Thread] = None
 716     hotword_thread: Optional[Thread] = None
 717
 718     with logging_utils.LoggingContext(logger, prefix="watchdog:"):
 719         while True:
 720             if hotword_thread is None or not hotword_thread.is_alive():
 721                 if hotword_thread is None:
 722                     logger.info("Starting up the hotword detector thread...")
 723                 else:
 724                     logger.warning(
 725                         "The hotword detector thread seems to have died; restarting it and hoping for the best."
 726                     )
 727                 keyword_paths = [pvporcupine.KEYWORD_PATHS[x] for x in ["bumblebee"]]
 728                 sensitivities = [0.7] * len(keyword_paths)
 729                 listener = listen.HotwordListener(
 730                     command_queue,
 731                     keyword_paths,
 732                     sensitivities,
 733                 )
 734                 hotword_thread = Thread(target=listener.listen_forever, args=())
 735                 hotword_thread.start()
 736
 737             if changer_thread is None or not changer_thread.is_alive():
 738                 if changer_thread is None:
 739                     logger.info("Starting up the current page changer thread...")
 740                 else:
 741                     logger.warning(
 742                         "The current page changer thread seems to have died; restarting it and hoping for the best."
 743                     )
 744                 changer_thread = Thread(
 745                     target=thread_change_current, args=(command_queue,)
 746                 )
 747                 changer_thread.start()
 748
 749             if renderer_thread is None or not renderer_thread.is_alive():
 750                 if renderer_thread is None:
 751                     logger.info("Starting up the page renderer thread...")
 752                 else:
 753                     logger.warning(
 754                         "The page renderer thread seems to have died; restarting it and hoping for the best."
 755                     )
 756                 renderer_thread = Thread(target=thread_invoke_renderers, args=())
 757                 renderer_thread.start()
 758
 759             if janitor_thread is None or not janitor_thread.is_alive():
 760                 if janitor_thread is None:
 761                     logger.info("Starting up the memory janitor thread...")
 762                 else:
 763                     logger.warning(
 764                         "The memory janitor thread seems to have died; restarting it and hoping for the best."
 765                     )
 766                 janitor_thread = Thread(target=thread_janitor, args=())
 767                 janitor_thread.start()
 768
 769             # Have a little break and then check to make sure all threads are still alive.
 770             logger.debug("Having a little nap...")
 771             time.sleep(kiosk_constants.check_threads_period_sec)
 772
 773
 774 if __name__ == "__main__":
 775     main()