Fix cameras, improve weather, delegate health renderer to a helper,
[kiosk.git] / kiosk.py
1 #!/usr/bin/env python3
2
3 import collections
4 from datetime import datetime
5 from difflib import SequenceMatcher
6 import gc
7 import logging
8 import os
9 import re
10 from threading import Thread
11 import time
12 import tracemalloc
13 from typing import Dict, List, Optional
14 from queue import Queue
15
16 import astral  # type: ignore
17 from astral.sun import sun  # type: ignore
18 import numpy as np
19 import pvporcupine
20 import pytz
21
22 import bootstrap
23 import config
24 import datetime_utils
25
26 import constants
27 import file_writer
28 import renderer_catalog
29 import chooser
30 import listen
31 import trigger_catalog
32 import utils
33
34
35 cfg = config.add_commandline_args(
36     f'Kiosk Server ({__file__})',
37     'A python server that runs a kiosk.'
38 )
39 logger = logging.getLogger(__file__)
40
41
42 def thread_janitor() -> None:
43     tracemalloc.start()
44     tracemalloc_target = 0.0
45     gc_target = 0.0
46     gc.enable()
47
48     # Main janitor loop; dump the largest pigs and force regular gcs.
49     while True:
50         now = time.time()
51         if now > tracemalloc_target:
52             tracemalloc_target = now + 30.0
53             snapshot = tracemalloc.take_snapshot()
54             snapshot = snapshot.filter_traces((
55                 tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
56                 tracemalloc.Filter(False, "<unknown>"),
57             ))
58             key_type = 'lineno'
59             limit = 10
60             top_stats = snapshot.statistics(key_type)
61             logger.info(f'janitor: Top {limit} lines')
62             for index, stat in enumerate(top_stats[:limit], 1):
63                 frame = stat.traceback[0]
64
65                 # replace "/path/to/module/file.py" with "module/file.py"
66                 filename = os.sep.join(frame.filename.split(os.sep)[-2:])
67                 logger.info(
68                     f'janitor: #{index}: {filename}:{frame.lineno}: {stat.size / 1024:.1f} KiB'
69                 )
70
71             other = top_stats[limit:]
72             if other:
73                 size = sum(stat.size for stat in other)
74                 logger.info(
75                     f'janitor: {len(other)} others: {size/1024:.1f} KiB'
76                 )
77             total = sum(stat.size for stat in top_stats)
78             logger.info(
79                 f'janitor: Total allocated size: {total / 1024:.1f} KiB'
80             )
81         if now > gc_target:
82             logger.info('janitor: kicking off a manual gc operation now.')
83             gc.collect()
84             gc_target = now + 120.0
85         time.sleep(30.0)
86
87
88 def guess_page(command: str, page_chooser: chooser.chooser) -> str:
89
90     def normalize_page(page: str) -> str:
91         logger.debug(f'normalize_page input: {page}')
92         page = page.replace('(', ' ')
93         page = page.replace('_', ' ')
94         page = page.replace(')', ' ')
95         page = page.replace('.html', '')
96         page = page.replace('CNNNews', 'news')
97         page = page.replace('CNNTechnology', 'technology')
98         page = page.replace('gocostco', 'costco list')
99         page = page.replace('gohardware', 'hardware list')
100         page = page.replace('gohouse', 'house list honey do')
101         page = page.replace('gcal', 'google calendar events')
102         page = page.replace('mynorthwest', 'northwest news')
103         page = page.replace('myq', 'myq garage door status')
104         page = page.replace('gomenu', 'dinner menu')
105         page = page.replace('gmaps-seattle-unwrapped', 'traffic')
106         page = page.replace('gomenu', 'dinner menu')
107         page = page.replace('WSJNews', 'news')
108         page = page.replace('telma', 'telma cabin')
109         page = page.replace('WSJBusiness', 'business news')
110         page = re.sub(r'[0-9]+', '', page)
111         logger.debug(f'normalize_page output: {page}')
112         return page
113
114     logger.info(f'No exact match for f{command}; trying to guess...')
115     best_page = None
116     best_score = None
117     for page in page_chooser.get_page_list():
118         npage = normalize_page(page)
119         score = SequenceMatcher(None, command, npage).ratio()
120         if best_score is None or score > best_score:
121             best_page = page
122             best_score = score
123     assert best_page is not None
124     logger.info(f'Best guess for f{command} => {best_page} (score = {best_score})')
125     return best_page
126
127
128 def process_command(command: str, page_history: List[str], page_chooser) -> str:
129     command = command.lower()
130     logger.info(f'Parsing verbal command: {command}')
131     page = None
132     if 'hold' in command:
133         page = page_history[0]
134     elif 'back' in command:
135         page = page_history[1]
136     elif 'skip' in command:
137         while True:
138             (page, _) = page_chooser.choose_next_page()
139             if page == page_history[0]:
140                 logger.debug(f'chooser: {page} is the same as last time!  Try again.')
141             else:
142                 break
143     elif 'internal' in command:
144         if 'render' in command:
145             page = constants.render_stats_pagename
146         else:
147             page = constants.render_stats_pagename
148     elif 'weather' in command:
149         if 'telma' in command or 'cabin' in command:
150             page = 'weather-telma_3_10800.html'
151         elif 'stevens' in command:
152             page = 'weather-stevens_3_10800.html'
153         else:
154             page = 'weather-home_3_10800.html'
155     elif 'cabin' in command:
156         if 'list' in command:
157             page = 'Cabin-(gocabin)_2_3600.html'
158         else:
159             page = 'hidden/cabin_driveway.html'
160     elif 'news' in command or 'headlines' in command:
161         page = 'cnn-CNNNews_4_25900.html'
162     elif 'clock' in command or 'time' in command:
163         page = 'clock_10_none.html'
164     elif 'countdown' in command or 'countdowns' in command:
165         page = 'countdown_3_7200.html'
166     elif 'costco' in command:
167         page = 'Costco-(gocostco)_2_3600.html'
168     elif 'calendar' in command or 'events' in command:
169         page = 'gcal_3_86400.html'
170     elif 'countdown' in command or 'countdowns' in command:
171         page = 'countdown_3_7200.html'
172     elif 'grocery' in command or 'groceries' in command:
173         page = 'Grocery-(gogrocery)_2_3600.html'
174     elif 'hardware' in command:
175         page = 'Hardware-(gohardware)_2_3600.html'
176     elif 'garage' in command:
177         page = 'myq_4_300.html'
178     elif 'menu' in command:
179         page = 'Menu-(gomenu)_2_3600.html'
180     elif 'cron' in command or 'health' in command:
181         page = 'periodic-health_6_300.html'
182     elif 'photo' in command or 'picture' in command:
183         page = 'photo_23_3600.html'
184     elif 'quote' in command or 'quotation' in command or 'quotes' in command:
185         page = 'quotes_4_10800.html'
186     elif 'stevens' in command:
187         page = 'stevens-conditions_1_86400.html'
188     elif 'stock' in command or 'stocks' in command:
189         page = 'stock_3_86400.html'
190     elif 'twitter' in command:
191         page = 'twitter_10_3600.html'
192     elif 'traffic' in command:
193         page = 'gmaps-seattle-unwrapped_5_none.html'
194     elif 'front' in command and 'door' in command:
195         page = 'hidden/frontdoor.html'
196     elif 'driveway' in command:
197         page = 'hidden/driveway.html'
198     elif 'backyard' in command:
199         page = 'hidden/backyard.html'
200     else:
201         page = guess_page(command, page_chooser)
202     assert page is not None
203     logger.info(f'Parsed to {page}')
204     return page
205
206
207 def thread_change_current(command_queue: Queue) -> None:
208     page_history = ["", ""]
209     swap_page_target = 0.0
210
211     def filter_news_during_dinnertime(page: str) -> bool:
212         now = datetime.now(tz=pytz.timezone("US/Pacific"))
213         is_dinnertime = now.hour >= 18 and now.hour <= 20
214         return not is_dinnertime or not (
215             "cnn" in page
216             or "news" in page
217             or "mynorthwest" in page
218             or "seattle" in page
219             or "wsj" in page
220         )
221
222     page_chooser = chooser.weighted_random_chooser_with_triggers(
223         trigger_catalog.get_triggers(), [filter_news_during_dinnertime]
224     )
225     current_file = os.path.join(constants.pages_dir, "current.shtml")
226     emergency_file = os.path.join(constants.pages_dir, "reload_immediately.html")
227
228     # Main chooser loop
229     while True:
230         now = time.time()
231
232         # Check for a verbal command.
233         command = None
234         try:
235             command = command_queue.get(block=False)
236         except Exception:
237             command = None
238
239         if command is not None:
240             logger.info(f'chooser: We got a verbal command ("{command}"), parsing it...')
241             triggered = True
242             page = process_command(command, page_history, page_chooser)
243         else:
244             while True:
245                 (page, triggered) = page_chooser.choose_next_page()
246                 if triggered:
247                     logger.info('chooser: A trigger is active...')
248                     break
249                 else:
250                     if page == page_history[0]:
251                         logger.debug(f'chooser: {page} is the same as last time! Try again.')
252                     else:
253                         break
254
255         if triggered:
256             if page != page_history[0] or (swap_page_target - now) < 10.0:
257                 logger.info(f'chooser: An emergency page reload to {page} is needed at this time.')
258                 swap_page_target = now + constants.emergency_refresh_period_sec
259
260                 # Set current.shtml to the right page.
261                 try:
262                     with open(current_file, "w") as f:
263                         emit(
264                             f,
265                             page,
266                             override_refresh_sec = constants.emergency_refresh_period_sec,
267                             command = command
268                         )
269                     logger.debug(f'chooser: Wrote {current_file}.')
270                 except Exception as e:
271                     logger.exception(e)
272                     logger.error(f'chooser: Unexpected exception; assuming {page} doesn\'t exist?!')
273                     continue
274
275                 # Also notify XMLHTTP clients that they need to refresh now.
276                 with open(emergency_file, "w") as f:
277                     f.write(f'Reload, suckers... you HAVE to see {page}!')
278                 logger.debug(f'chooser: Wrote {emergency_file}...')
279
280                 # Fix this hack... maybe read the webserver logs and see if it
281                 # actually was picked up?
282                 time.sleep(0.999)
283                 os.remove(emergency_file)
284                 logger.debug(f'chooser: ...and removed {emergency_file}.')
285
286         elif now >= swap_page_target:
287             assert page != page_history[0]
288             logger.info(
289                 f'chooser: Nominal choice of {page} as the next to show.'
290             )
291             swap_page_target = now + constants.refresh_period_sec
292             try:
293                 with open(current_file, "w") as f:
294                     emit(f, page)
295                 logger.debug(f'chooser: Wrote {current_file}.')
296             except Exception as e:
297                 logger.exception(e)
298                 logger.error(f'chooser: Unexpected exception; assuming {page} doesn\'t exist?!')
299                 continue
300         page_history.insert(0, page)
301         page_history = page_history[0:10]
302         time.sleep(0.5)
303
304
305 def emit(f,
306          filename: str,
307          *,
308          override_refresh_sec: int = None,
309          command: str = None):
310     if 'unwrapped' not in filename:
311         logger.debug(f'Emitting {filename} wrapped.')
312         emit_wrapped(f, filename, override_refresh_sec=override_refresh_sec, command=command)
313     else:
314         logger.debug(f'Emitting {filename} raw.')
315         emit_raw(f, filename)
316
317
318 def emit_raw(f, filename: str):
319     f.write(f'<!--#include virtual="{filename}"-->')
320
321
322 def emit_wrapped(f,
323                  filename: str,
324                  *,
325                  override_refresh_sec: int = None,
326                  command: str = None) -> None:
327
328     def pick_background_color() -> str:
329         now = datetime_utils.now_pacific()
330         city = astral.LocationInfo(
331             "Bellevue", "USA", "US/Pacific", 47.610, -122.201
332         )
333         s = sun(city.observer, date=now, tzinfo=pytz.timezone("US/Pacific"))
334         sunrise_mod = utils.minute_number(s["sunrise"].hour, s["sunrise"].minute)
335         sunset_mod = utils.minute_number(s["sunset"].hour, s["sunset"].minute)
336         now_mod = utils.minute_number(now.hour, now.minute)
337         if now_mod < sunrise_mod or now_mod > (sunset_mod + 45):
338             return "E6B8B8"
339         elif now_mod < (sunrise_mod + 45) or now_mod > (sunset_mod + 120):
340             return "EECDCD"
341         else:
342             return "FFFFFF"
343
344     def get_refresh_period() -> float:
345         if override_refresh_sec is not None:
346             return float(override_refresh_sec * 1000.0)
347         now = datetime.now(tz=pytz.timezone("US/Pacific"))
348         if now.hour < 6:
349             return float(constants.refresh_period_night_sec * 1000.0)
350         else:
351             return float(constants.refresh_period_sec * 1000.0)
352
353     age = utils.describe_age_of_file_briefly(f"pages/{filename}")
354     bgcolor = pick_background_color()
355     if command is None:
356         pageid = filename
357     else:
358         pageid = f'"{command}" -> {filename}'
359
360     f.write(
361 """
362 <HEAD>
363   <TITLE>Kitchen Kiosk</TITLE>
364   <LINK rel="stylesheet" type="text/css" href="style.css">
365   <SCRIPT TYPE="text/javascript">
366
367   // Zoom the 'contents' div to fit without scrollbars and then make
368   // it visible.
369   function zoomScreen() {
370     z = 285;
371     do {
372       document.getElementById("content").style.zoom = z+"%";
373       var body = document.body;
374       var html = document.documentElement;
375       var height = Math.max(body.scrollHeight,
376                             body.offsetHeight,
377                             html.clientHeight,
378                             html.scrollHeight,
379                             html.offsetHeight);
380       var windowHeight = window.innerHeight;
381       var width = Math.max(body.scrollWidth,
382                            body.offsetWidth,
383                            html.clientWidth,
384                            html.scrollWidth,
385                            html.offsetWidth);
386       var windowWidth = window.innerWidth;
387       var heightRatio = height / windowHeight;
388       var widthRatio = width / windowWidth;
389
390       if (heightRatio <= 1.0 && widthRatio <= 1.0) {
391         break;
392       }
393       z -= 4;
394     } while(z >= 70);
395     document.getElementById("content").style.visibility = "visible";
396   }
397
398   // Load IMG tags with DATA-SRC attributes late.
399   function lateLoadImages() {
400     var image = document.getElementsByTagName('img');
401     for (var i = 0; i < image.length; i++) {
402       if (image[i].getAttribute('DATA-SRC')) {
403         image[i].setAttribute('SRC', image[i].getAttribute('DATA-SRC'));
404       }
405     }
406   }
407 """
408     )
409     f.write(
410 """
411   // Operate the clock at the top of the page.
412   function runClock() {
413     var today = new Date();
414     var h = today.getHours();
415     var ampm = h >= 12 ? 'pm' : 'am';
416     h = h %% 12;
417     h = h ? h : 12; // the hour '0' should be '12'
418     var m = maybeAddZero(today.getMinutes());
419     var colon = ":";
420     if (today.getSeconds() %% 2 == 0) {
421       colon = "<FONT STYLE='color: #%s; font-size: 4vmin; font-weight: bold'>:</FONT>";
422     }
423     document.getElementById("time").innerHTML = h + colon + m + ampm;
424     document.getElementById("date").innerHTML = today.toDateString();
425     var t = setTimeout(function(){runClock()}, 1000);
426   }
427 """ % bgcolor
428     )
429     f.write(
430 """
431   // Helper method for running the clock.
432   function maybeAddZero(x) {
433     return (x < 10) ? "0" + x : x;
434   }
435
436   // Do something on page load.
437   function addLoadEvent(func) {
438     var oldonload = window.onload;
439     if (typeof window.onload != 'function') {
440       window.onload = func;
441     } else {
442       window.onload = function() {
443         if (oldonload) {
444           oldonload();
445         }
446         func();
447       }
448     }
449   }
450
451   // Sleep thread helper.
452   const sleep = (milliseconds) => {
453     return new Promise(resolve => setTimeout(resolve, milliseconds))
454   }
455
456   var loadedDate = new Date();
457
458   addLoadEvent(zoomScreen);
459   addLoadEvent(runClock);
460   addLoadEvent(lateLoadImages);
461 """
462     )
463     f.write(
464 """
465   // Runs the countdown line at the bottom and is responsible for
466   // normal page reloads caused by the expiration of a timer.
467   (function countdown() {
468     setTimeout(
469       function() {
470         var now = new Date();
471         var deltaMs = now.getTime() - loadedDate.getTime();
472         var totalMs = %d;
473         var remainingMs = (totalMs - deltaMs);
474
475         if (remainingMs > 0) {
476           var hr = document.getElementById("countdown");
477           var width = (remainingMs / (totalMs - 5000)) * 100.0;
478           if (width <= 100) {
479             hr.style.visibility = "visible";
480             hr.style.width = " ".concat(width, "%%");
481             hr.style.backgroundColor = "maroon";
482           }
483         } else {
484           // Reload unconditionally after 22 sec.
485           window.location.reload(true);
486         }
487
488         // Brief sleep before doing it all over again.
489         sleep(50).then(() => {
490           countdown();
491         });
492       }, 50)
493   })();
494 """ % get_refresh_period()
495     )
496     f.write(
497 """
498   function check_reload() {
499     var xhr = new XMLHttpRequest();
500     xhr.open('GET',
501              '%s/reload_immediately.html');
502     xhr.onload =
503       function() {
504         if (xhr.status === 200) {
505           window.location.reload(true);
506         }
507       };
508     xhr.send();
509   }
510
511   // Periodically checks for emergency reload events.
512   setInterval(check_reload, 500);
513   </SCRIPT>
514 </HEAD>
515 """ % constants.root_url)
516     f.write(f'<BODY BGCOLOR="#{bgcolor}">')
517     f.write(
518 """
519     <TABLE style="height:100%; width:100%" BORDER=0>
520     <TR HEIGHT=30>
521         <TD ALIGN="left">
522             <DIV id="date">&nbsp;</DIV>
523         </TD>
524         <TD ALIGN="center"><FONT COLOR=#bbbbbb>
525             <DIV id="info"></DIV></FONT>
526         </TD>
527         <TD ALIGN="right">
528             <DIV id="time">&nbsp;</DIV>
529         </TD>
530     </TR>
531     <TR STYLE="vertical-align:top">
532         <TD COLSPAN=3>
533             <DIV ID="content" STYLE="zoom: 1; visibility: hidden;">
534               <!-- BEGIN main page contents. -->
535 """
536     )
537     f.write(f'<!--#include virtual="{filename}"-->')
538     f.write(
539 """
540             <!-- END main page contents. -->
541             </DIV>
542             <BR>
543             <DIV STYLE="position: absolute; top:1030px; width:99%">
544             <P ALIGN="right">
545 """
546     )
547     f.write(f'<FONT SIZE=2 COLOR=#bbbbbb>{pageid} @ {age} ago.</FONT>')
548     f.write(
549 """
550             </P>
551             <HR id="countdown" STYLE="width:0px;
552                                       text-align:left;
553                                       margin:0;
554                                       border:none;
555                                       border-width:0;
556                                       height:5px;
557                                       visibility:hidden;
558                                       background-color:#ffffff;">
559             </DIV>
560         </TD>
561     </TR>
562     </TABLE>
563 </BODY>"""
564     )
565
566
567 def renderer_update_internal_stats_page(
568         last_render: Dict[str, datetime],
569         render_counts: collections.Counter,
570         render_times: Dict[str, np.array],
571 ) -> None:
572     logger.info(
573         'renderer: Updating internal render statistics page.'
574     )
575     with file_writer.file_writer(constants.render_stats_pagename) as f:
576         f.write(
577 '''
578 <CENTER>
579 <TABLE BORDER=0 WIDTH=95%>
580     <TR>
581     <TH><B>Renderer Name</B></TH>
582     <TH><B>Last Run</B></TH>
583     <TH><B>Num Invocations</B></TH>
584     <TH><B>Render Latency</B></TH>
585     </TR>
586 ''')
587         for n, r in enumerate(renderer_catalog.get_renderers()):
588             if n % 2 == 0:
589                 style = 'style="margin: 0; padding: 0; background: #c6b0b0;"'
590             else:
591                 style = 'style="margin: 0; padding: 0; background: #eeeeee;"'
592             name = r.get_name()
593             last = last_render.get(name, None)
594             if last is None:
595                 last = 'never'
596             else:
597                 last = last.strftime('%Y/%m/%d %I:%M:%S%P')
598             count = render_counts.get(name, 0)
599             latency = render_times.get(name, np.array([]))
600             p25 = p50 = p75 = p90 = p99 = 'N/A'
601             try:
602                 p25 = np.percentile(latency, 25)
603                 p50 = np.percentile(latency, 50)
604                 p75 = np.percentile(latency, 75)
605                 p90 = np.percentile(latency, 90)
606                 p99 = np.percentile(latency, 99)
607                 f.write(
608 f'''
609     <TR>
610     <TD {style}>{name}&nbsp;</TD>
611     <TD {style}>&nbsp;{last}&nbsp;</TD>
612     <TD {style}><CENTER>&nbsp;{count}&nbsp;</CENTER></TD>
613     <TD {style}>&nbsp;p25={p25:5.2f}, p50={p50:5.2f}, p75={p75:5.2f}, p90={p90:5.2f}, p99={p99:5.2f}</TD>
614     </TR>
615 '''
616                 )
617             except IndexError:
618                 pass
619         f.write('</TABLE>')
620
621
622 def thread_invoke_renderers() -> None:
623     render_times: Dict[str, np.array] = {}
624     render_counts: collections.Counter = collections.Counter()
625     last_render: Dict[str, datetime] = {}
626
627     # Touch the internal render page now to signal that we're alive.
628     renderer_update_internal_stats_page(last_render, render_counts, render_times)
629
630     # Main renderer loop
631     while True:
632         logger.info(
633             'renderer: invoking all overdue renderers in catalog...'
634         )
635         for r in renderer_catalog.get_renderers():
636             name = r.get_name()
637             now = time.time()
638             logger.info(f'renderer: Invoking {name}\'s render method.')
639             try:
640                 r.render()
641             except Exception as e:
642                 logger.exception(e)
643                 logger.error(
644                     f'renderer: Unexpected and unhandled exception ({e}) in {name}, swallowing it.'
645                 )
646                 continue
647
648             # Increment the count of render operations per renderer.
649             render_counts[name] += 1
650
651             # Keep track of the last time we invoked each renderer.
652             last_render[name] = datetime_utils.now_pacific()
653
654             # Record how long each render operation takes and warn if very long.
655             delta = time.time() - now
656             times = render_times.get(name, np.array([]))
657             times = np.insert(times, 0, delta)
658             render_times[name] = times
659             if delta > 1.0:
660                 hdr = 'renderer: '
661                 logger.warning(
662 f'''
663 {hdr} Warning: {name}'s rendering took {delta:5.2f}s.
664 {hdr} FYI: {name}'s render times: p25={np.percentile(times, 25):5.2f}, p50={np.percentile(times, 50):5.2f}, p75={np.percentile(times, 75):5.2f}, p90={np.percentile(times, 90):5.2f}, p99={np.percentile(times, 99):5.2f}
665 '''
666                 )
667
668         # Update a page about internal stats of renderers.
669         renderer_update_internal_stats_page(last_render, render_counts, render_times)
670         logger.info('renderer: having a little nap...')
671         time.sleep(constants.render_period_sec)
672
673
674 @bootstrap.initialize
675 def main() -> None:
676     command_queue: Queue = Queue()
677     changer_thread: Optional[Thread] = None
678     renderer_thread: Optional[Thread] = None
679     janitor_thread: Optional[Thread] = None
680     hotword_thread: Optional[Thread] = None
681     while True:
682         if hotword_thread is None or not hotword_thread.is_alive():
683             if hotword_thread is None:
684                 logger.info('watchdog: Starting up the hotword detector thread...')
685             else:
686                 logger.warning(
687                     'watchdog: The hotword detector thread seems to have died; restarting it and hoping for the best.'
688                 )
689             keyword_paths = [pvporcupine.KEYWORD_PATHS[x] for x in ["bumblebee"]]
690             sensitivities = [0.7] * len(keyword_paths)
691             listener = listen.HotwordListener(
692                 command_queue,
693                 keyword_paths,
694                 sensitivities,
695             )
696             hotword_thread = Thread(target=listener.listen_forever, args=())
697             hotword_thread.start()
698
699         if changer_thread is None or not changer_thread.is_alive():
700             if changer_thread is None:
701                 logger.info('watchdog: Starting up the current page changer thread...')
702             else:
703                 logger.warning(
704                     'watchdog: The current page changer thread seems to have died; restarting it and hoping for the best.'
705                 )
706             changer_thread = Thread(target=thread_change_current, args=(command_queue,))
707             changer_thread.start()
708
709         if renderer_thread is None or not renderer_thread.is_alive():
710             if renderer_thread is None:
711                 logger.info('watchdog: Starting up the page renderer thread...')
712             else:
713                 logger.warning(
714                     'watchdog: The page renderer thread seems to have died; restarting it and hoping for the best.'
715                 )
716             renderer_thread = Thread(target=thread_invoke_renderers, args=())
717             renderer_thread.start()
718
719         if janitor_thread is None or not janitor_thread.is_alive():
720             if janitor_thread is None:
721                 logger.info('watchdog: Starting up the memory janitor thread...')
722             else:
723                 logger.warning(
724                     'watchdog: The memory janitor thread seems to have died; restarting it and hoping for the best.'
725                 )
726             janitor_thread = Thread(target=thread_janitor, args=())
727             janitor_thread.start()
728
729         # Have a little break and then check to make sure all threads are still alive.
730         logger.debug('watchdog: having a little nap.')
731         time.sleep(constants.check_threads_period_sec)
732
733
734 if __name__ == "__main__":
735     main()