3 """Utilities related to logging."""
9 from logging.handlers import RotatingFileHandler, SysLogHandler
14 # This module is commonly used by others in here and should avoid
15 # taking any unnecessary dependencies back on them.
19 cfg = config.add_commandline_args(
20 f'Logging ({__file__})',
21 'Args related to logging')
23 '--logging_config_file',
24 type=argparse_utils.valid_filename,
27 help='Config file containing the logging setup, see: https://docs.python.org/3/howto/logging.html#logging-advanced-tutorial',
33 choices=['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
35 help='The level below which to squelch log messages.',
40 default='%(levelname).1s:%(asctime)s: %(message)s',
41 help='The format for lines logged via the logger module.'
44 '--logging_date_format',
46 default='%Y/%m/%dT%H:%M:%S.%f%z',
48 help='The format of any dates in --logging_format.'
52 action=argparse_utils.ActionNoYes,
54 help='Should we log to the console (stderr)',
61 help='The filename of the logfile to write.'
64 '--logging_filename_maxsize',
68 help='The maximum size (in bytes) to write to the logging_filename.'
71 '--logging_filename_count',
75 help='The number of logging_filename copies to keep before deleting.'
79 action=argparse_utils.ActionNoYes,
81 help='Should we log to localhost\'s syslog.'
84 '--logging_debug_threads',
85 action=argparse_utils.ActionNoYes,
87 help='Should we prepend pid/tid data to all log messages?'
90 '--logging_info_is_print',
91 action=argparse_utils.ActionNoYes,
93 help='logging.info also prints to stdout.'
96 # See also: OutputMultiplexer/OutputContext
98 '--logging_captures_prints',
99 action=argparse_utils.ActionNoYes,
101 help='When calling print also log.info too'
104 built_in_print = print
107 class OnlyInfoFilter(logging.Filter):
108 def filter(self, record):
109 return record.levelno == logging.INFO
112 class MillisecondAwareFormatter(logging.Formatter):
113 converter = datetime.datetime.fromtimestamp
115 def formatTime(self, record, datefmt=None):
116 ct = MillisecondAwareFormatter.converter(
117 record.created, pytz.timezone("US/Pacific")
120 s = ct.strftime(datefmt)
122 t = ct.strftime("%Y-%m-%d %H:%M:%S")
123 s = "%s,%03d" % (t, record.msecs)
127 def initialize_logging(logger=None) -> logging.Logger:
128 assert config.has_been_parsed()
130 logger = logging.getLogger() # Root logger
132 if config.config['logging_config_file'] is not None:
133 logging.config.fileConfig('logging.conf')
137 numeric_level = getattr(
139 config.config['logging_level'].upper(),
142 if not isinstance(numeric_level, int):
143 raise ValueError('Invalid level: %s' % config.config['logging_level'])
145 fmt = config.config['logging_format']
146 if config.config['logging_debug_threads']:
147 fmt = f'%(process)d.%(thread)d|{fmt}'
149 if config.config['logging_syslog']:
150 if sys.platform not in ('win32', 'cygwin'):
151 handler = SysLogHandler()
152 # for k, v in encoded_priorities.items():
153 # handler.encodePriority(k, v)
154 handler.setFormatter(
155 MillisecondAwareFormatter(
157 datefmt=config.config['logging_date_format'],
160 handler.setLevel(numeric_level)
161 handlers.append(handler)
163 if config.config['logging_filename']:
164 handler = RotatingFileHandler(
165 config.config['logging_filename'],
166 maxBytes = config.config['logging_filename_maxsize'],
167 backupCount = config.config['logging_filename_count'],
169 handler.setLevel(numeric_level)
170 handler.setFormatter(
171 MillisecondAwareFormatter(
173 datefmt=config.config['logging_date_format'],
176 handlers.append(handler)
178 if config.config['logging_console']:
179 handler = logging.StreamHandler(sys.stderr)
180 handler.setLevel(numeric_level)
181 handler.setFormatter(
182 MillisecondAwareFormatter(
184 datefmt=config.config['logging_date_format'],
187 handlers.append(handler)
189 if len(handlers) == 0:
190 handlers.append(logging.NullHandler())
192 for handler in handlers:
193 logger.addHandler(handler)
195 if config.config['logging_info_is_print']:
196 handler = logging.StreamHandler(sys.stdout)
197 handler.addFilter(OnlyInfoFilter())
198 logger.addHandler(handler)
200 logger.setLevel(numeric_level)
201 logger.propagate = False
203 if config.config['logging_captures_prints']:
205 global built_in_print
207 def print_and_also_log(*arg, **kwarg):
208 f = kwarg.get('file', None)
213 built_in_print(*arg, **kwarg)
214 builtins.print = print_and_also_log
219 def get_logger(name: str = ""):
220 logger = logging.getLogger(name)
221 return initialize_logging(logger)
224 def tprint(*args, **kwargs) -> None:
225 if config.config['logging_debug_threads']:
226 from thread_utils import current_thread_id
227 print(f'{current_thread_id()}', end="")
228 print(*args, **kwargs)
233 def dprint(*args, **kwargs) -> None:
234 print(*args, file=sys.stderr, **kwargs)
237 class OutputMultiplexer(object):
239 class Destination(enum.IntEnum):
240 """Bits in the destination_bitv bitvector. Used to indicate the
241 output destination."""
246 LOG_WARNING = 0x10 # > Should provide logger to the c'tor.
248 LOG_CRITICAL = 0x40 # _/
249 FILENAME = 0x80 # Must provide a filename to the c'tor.
250 FILEHANDLE = 0x100 # Must provide a handle to the c'tor.
252 ALL_LOG_DESTINATIONS = (
253 LOG_DEBUG | LOG_INFO | LOG_WARNING | LOG_ERROR | LOG_CRITICAL
255 ALL_OUTPUT_DESTINATIONS = 0x2FF
258 destination_bitv: int,
264 logger = logging.getLogger(None)
267 if filename is not None:
268 self.f = open(filename, "wb", buffering=0)
270 if self.destination_bitv & OutputMultiplexer.FILENAME:
272 "Filename argument is required if bitv & FILENAME"
276 if handle is not None:
279 if self.destination_bitv & OutputMultiplexer.FILEHANDLE:
281 "Handle argument is required if bitv & FILEHANDLE"
285 self.set_destination_bitv(destination_bitv)
287 def get_destination_bitv(self):
288 return self.destination_bitv
290 def set_destination_bitv(self, destination_bitv: int):
291 if destination_bitv & self.Destination.FILENAME and self.f is None:
293 "Filename argument is required if bitv & FILENAME"
295 if destination_bitv & self.Destination.FILEHANDLE and self.h is None:
297 "Handle argument is required if bitv & FILEHANDLE"
299 self.destination_bitv = destination_bitv
301 def print(self, *args, **kwargs):
302 from string_utils import sprintf, strip_escape_sequences
303 end = kwargs.pop("end", None)
305 if not isinstance(end, str):
306 raise TypeError("end must be None or a string")
307 sep = kwargs.pop("sep", None)
309 if not isinstance(sep, str):
310 raise TypeError("sep must be None or a string")
312 raise TypeError("invalid keyword arguments to print()")
313 buf = sprintf(*args, end="", sep=sep)
318 if self.destination_bitv & self.Destination.STDOUT:
319 print(buf, file=sys.stdout, sep=sep, end=end)
320 if self.destination_bitv & self.Destination.STDERR:
321 print(buf, file=sys.stderr, sep=sep, end=end)
325 self.destination_bitv & self.Destination.FILENAME and
328 self.f.write(buf.encode('utf-8'))
332 self.destination_bitv & self.Destination.FILEHANDLE and
338 buf = strip_escape_sequences(buf)
339 if self.logger is not None:
340 if self.destination_bitv & self.Destination.LOG_DEBUG:
341 self.logger.debug(buf)
342 if self.destination_bitv & self.Destination.LOG_INFO:
343 self.logger.info(buf)
344 if self.destination_bitv & self.Destination.LOG_WARNING:
345 self.logger.warning(buf)
346 if self.destination_bitv & self.Destination.LOG_ERROR:
347 self.logger.error(buf)
348 if self.destination_bitv & self.Destination.LOG_CRITICAL:
349 self.logger.critical(buf)
350 if self.destination_bitv & self.Destination.HLOG:
354 if self.f is not None:
358 class OutputContext(OutputMultiplexer, contextlib.ContextDecorator):
360 destination_bitv: OutputMultiplexer.Destination,
374 def __exit__(self, etype, value, traceback) -> bool:
376 if etype is not None:
381 def hlog(message: str) -> None:
382 message = message.replace("'", "'\"'\"'")
383 os.system(f"/usr/bin/logger -p local7.info -- '{message}'")