3 """Utilities related to logging."""
8 from logging.handlers import RotatingFileHandler, SysLogHandler
16 cfg = config.add_commandline_args(
17 f'Logging ({__file__})',
18 'Args related to logging')
20 '--logging_config_file',
21 type=argparse_utils.valid_filename,
24 help='Config file containing the logging setup, see: https://docs.python.org/3/howto/logging.html#logging-advanced-tutorial',
30 choices=['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
32 help='The level below which to squelch log messages.',
37 default='%(levelname)s:%(asctime)s: %(message)s',
38 help='The format for lines logged via the logger module.'
41 '--logging_date_format',
43 default='%Y/%m/%dT%H:%M:%S.%f%z',
45 help='The format of any dates in --logging_format.'
49 action=argparse_utils.ActionNoYes,
51 help='Should we log to the console (stderr)',
58 help='The filename of the logfile to write.'
61 '--logging_filename_maxsize',
65 help='The maximum size (in bytes) to write to the logging_filename.'
68 '--logging_filename_count',
72 help='The number of logging_filename copies to keep before deleting.'
76 action=argparse_utils.ActionNoYes,
78 help='Should we log to localhost\'s syslog.'
81 '--logging_debug_threads',
82 action=argparse_utils.ActionNoYes,
84 help='Should we prepend pid/tid data to all log messages?'
87 '--logging_info_is_print',
88 action=argparse_utils.ActionNoYes,
90 help='logging.info also prints to stdout.'
94 class OnlyInfoFilter(logging.Filter):
95 def filter(self, record):
96 return record.levelno == logging.INFO
99 class MillisecondAwareFormatter(logging.Formatter):
100 converter = datetime.datetime.fromtimestamp
102 def formatTime(self, record, datefmt=None):
103 ct = self.converter(record.created, pytz.timezone("US/Pacific"))
105 s = ct.strftime(datefmt)
107 t = ct.strftime("%Y-%m-%d %H:%M:%S")
108 s = "%s,%03d" % (t, record.msecs)
112 def initialize_logging(logger=None) -> logging.Logger:
113 assert config.has_been_parsed()
115 logger = logging.getLogger() # Root logger
117 if config.config['logging_config_file'] is not None:
118 logging.config.fileConfig('logging.conf')
122 numeric_level = getattr(
124 config.config['logging_level'].upper(),
127 if not isinstance(numeric_level, int):
128 raise ValueError('Invalid level: %s' % config.config['logging_level'])
130 fmt = config.config['logging_format']
131 if config.config['logging_debug_threads']:
132 fmt = f'%(process)d.%(thread)d|{fmt}'
134 if config.config['logging_syslog']:
135 if sys.platform in ('win32', 'cygwin'):
137 "WARNING: Current platform does not support syslog; IGNORING.",
141 handler = SysLogHandler()
142 # for k, v in encoded_priorities.items():
143 # handler.encodePriority(k, v)
144 handler.setFormatter(
145 MillisecondAwareFormatter(
147 datefmt=config.config['logging_date_format'],
150 handler.setLevel(numeric_level)
151 handlers.append(handler)
153 if config.config['logging_filename'] is not None:
154 handler = RotatingFileHandler(
155 config.config['logging_filename'],
156 maxBytes = config.config['logging_filename_maxsize'],
157 backupCount = config.config['logging_filename_count'],
159 handler.setLevel(numeric_level)
160 handler.setFormatter(
161 MillisecondAwareFormatter(
163 datefmt=config.config['logging_date_format'],
166 handlers.append(handler)
168 if config.config['logging_console']:
169 handler = logging.StreamHandler(sys.stderr)
170 handler.setLevel(numeric_level)
171 handler.setFormatter(
172 MillisecondAwareFormatter(
174 datefmt=config.config['logging_date_format'],
177 handlers.append(handler)
179 if len(handlers) == 0:
180 handlers.append(logging.NullHandler())
182 for handler in handlers:
183 logger.addHandler(handler)
184 if config.config['logging_info_is_print']:
185 handler = logging.StreamHandler(sys.stdout)
186 handler.addFilter(OnlyInfoFilter())
187 logger.addHandler(handler)
188 logger.setLevel(numeric_level)
189 logger.propagate = False
193 def get_logger(name: str = ""):
194 logger = logging.getLogger(name)
195 return initialize_logging(logger)
198 def tprint(*args, **kwargs) -> None:
199 if config.config['logging_debug_threads']:
200 from thread_utils import current_thread_id
201 print(f'{current_thread_id()}', end="")
202 print(*args, **kwargs)
207 def dprint(*args, **kwargs) -> None:
208 print(*args, file=sys.stderr, **kwargs)
211 class OutputSink(object):
213 # Bits in the destination_bitv bitvector. Used to indicate the
214 # output destination.
219 LOG_WARNING = 0x10 # > Should provide logger to the c'tor.
221 LOG_CRITICAL = 0x40 # _/
222 FILENAME = 0x80 # Must provide a filename to the c'tor.
225 ALL_LOG_DESTINATIONS = (
226 LOG_DEBUG | LOG_INFO | LOG_WARNING | LOG_ERROR | LOG_CRITICAL
228 ALL_OUTPUT_DESTINATIONS = 0x1FF
231 destination_bitv: int,
236 logger = logging.getLogger(None)
239 if filename is not None:
240 self.f = open(filename, "wb", buffering=0)
242 if self.destination_bitv & OutputSink.FILENAME:
244 "Filename argument is required if bitv & FILENAME"
247 self.set_destination_bitv(destination_bitv)
249 def get_destination_bitv(self):
250 return self.destination_bitv
252 def set_destination_bitv(self, destination_bitv: int):
253 if destination_bitv & self.FILENAME and self.f is None:
255 "Filename argument is required if bitv & FILENAME"
257 self.destination_bitv = destination_bitv
259 def print(self, *args, **kwargs):
260 from string_utils import sprintf, strip_escape_sequences
261 end = kwargs.pop("end", None)
263 if not isinstance(end, str):
264 raise TypeError("end must be None or a string")
265 sep = kwargs.pop("sep", None)
267 if not isinstance(sep, str):
268 raise TypeError("sep must be None or a string")
270 raise TypeError("invalid keyword arguments to print()")
271 buf = sprintf(*args, end="", sep=sep)
276 if self.destination_bitv & self.STDOUT:
277 print(buf, file=sys.stdout, sep=sep, end=end)
278 if self.destination_bitv & self.STDERR:
279 print(buf, file=sys.stderr, sep=sep, end=end)
282 if self.destination_bitv & self.FILENAME and self.f is not None:
283 self.f.write(buf.encode('utf-8'))
285 buf = strip_escape_sequences(buf)
286 if self.logger is not None:
287 if self.destination_bitv & self.LOG_DEBUG:
288 self.logger.debug(buf)
289 if self.destination_bitv & self.LOG_INFO:
290 self.logger.info(buf)
291 if self.destination_bitv & self.LOG_WARNING:
292 self.logger.warning(buf)
293 if self.destination_bitv & self.LOG_ERROR:
294 self.logger.error(buf)
295 if self.destination_bitv & self.LOG_CRITICAL:
296 self.logger.critical(buf)
297 if self.destination_bitv & self.HLOG:
301 if self.f is not None:
305 class OutputContext(OutputSink, contextlib.ContextDecorator):
307 destination_bitv: int,
311 super().__init__(destination_bitv, logger=logger, filename=filename)
316 def __exit__(self, etype, value, traceback):
318 if etype is not None:
323 def hlog(message: str) -> None:
324 message = message.replace("'", "'\"'\"'")
325 os.system(f"/usr/bin/logger -p local7.info -- '{message}'")