# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. import logging import os import sys from logging import StreamHandler, Handler, getLevelName # this class is a copy of logging.FileHandler except we end self.close() # at the end of each emit. While closing file and reopening file after each # write is not efficient, it allows us to see partial logs when writing to # fused Azure blobs, which is very convenient class FileHandler(StreamHandler): """ A handler class which writes formatted logging records to disk files. """ def __init__(self, filename, mode='a', encoding=None, delay=False): """ Open the specified file and use it as the stream for logging. """ # Issue #27493: add support for Path objects to be passed in filename = os.fspath(filename) #keep the absolute path, otherwise derived classes which use this #may come a cropper when the current directory changes self.baseFilename = os.path.abspath(filename) self.mode = mode self.encoding = encoding self.delay = delay if delay: #We don't open the stream, but we still need to call the #Handler constructor to set level, formatter, lock etc. Handler.__init__(self) self.stream = None else: StreamHandler.__init__(self, self._open()) def close(self): """ Closes the stream. """ self.acquire() try: try: if self.stream: try: self.flush() finally: stream = self.stream self.stream = None if hasattr(stream, "close"): stream.close() finally: # Issue #19523: call unconditionally to # prevent a handler leak when delay is set StreamHandler.close(self) finally: self.release() def _open(self): """ Open the current base file with the (original) mode and encoding. Return the resulting stream. """ return open(self.baseFilename, self.mode, encoding=self.encoding) def emit(self, record): """ Emit a record. If the stream was not opened because 'delay' was specified in the constructor, open it before calling the superclass's emit. """ if self.stream is None: self.stream = self._open() StreamHandler.emit(self, record) self.close() def __repr__(self): level = getLevelName(self.level) return '<%s %s (%s)>' % (self.__class__.__name__, self.baseFilename, level) def setup_logger(name, save_dir, distributed_rank, filename="log.txt"): logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) # don't log results for the non-master process if distributed_rank > 0: return logger ch = logging.StreamHandler(stream=sys.stdout) ch.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s: %(message)s") ch.setFormatter(formatter) logger.addHandler(ch) if save_dir: fh = FileHandler(os.path.join(save_dir, filename)) fh.setLevel(logging.DEBUG) fh.setFormatter(formatter) logger.addHandler(fh) return logger