From c44d04a5af784e6f7909456d86e887afc2c92403 Mon Sep 17 00:00:00 2001 From: Renat Kasimov Date: Tue, 27 Jan 2026 13:59:28 +0100 Subject: [PATCH 1/4] Adds log rotation Turned off by default --- agent360-example.ini | 32 +++++++++++++++ agent360/agent360.py | 94 +++++++++++++++++++++++++++++++++----------- 2 files changed, 103 insertions(+), 23 deletions(-) diff --git a/agent360-example.ini b/agent360-example.ini index 9166546..672cfda 100644 --- a/agent360-example.ini +++ b/agent360-example.ini @@ -15,6 +15,38 @@ ###################### Default sections # [agent] ; Main thread # +###################### Log rotation settings +# rotation = none ; Rotation mode: 'size', 'time', or 'none' (default: none) +# ; 'none' = no rotation (recommended for external logrotate) +# ; 'size' = rotate when file reaches max size +# ; 'time' = rotate at specified time intervals +# +# Size-based rotation (when rotation = size): +# rotation_max_bytes = 10485760 ; Max file size before rotation (default: 10MB = 10485760 bytes) +# rotation_backup_count = 7 ; Number of backup files to keep (default: 7) +# +# Time-based rotation (when rotation = time): +# rotation_when = midnight ; When to rotate: 'S'=seconds, 'M'=minutes, 'H'=hours, +# ; 'D'=days, 'midnight', 'W0'-'W6'=weekday (0=Monday) +# rotation_interval = 1 ; Interval for rotation (default: 1) +# rotation_utc = False ; Use UTC for time rotation (default: False) +# +# Examples: +# Daily rotation with 30 day retention: +# rotation = time +# rotation_when = midnight +# rotation_interval = 1 +# rotation_backup_count = 30 +# +# Rotate every 50MB, keep 10 files: +# rotation = size +# rotation_max_bytes = 52428800 +# rotation_backup_count = 10 +# +# External logrotate (recommended for Linux): +# rotation = none +# # Then configure /etc/logrotate.d/agent360 +# # [execution] ; Plugin execution threads # ttl = 15 ; Example: plugins will be killed after 15 sec # diff --git a/agent360/agent360.py b/agent360/agent360.py index e20b626..fa0b3b8 100755 --- a/agent360/agent360.py +++ b/agent360/agent360.py @@ -42,6 +42,9 @@ import time import types from optparse import OptionParser +from logging import Formatter, getLogger, StreamHandler +from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler + try: from urllib.parse import urlparse, urlencode @@ -366,38 +369,83 @@ def _config_section_create(self, section): if not self.config.has_section(section): self.config.add_section(section) - def _logging_init(self): - ''' - Initialize logging faculty - ''' - level = self.config.getint('agent', 'logging_level') - if os.name == 'nt': - log_file = os.path.expandvars(self.config.get('agent', 'log_file')) - else: - log_file = self.config.get('agent', 'log_file') +def _logging_init(self): + """ + Initialize logging with optional in-app log rotation. + Supported rotation modes: + - size (default): RotatingFileHandler + - time: TimedRotatingFileHandler + - none: plain FileHandler + """ + level = self.config.getint('agent', 'logging_level') + + # Resolve target log file path + if os.name == 'nt': + log_file = os.path.expandvars(self.config.get('agent', 'log_file')) + else: + log_file = self.config.get('agent', 'log_file') - log_file_mode = self.config.get('agent', 'log_file_mode') - if log_file_mode in ('w', 'a'): - pass - elif log_file_mode == 'truncate': + # Normalize legacy 'log_file_mode' (kept for compatibility) + log_file_mode = self.config.get('agent', 'log_file_mode') + if log_file_mode not in ('w', 'a'): + if log_file_mode == 'truncate': log_file_mode = 'w' - elif log_file_mode == 'append': - log_file_mode = 'a' else: log_file_mode = 'a' + # Rotation config (all optional) + rotation = self.config.get('agent', 'rotation') if self.config.has_option('agent', 'rotation') else 'none' + # Size-based defaults (10MB, keep 7 files) + rotation_max_bytes = self.config.getint('agent', 'rotation_max_bytes') if self.config.has_option('agent', 'rotation_max_bytes') else 10 * 1024 * 1024 + rotation_backup_count = self.config.getint('agent', 'rotation_backup_count') if self.config.has_option('agent', 'rotation_backup_count') else 7 + # Time-based defaults (roll at midnight, keep 7 files) + rotation_when = self.config.get('agent', 'rotation_when') if self.config.has_option('agent', 'rotation_when') else 'midnight' + rotation_interval = self.config.getint('agent', 'rotation_interval') if self.config.has_option('agent', 'rotation_interval') else 1 + rotation_utc = self.config.getboolean('agent', 'rotation_utc') if self.config.has_option('agent', 'rotation_utc') else False + + # Configure root logger explicitly (avoid basicConfig) + root = getLogger() + root.setLevel(level) + fmt = Formatter("%(asctime)-15s %(levelname)s %(message)s") + + try: if log_file == '-': - logging.basicConfig(level=level) # Log to sys.stderr by default + # Container/journald-friendly: write to stderr + handler = StreamHandler() else: - try: - logging.basicConfig(filename=log_file, filemode=log_file_mode, level=level, format="%(asctime)-15s %(levelname)s %(message)s") - except IOError as e: - logging.basicConfig(level=level) - logging.info('IOError: %s', e) - logging.info('Drop logging to stderr') + if rotation == 'none': + # No rotation at all + handler = logging.FileHandler(log_file, mode=log_file_mode) + elif rotation == 'time': + # Time-based rotation + handler = TimedRotatingFileHandler( + log_file, + when=rotation_when, # e.g., 'S', 'M', 'H', 'D', 'midnight', 'W0' + interval=rotation_interval, + backupCount=rotation_backup_count, + utc=rotation_utc + ) + else: + # Default: size-based rotation + handler = RotatingFileHandler( + log_file, + mode=log_file_mode, + maxBytes=rotation_max_bytes, + backupCount=rotation_backup_count + ) + + handler.setFormatter(fmt) + root.handlers = [handler] + except IOError as e: + # Fall back to stderr if file cannot be opened + root.handlers = [StreamHandler()] + root.handlers[0].setFormatter(fmt) + logging.info('IOError: %s', e) + logging.info('Drop logging to stderr') + + logging.info('Agent logging_level %i', level) - logging.info('Agent logging_level %i', level) def _plugins_init(self): ''' From de7a169a9eaa826abc389c422abc80a1fcf0e09e Mon Sep 17 00:00:00 2001 From: Renat Kasimov Date: Tue, 27 Jan 2026 14:07:23 +0100 Subject: [PATCH 2/4] Revert "Adds log rotation" This reverts commit c44d04a5af784e6f7909456d86e887afc2c92403. --- agent360-example.ini | 32 --------------- agent360/agent360.py | 94 +++++++++++--------------------------------- 2 files changed, 23 insertions(+), 103 deletions(-) diff --git a/agent360-example.ini b/agent360-example.ini index 672cfda..9166546 100644 --- a/agent360-example.ini +++ b/agent360-example.ini @@ -15,38 +15,6 @@ ###################### Default sections # [agent] ; Main thread # -###################### Log rotation settings -# rotation = none ; Rotation mode: 'size', 'time', or 'none' (default: none) -# ; 'none' = no rotation (recommended for external logrotate) -# ; 'size' = rotate when file reaches max size -# ; 'time' = rotate at specified time intervals -# -# Size-based rotation (when rotation = size): -# rotation_max_bytes = 10485760 ; Max file size before rotation (default: 10MB = 10485760 bytes) -# rotation_backup_count = 7 ; Number of backup files to keep (default: 7) -# -# Time-based rotation (when rotation = time): -# rotation_when = midnight ; When to rotate: 'S'=seconds, 'M'=minutes, 'H'=hours, -# ; 'D'=days, 'midnight', 'W0'-'W6'=weekday (0=Monday) -# rotation_interval = 1 ; Interval for rotation (default: 1) -# rotation_utc = False ; Use UTC for time rotation (default: False) -# -# Examples: -# Daily rotation with 30 day retention: -# rotation = time -# rotation_when = midnight -# rotation_interval = 1 -# rotation_backup_count = 30 -# -# Rotate every 50MB, keep 10 files: -# rotation = size -# rotation_max_bytes = 52428800 -# rotation_backup_count = 10 -# -# External logrotate (recommended for Linux): -# rotation = none -# # Then configure /etc/logrotate.d/agent360 -# # [execution] ; Plugin execution threads # ttl = 15 ; Example: plugins will be killed after 15 sec # diff --git a/agent360/agent360.py b/agent360/agent360.py index fa0b3b8..e20b626 100755 --- a/agent360/agent360.py +++ b/agent360/agent360.py @@ -42,9 +42,6 @@ import time import types from optparse import OptionParser -from logging import Formatter, getLogger, StreamHandler -from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler - try: from urllib.parse import urlparse, urlencode @@ -369,83 +366,38 @@ def _config_section_create(self, section): if not self.config.has_section(section): self.config.add_section(section) + def _logging_init(self): + ''' + Initialize logging faculty + ''' + level = self.config.getint('agent', 'logging_level') -def _logging_init(self): - """ - Initialize logging with optional in-app log rotation. - Supported rotation modes: - - size (default): RotatingFileHandler - - time: TimedRotatingFileHandler - - none: plain FileHandler - """ - level = self.config.getint('agent', 'logging_level') - - # Resolve target log file path - if os.name == 'nt': - log_file = os.path.expandvars(self.config.get('agent', 'log_file')) - else: - log_file = self.config.get('agent', 'log_file') + if os.name == 'nt': + log_file = os.path.expandvars(self.config.get('agent', 'log_file')) + else: + log_file = self.config.get('agent', 'log_file') - # Normalize legacy 'log_file_mode' (kept for compatibility) - log_file_mode = self.config.get('agent', 'log_file_mode') - if log_file_mode not in ('w', 'a'): - if log_file_mode == 'truncate': + log_file_mode = self.config.get('agent', 'log_file_mode') + if log_file_mode in ('w', 'a'): + pass + elif log_file_mode == 'truncate': log_file_mode = 'w' + elif log_file_mode == 'append': + log_file_mode = 'a' else: log_file_mode = 'a' - # Rotation config (all optional) - rotation = self.config.get('agent', 'rotation') if self.config.has_option('agent', 'rotation') else 'none' - # Size-based defaults (10MB, keep 7 files) - rotation_max_bytes = self.config.getint('agent', 'rotation_max_bytes') if self.config.has_option('agent', 'rotation_max_bytes') else 10 * 1024 * 1024 - rotation_backup_count = self.config.getint('agent', 'rotation_backup_count') if self.config.has_option('agent', 'rotation_backup_count') else 7 - # Time-based defaults (roll at midnight, keep 7 files) - rotation_when = self.config.get('agent', 'rotation_when') if self.config.has_option('agent', 'rotation_when') else 'midnight' - rotation_interval = self.config.getint('agent', 'rotation_interval') if self.config.has_option('agent', 'rotation_interval') else 1 - rotation_utc = self.config.getboolean('agent', 'rotation_utc') if self.config.has_option('agent', 'rotation_utc') else False - - # Configure root logger explicitly (avoid basicConfig) - root = getLogger() - root.setLevel(level) - fmt = Formatter("%(asctime)-15s %(levelname)s %(message)s") - - try: if log_file == '-': - # Container/journald-friendly: write to stderr - handler = StreamHandler() + logging.basicConfig(level=level) # Log to sys.stderr by default else: - if rotation == 'none': - # No rotation at all - handler = logging.FileHandler(log_file, mode=log_file_mode) - elif rotation == 'time': - # Time-based rotation - handler = TimedRotatingFileHandler( - log_file, - when=rotation_when, # e.g., 'S', 'M', 'H', 'D', 'midnight', 'W0' - interval=rotation_interval, - backupCount=rotation_backup_count, - utc=rotation_utc - ) - else: - # Default: size-based rotation - handler = RotatingFileHandler( - log_file, - mode=log_file_mode, - maxBytes=rotation_max_bytes, - backupCount=rotation_backup_count - ) - - handler.setFormatter(fmt) - root.handlers = [handler] - except IOError as e: - # Fall back to stderr if file cannot be opened - root.handlers = [StreamHandler()] - root.handlers[0].setFormatter(fmt) - logging.info('IOError: %s', e) - logging.info('Drop logging to stderr') - - logging.info('Agent logging_level %i', level) + try: + logging.basicConfig(filename=log_file, filemode=log_file_mode, level=level, format="%(asctime)-15s %(levelname)s %(message)s") + except IOError as e: + logging.basicConfig(level=level) + logging.info('IOError: %s', e) + logging.info('Drop logging to stderr') + logging.info('Agent logging_level %i', level) def _plugins_init(self): ''' From 66b1eefe89d19e41a21d28607acdd6404b608e59 Mon Sep 17 00:00:00 2001 From: Renat Kasimov Date: Tue, 3 Feb 2026 10:40:25 +0100 Subject: [PATCH 3/4] [CPCLOUD-1798] Added log rotation --- agent360-example.ini | 36 +++++++++++++++ agent360/agent360.py | 105 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 121 insertions(+), 20 deletions(-) diff --git a/agent360-example.ini b/agent360-example.ini index 9166546..40c0e4a 100644 --- a/agent360-example.ini +++ b/agent360-example.ini @@ -15,6 +15,42 @@ ###################### Default sections # [agent] ; Main thread # +###################### Log rotation settings +# rotation = none ; Rotation mode: 'size', 'time', or 'none' (default: none) +# ; 'none' = no rotation (recommended for external logrotate) +# ; 'size' = rotate when file reaches max size +# ; 'time' = rotate at specified time intervals +# rotation_compress = False ; Compress rotated logs with gzip (default: False) +# ; Creates .gz files (e.g., agent360.log.1.gz) +# +# Size-based rotation (when rotation = size): +# rotation_max_bytes = 10485760 ; Max file size before rotation (default: 10MB = 10485760 bytes) +# rotation_backup_count = 7 ; Number of backup files to keep (default: 7) +# +# Time-based rotation (when rotation = time): +# rotation_when = midnight ; When to rotate: 'S'=seconds, 'M'=minutes, 'H'=hours, +# ; 'D'=days, 'midnight', 'W0'-'W6'=weekday (0=Monday) +# rotation_interval = 1 ; Interval for rotation (default: 1) +# rotation_utc = False ; Use UTC for time rotation (default: False) +# +# Examples: +# Daily rotation with 30 day retention and compression: +# rotation = time +# rotation_when = midnight +# rotation_interval = 1 +# rotation_backup_count = 30 +# rotation_compress = True +# +# Rotate every 50MB, keep 10 files, compressed: +# rotation = size +# rotation_max_bytes = 52428800 +# rotation_backup_count = 10 +# rotation_compress = True +# +# External logrotate (recommended for Linux): +# rotation = none +# # Then configure /etc/logrotate.d/agent360 +# # [execution] ; Plugin execution threads # ttl = 15 ; Example: plugins will be killed after 15 sec # diff --git a/agent360/agent360.py b/agent360/agent360.py index e20b626..af17212 100755 --- a/agent360/agent360.py +++ b/agent360/agent360.py @@ -42,6 +42,8 @@ import time import types from optparse import OptionParser +from logging import Formatter, getLogger, StreamHandler +from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler try: from urllib.parse import urlparse, urlencode @@ -367,35 +369,98 @@ def _config_section_create(self, section): self.config.add_section(section) def _logging_init(self): - ''' - Initialize logging faculty - ''' + """ + Initialize logging with optional in-app log rotation. + Supported rotation modes: + - none (default): plain FileHandler (use with external logrotate) + - size: RotatingFileHandler + - time: TimedRotatingFileHandler + """ level = self.config.getint('agent', 'logging_level') + # Resolve target log file path if os.name == 'nt': log_file = os.path.expandvars(self.config.get('agent', 'log_file')) else: log_file = self.config.get('agent', 'log_file') + # Normalize legacy 'log_file_mode' (kept for compatibility) log_file_mode = self.config.get('agent', 'log_file_mode') - if log_file_mode in ('w', 'a'): - pass - elif log_file_mode == 'truncate': - log_file_mode = 'w' - elif log_file_mode == 'append': - log_file_mode = 'a' - else: - log_file_mode = 'a' + if log_file_mode not in ('w', 'a'): + if log_file_mode == 'truncate': + log_file_mode = 'w' + else: + log_file_mode = 'a' + + # Rotation config (all optional) + rotation = self.config.get('agent', 'rotation') if self.config.has_option('agent', 'rotation') else 'none' + rotation_compress = self.config.getboolean('agent', 'rotation_compress') if self.config.has_option('agent', 'rotation_compress') else False + # Size-based defaults (10MB, keep 7 files) + rotation_max_bytes = self.config.getint('agent', 'rotation_max_bytes') if self.config.has_option('agent', 'rotation_max_bytes') else 10 * 1024 * 1024 + rotation_backup_count = self.config.getint('agent', 'rotation_backup_count') if self.config.has_option('agent', 'rotation_backup_count') else 7 + # Time-based defaults (roll at midnight, keep 7 files) + rotation_when = self.config.get('agent', 'rotation_when') if self.config.has_option('agent', 'rotation_when') else 'midnight' + rotation_interval = self.config.getint('agent', 'rotation_interval') if self.config.has_option('agent', 'rotation_interval') else 1 + rotation_utc = self.config.getboolean('agent', 'rotation_utc') if self.config.has_option('agent', 'rotation_utc') else False + + # Configure root logger explicitly (avoid basicConfig) + root = getLogger() + root.setLevel(level) + fmt = Formatter("%(asctime)-15s %(levelname)s %(message)s") - if log_file == '-': - logging.basicConfig(level=level) # Log to sys.stderr by default - else: - try: - logging.basicConfig(filename=log_file, filemode=log_file_mode, level=level, format="%(asctime)-15s %(levelname)s %(message)s") - except IOError as e: - logging.basicConfig(level=level) - logging.info('IOError: %s', e) - logging.info('Drop logging to stderr') + try: + if log_file == '-': + # Container/journald-friendly: write to stderr + handler = StreamHandler() + else: + if rotation == 'none': + # No rotation at all + handler = logging.FileHandler(log_file, mode=log_file_mode) + elif rotation == 'time': + # Time-based rotation + handler = TimedRotatingFileHandler( + log_file, + when=rotation_when, # e.g., 'S', 'M', 'H', 'D', 'midnight', 'W0' + interval=rotation_interval, + backupCount=rotation_backup_count, + utc=rotation_utc + ) + else: + # Size-based rotation + handler = RotatingFileHandler( + log_file, + mode=log_file_mode, + maxBytes=rotation_max_bytes, + backupCount=rotation_backup_count + ) + + # Add gzip compression if enabled + if rotation != 'none' and rotation_compress: + import gzip + import shutil + + def namer(name): + """Add .gz suffix to rotated files""" + return name + ".gz" + + def rotator(source, dest): + """Compress rotated log files with gzip""" + with open(source, 'rb') as f_in: + with gzip.open(dest, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + os.remove(source) + + handler.namer = namer + handler.rotator = rotator + + handler.setFormatter(fmt) + root.handlers = [handler] + except IOError as e: + # Fall back to stderr if file cannot be opened + root.handlers = [StreamHandler()] + root.handlers[0].setFormatter(fmt) + logging.info('IOError: %s', e) + logging.info('Drop logging to stderr') logging.info('Agent logging_level %i', level) From d72d60f87e518e24e96fe69324dff1744202fe56 Mon Sep 17 00:00:00 2001 From: Renat Kasimov Date: Fri, 9 Oct 2026 22:25:40 +0200 Subject: [PATCH 4/4] [CPCLOUD-1798] Add logs rotation --- agent360-example.ini | 2 +- agent360/agent360.py | 164 ++++++++++++++++++++++++++++--------------- 2 files changed, 107 insertions(+), 59 deletions(-) diff --git a/agent360-example.ini b/agent360-example.ini index 40c0e4a..cbdcfce 100644 --- a/agent360-example.ini +++ b/agent360-example.ini @@ -28,7 +28,7 @@ # rotation_backup_count = 7 ; Number of backup files to keep (default: 7) # # Time-based rotation (when rotation = time): -# rotation_when = midnight ; When to rotate: 'S'=seconds, 'M'=minutes, 'H'=hours, +# rotation_when = midnight ; When to rotate: 'S'=seconds, 'M'=minutes, 'H'=hours, # ; 'D'=days, 'midnight', 'W0'-'W6'=weekday (0=Monday) # rotation_interval = 1 ; Interval for rotation (default: 1) # rotation_utc = False ; Use UTC for time rotation (default: False) diff --git a/agent360/agent360.py b/agent360/agent360.py index af17212..d5a655e 100755 --- a/agent360/agent360.py +++ b/agent360/agent360.py @@ -25,6 +25,9 @@ import imp import glob +import gzip +import re +import shutil import certifi import ssl @@ -42,7 +45,7 @@ import time import types from optparse import OptionParser -from logging import Formatter, getLogger, StreamHandler +from logging import FileHandler, Formatter, getLogger, StreamHandler from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler try: @@ -340,6 +343,13 @@ def _config_init(self): 'api_path': '/v2/server/poll', 'log_file': '/var/log/agent360.log', 'log_file_mode': 'a', + 'rotation': 'none', + 'rotation_compress': 'no', + 'rotation_max_bytes': 10 * 1024 * 1024, + 'rotation_backup_count': 7, + 'rotation_when': 'midnight', + 'rotation_interval': 1, + 'rotation_utc': 'no', 'max_cached_collections': 10, } sections = [ @@ -392,78 +402,116 @@ def _logging_init(self): else: log_file_mode = 'a' - # Rotation config (all optional) - rotation = self.config.get('agent', 'rotation') if self.config.has_option('agent', 'rotation') else 'none' - rotation_compress = self.config.getboolean('agent', 'rotation_compress') if self.config.has_option('agent', 'rotation_compress') else False - # Size-based defaults (10MB, keep 7 files) - rotation_max_bytes = self.config.getint('agent', 'rotation_max_bytes') if self.config.has_option('agent', 'rotation_max_bytes') else 10 * 1024 * 1024 - rotation_backup_count = self.config.getint('agent', 'rotation_backup_count') if self.config.has_option('agent', 'rotation_backup_count') else 7 - # Time-based defaults (roll at midnight, keep 7 files) - rotation_when = self.config.get('agent', 'rotation_when') if self.config.has_option('agent', 'rotation_when') else 'midnight' - rotation_interval = self.config.getint('agent', 'rotation_interval') if self.config.has_option('agent', 'rotation_interval') else 1 - rotation_utc = self.config.getboolean('agent', 'rotation_utc') if self.config.has_option('agent', 'rotation_utc') else False - # Configure root logger explicitly (avoid basicConfig) root = getLogger() root.setLevel(level) - fmt = Formatter("%(asctime)-15s %(levelname)s %(message)s") + fmt = Formatter("%(asctime)-15s %(levelname)s %(message)s") + warnings = [] try: + # Rotation config (defaults live in _config_init) + rotation = self.config.get('agent', 'rotation').strip().lower() + if rotation not in ('none', 'size', 'time'): + warnings.append("Unknown rotation mode '%s', rotation disabled" % rotation) + rotation = 'none' + rotation_compress = self.config.getboolean('agent', 'rotation_compress') + rotation_max_bytes = self.config.getint('agent', 'rotation_max_bytes') + rotation_backup_count = self.config.getint('agent', 'rotation_backup_count') + rotation_when = self.config.get('agent', 'rotation_when') + rotation_interval = self.config.getint('agent', 'rotation_interval') + rotation_utc = self.config.getboolean('agent', 'rotation_utc') + if log_file == '-': # Container/journald-friendly: write to stderr handler = StreamHandler() + elif rotation == 'none': + handler = FileHandler(log_file, mode=log_file_mode) + elif rotation == 'time': + if log_file_mode == 'w': + warnings.append("log_file_mode 'w' is ignored with time-based rotation") + handler = TimedRotatingFileHandler( + log_file, + when=rotation_when, # e.g., 'S', 'M', 'H', 'D', 'midnight', 'W0' + interval=rotation_interval, + backupCount=rotation_backup_count, + utc=rotation_utc + ) else: - if rotation == 'none': - # No rotation at all - handler = logging.FileHandler(log_file, mode=log_file_mode) - elif rotation == 'time': - # Time-based rotation - handler = TimedRotatingFileHandler( - log_file, - when=rotation_when, # e.g., 'S', 'M', 'H', 'D', 'midnight', 'W0' - interval=rotation_interval, - backupCount=rotation_backup_count, - utc=rotation_utc - ) + handler = RotatingFileHandler( + log_file, + mode=log_file_mode, + maxBytes=rotation_max_bytes, + backupCount=rotation_backup_count + ) + + if log_file != '-' and rotation != 'none' and rotation_compress: + if sys.version_info < (3, 3): + warnings.append('rotation_compress requires Python 3.3+, ignored') else: - # Size-based rotation - handler = RotatingFileHandler( - log_file, - mode=log_file_mode, - maxBytes=rotation_max_bytes, - backupCount=rotation_backup_count - ) - - # Add gzip compression if enabled - if rotation != 'none' and rotation_compress: - import gzip - import shutil - - def namer(name): - """Add .gz suffix to rotated files""" - return name + ".gz" - - def rotator(source, dest): - """Compress rotated log files with gzip""" - with open(source, 'rb') as f_in: - with gzip.open(dest, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - os.remove(source) - - handler.namer = namer - handler.rotator = rotator + self._logging_enable_gzip(handler) handler.setFormatter(fmt) - root.handlers = [handler] - except IOError as e: - # Fall back to stderr if file cannot be opened - root.handlers = [StreamHandler()] - root.handlers[0].setFormatter(fmt) - logging.info('IOError: %s', e) - logging.info('Drop logging to stderr') + for old in root.handlers[:]: + root.removeHandler(old) + old.close() + root.addHandler(handler) + except (IOError, OSError, ValueError) as e: + # Fall back to stderr if the file cannot be opened or the rotation config is invalid + handler = StreamHandler() + handler.setFormatter(fmt) + for old in root.handlers[:]: + root.removeHandler(old) + root.addHandler(handler) + logging.warning('%s: %s', type(e).__name__, e) + logging.warning('Drop logging to stderr') + + for warning in warnings: + logging.warning(warning) logging.info('Agent logging_level %i', level) + @staticmethod + def _logging_enable_gzip(handler): + """ + Gzip rotated log files. The rotator runs under the handler lock, so + keep rotated files reasonably small. + """ + def namer(name): + return name + '.gz' + + def rotator(source, dest): + # Write to a temp name first so a failure never leaves a partial .gz + tmp = dest + '.tmp' + try: + with open(source, 'rb') as f_in: + with gzip.open(tmp, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + os.replace(tmp, dest) + except Exception: + if os.path.exists(tmp): + os.remove(tmp) + raise + os.remove(source) + + handler.namer = namer + handler.rotator = rotator + if isinstance(handler, TimedRotatingFileHandler): + # The stdlib backupCount cleanup does not reliably recognise names + # produced by namer across Python versions, so do it ourselves. + stamp = re.compile(r'\d{4}-\d{2}-\d{2}(_\d{2}(-\d{2}(-\d{2})?)?)?\.gz') + + def get_files_to_delete(): + dir_name, base_name = os.path.split(handler.baseFilename) + prefix = base_name + '.' + found = sorted( + os.path.join(dir_name, name) for name in os.listdir(dir_name) + if name.startswith(prefix) and stamp.fullmatch(name[len(prefix):]) + ) + return found[:max(0, len(found) - handler.backupCount)] + + handler.getFilesToDelete = get_files_to_delete + + def _plugins_init(self): ''' Discover the plugins