v0.22.1: Log update with closing queue on exit or abort

to avoid hung threads on errors

ToDo.md

@@ -3,3 +3,5 @@
 - [x] stub files .pyi
 - [ ] Add tests for all, we need 100% test coverate
 - [x] Log: add custom format for "stack_correct" if set, this will override the normal stack block
+- [ ] Log: add rotate for size based
+- [ ] All folders and file names need to be revisited for naming and content collection

pyproject.toml

@@ -1,7 +1,7 @@
 # MARK: Project info
 [project]
 name = "corelibs"
-version = "0.21.1"
+version = "0.22.1"
 description = "Collection of utils for Python scripts"
 readme = "README.md"
 requires-python = ">=3.13"

src/corelibs/logging_handling/log.py

@@ -7,8 +7,10 @@ attach "init_worker_logging" with the set log_queue
 import re
 import logging.handlers
 import logging
+from datetime import datetime
 import time
 from pathlib import Path
+import atexit
 from typing import MutableMapping, TextIO, TypedDict, Any, TYPE_CHECKING, cast
 from corelibs.logging_handling.logging_level_handling.logging_level import LoggingLevel
 from corelibs.string_handling.text_colors import Colors
@@ -23,6 +25,7 @@ class LogSettings(TypedDict):
     """log settings, for Log setup"""
     log_level_console: LoggingLevel
     log_level_file: LoggingLevel
+    per_run_log: bool
     console_enabled: bool
     console_color_output_enabled: bool
     add_start_info: bool
@@ -119,6 +122,9 @@ class LogParent:
         self.log_queue: 'Queue[str] | None' = None
         self.handlers: dict[str, Any] = {}
 
+    def __del__(self):
+        self._cleanup()
+
     # FIXME: we need to add a custom formater to add stack level listing if we want to
     # Important note, although they exist, it is recommended to use self.logger.NAME directly
     # so that the correct filename, method and row number is set
@@ -280,6 +286,15 @@ class LogParent:
             return False
         return True
 
+    def _cleanup(self):
+        """cleanup for any open queues in case we have an abort"""
+        if not self.log_queue:
+            return
+        self.flush()
+        # Close the queue properly
+        self.log_queue.close()
+        self.log_queue.join_thread()
+
     # MARK: log level handling
     def set_log_level(self, handler_name: str, log_level: LoggingLevel) -> bool:
         """
@@ -392,6 +407,7 @@ class Log(LogParent):
     DEFAULT_LOG_SETTINGS: LogSettings = {
         "log_level_console": DEFAULT_LOG_LEVEL_CONSOLE,
         "log_level_file": DEFAULT_LOG_LEVEL_FILE,
+        "per_run_log": False,
         "console_enabled": True,
         "console_color_output_enabled": True,
         "add_start_info": True,
@@ -440,7 +456,7 @@ class Log(LogParent):
         # in the file writer too, for the ones where color is set BEFORE the format
         # Any is logging.StreamHandler, logging.FileHandler and all logging.handlers.*
         self.handlers: dict[str, Any] = {}
-        self.add_handler('file_handler', self.__create_timed_rotating_file_handler(
+        self.add_handler('file_handler', self.__create_file_handler(
             'file_handler', self.log_settings['log_level_file'], log_path)
         )
         if self.log_settings['console_enabled']:
@@ -492,6 +508,7 @@ class Log(LogParent):
             default_log_settings[__log_entry] = LoggingLevel.from_any(__log_level)
         # check bool
         for __log_entry in [
+            "per_run_log",
             "console_enabled",
             "console_color_output_enabled",
             "add_start_info",
@@ -566,24 +583,35 @@ class Log(LogParent):
         return console_handler
 
     # MARK: file handler
-    def __create_timed_rotating_file_handler(
+    def __create_file_handler(
         self, handler_name: str,
         log_level_file: LoggingLevel, log_path: Path,
+        # for TimedRotating, if per_run_log is off
         when: str = "D", interval: int = 1, backup_count: int = 0
-    ) -> logging.handlers.TimedRotatingFileHandler:
+    ) -> logging.handlers.TimedRotatingFileHandler | logging.FileHandler:
         # file logger
         # when: S/M/H/D/W0-W6/midnight
         # interval: how many, 1D = every day
         # backup_count: how many old to keep, 0 = all
         if not self.validate_log_level(log_level_file):
             log_level_file = self.DEFAULT_LOG_LEVEL_FILE
-        file_handler = logging.handlers.TimedRotatingFileHandler(
+        if self.log_settings['per_run_log']:
-            filename=log_path,
+            # log path, remove them stem (".log"), then add the datetime and add .log again
-            encoding="utf-8",
+            now = datetime.now()
-            when=when,
+            # we add microseconds part to get milli seconds
-            interval=interval,
+            new_stem=f"{log_path.stem}.{now.strftime('%Y-%m-%d_%H-%M-%S')}.{str(now.microsecond)[:3]}"
-            backupCount=backup_count
+            file_handler = logging.FileHandler(
-        )
+                filename=log_path.with_name(f"{new_stem}{log_path.suffix}"),
+                encoding="utf-8",
+            )
+        else:
+            file_handler = logging.handlers.TimedRotatingFileHandler(
+                filename=log_path,
+                encoding="utf-8",
+                when=when,
+                interval=interval,
+                backupCount=backup_count
+            )
         formatter_file_handler = logging.Formatter(
             (
                 # time stamp
@@ -619,6 +647,7 @@ class Log(LogParent):
         if log_queue is None:
             return
         self.log_queue = log_queue
+        atexit.register(self._cleanup)
         self.listener = logging.handlers.QueueListener(
             self.log_queue,
             *self.handlers.values(),

test-run/logging_handling/log.py

@@ -24,6 +24,7 @@ def main():
             # "log_level_console": None,
             "log_level_file": 'DEBUG',
             # "console_color_output_enabled": False,
+            "per_run_log": True
         }
     )
     logn = Logger(log.get_logger_settings())

uv.lock

@@ -1,5 +1,5 @@
 version = 1
-revision = 2
+revision = 3
 requires-python = ">=3.13"
 
 [[package]]
@@ -44,7 +44,7 @@ wheels = [
 
 [[package]]
 name = "corelibs"
-version = "0.21.0"
+version = "0.22.0"
 source = { editable = "." }
 dependencies = [
     { name = "jmespath" },