v0.14.0: logging exception automatically writes to error for console output

When calling log.exception it automatically writes a short message to the console as error
This error message is tagged <=EXCEPTION> and contains only the message without the stack trace

pyproject.toml

@@ -1,7 +1,7 @@
 # MARK: Project info
 [project]
 name = "corelibs"
-version = "0.12.4"
+version = "0.14.0"
 description = "Collection of utils for Python scripts"
 readme = "README.md"
 requires-python = ">=3.13"

src/corelibs/check_handling/regex_constants.py

@@ -5,10 +5,33 @@ List of regex compiled strings that can be used
 import re
 
 
-EMAIL_REGEX_BASIC = r"""
+def compile_re(reg: str) -> re.Pattern[str]:
+    """
+    compile a regex with verbose flag
+
+    Arguments:
+        reg {str} -- _description_
+
+    Returns:
+        re.Pattern[str] -- _description_
+    """
+    return re.compile(reg, re.VERBOSE)
+
+
+# email regex
+EMAIL_BASIC_REGEX = r"""
 ^[A-Za-z0-9!#$%&'*+\-\/=?^_`{|}~][A-Za-z0-9!#$%:\(\)&'*+\-\/=?^_`{|}~\.]{0,63}
 @(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[a-zA-Z]{2,6}$
 """
-EMAIL_REGEX_BASIC_COMPILED = re.compile(EMAIL_REGEX_BASIC)
+# Domain regex with localhost
+DOMAIN_WITH_LOCALHOST_REGEX = r"""
+^(?:localhost|(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[A-Za-z]{2,})$
+"""
+# domain regex with loclhost and optional port
+DOMAIN_WITH_LOCALHOST_PORT_REGEX = r"""
+^(?:localhost|(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[A-Za-z]{2,})(?::\d+)?$
+"""
+# Domain, no localhost
+DOMAIN_REGEX = r"^(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[A-Za-z]{2,}$"
 
 # __END__

src/corelibs/config_handling/settings_loader.py

@@ -5,7 +5,6 @@ Additional check for override settings as arguments
 """
 
 import re
-import sys
 import configparser
 from typing import Any, Tuple, Sequence, cast
 from pathlib import Path
@@ -66,7 +65,12 @@ class SettingsLoader:
         self._check_settings_abort: bool = False
 
     # MARK: load settings
-    def load_settings(self, config_id: str, config_validate: dict[str, list[str]]) -> dict[str, str]:
+    def load_settings(
+        self,
+        config_id: str,
+        config_validate: dict[str, list[str]],
+        allow_not_exist: bool = False
+    ) -> dict[str, str]:
         """
         neutral settings loader
 
@@ -84,9 +88,12 @@ class SettingsLoader:
         - convert: convert to int, float -> if element is number convert, else leave as is
         - empty: convert empty to, if nothing set on the right side then convert to None type
 
+        TODO: there should be a config/options argument for general settings
+
         Args:
             config_id (str): what block to load
-            config_allowed (list[str]): list of allowed entries sets
+            config_validate (dict[str, list[str]]): list of allowed entries sets
+            allow_not_exist (bool): If set to True, does not throw an error, but returns empty set
 
         Returns:
             dict[str, str]: key = value list
@@ -99,11 +106,12 @@ class SettingsLoader:
                 # load all data as is, validation is done afterwards
                 settings[config_id] = dict(self.config_parser[config_id])
             except KeyError as e:
-                self.__print(
+                if allow_not_exist is True:
+                    return {}
+                raise ValueError(self.__print(
                     f"[!] Cannot read [{config_id}] block in the {self.config_file}: {e}",
-                    'CRITICAL', raise_exception=True
+                    'CRITICAL'
-                )
+                )) from e
-                sys.exit(1)
             try:
                 for key, checks in config_validate.items():
                     skip = True
@@ -114,20 +122,16 @@ class SettingsLoader:
                             try:
                                 [_, convert_to] = check.split(":")
                                 if convert_to not in self.CONVERT_TO_LIST:
-                                    self.__print(
+                                    raise ValueError(self.__print(
                                         f"[!] In [{config_id}] the convert type is invalid {check}: {convert_to}",
-                                        'CRITICAL',
+                                        'CRITICAL'
-                                        raise_exception=True
+                                    ))
-                                    )
-                                    sys.exit(1)
                                 self.entry_convert[key] = convert_to
                             except ValueError as e:
-                                self.__print(
+                                raise ValueError(self.__print(
                                     f"[!] In [{config_id}] the convert type setup for entry failed: {check}: {e}",
-                                    'CRITICAL',
+                                    'CRITICAL'
-                                    raise_exception=True
+                                )) from e
-                                )
-                                sys.exit(1)
                         if check.startswith('empty:'):
                             try:
                                 [_, empty_set] = check.split(":")
@@ -136,12 +140,10 @@ class SettingsLoader:
                                 self.entry_set_empty[key] = empty_set
                             except ValueError as e:
                                 print(f"VALUE ERROR: {key}")
-                                self.__print(
+                                raise ValueError(self.__print(
                                     f"[!] In [{config_id}] the empty set type for entry failed: {check}: {e}",
-                                    'CRITICAL',
+                                    'CRITICAL'
-                                    raise_exception=True
+                                )) from e
-                                )
-                                sys.exit(1)
                         # split char, also check to not set it twice, first one only
                         if check.startswith("split:") and not self.entry_split_char.get(key):
                             try:
@@ -158,12 +160,10 @@ class SettingsLoader:
                                 self.entry_split_char[key] = split_char
                                 skip = False
                             except ValueError as e:
-                                self.__print(
+                                raise ValueError(self.__print(
                                     f"[!] In [{config_id}] the split character setup for entry failed: {check}: {e}",
-                                    'CRITICAL',
+                                    'CRITICAL'
-                                    raise_exception=True
+                                )) from e
-                                )
-                                sys.exit(1)
                     if skip:
                         continue
                     settings[config_id][key] = [
@@ -171,16 +171,14 @@ class SettingsLoader:
                         for __value in settings[config_id][key].split(split_char)
                     ]
             except KeyError as e:
-                self.__print(
+                raise ValueError(self.__print(
                     f"[!] Cannot read [{config_id}] block because the entry [{e}] could not be found",
-                    'CRITICAL', raise_exception=True
+                    'CRITICAL'
-                )
+                )) from e
-                sys.exit(1)
         else:
             # ignore error if arguments are set
             if not self.__check_arguments(config_validate, True):
-                self.__print(f"[!] Cannot find file: {self.config_file}", 'CRITICAL', raise_exception=True)
+                raise ValueError(self.__print(f"[!] Cannot find file: {self.config_file}", 'CRITICAL'))
-                sys.exit(1)
             else:
                 # base set
                 settings[config_id] = {}
@@ -260,9 +258,14 @@ class SettingsLoader:
                         self.__build_from_to_equal(entry, check)
                     ):
                         error = True
+                # after post clean up if we have empty entries and we are mandatory
+                if check == "mandatory:yes" and (
+                    not settings[config_id].get(entry) or settings[config_id].get(entry) == ['']
+                ):
+                    error = True
+                    self.__print(f"[!] Missing content entry for: {entry}", 'ERROR')
         if error is True:
-            self.__print("[!] Missing or incorrect settings data. Cannot proceed", 'CRITICAL', raise_exception=True)
+            raise ValueError(self.__print("[!] Missing or incorrect settings data. Cannot proceed", 'CRITICAL'))
-            sys.exit(1)
         # set empty
         for [entry, empty_set] in self.entry_set_empty.items():
             # if set, skip, else set to empty value
@@ -316,22 +319,20 @@ class SettingsLoader:
         try:
             [__from, __to] = check.split('-')
             if (__from and not is_float(__from)) or (__to and not is_float(__to)):
-                self.__print(
+                raise ValueError(self.__print(
                     f"[{entry}] Check value for length is not in: {check}",
-                    'CRITICAL', raise_exception=True
+                    'CRITICAL'
-                )
+                ))
-                sys.exit(1)
             if len(__from) == 0:
                 __from = None
             if len(__to) == 0:
                 __to = None
-        except ValueError:
+        except ValueError as e:
             if not is_float(__equal := check):
-                self.__print(
+                raise ValueError(self.__print(
                     f"[{entry}] Check value for length is not a valid integer: {check}",
-                    'CRITICAL', raise_exception=True
+                    'CRITICAL'
-                )
+                )) from e
-                sys.exit(1)
             if len(__equal) == 0:
                 __equal = None
         # makre sure this is all int or None
@@ -467,11 +468,10 @@ class SettingsLoader:
         # get the check settings
         __check_settings = SettingsLoaderCheck.CHECK_SETTINGS.get(check)
         if __check_settings is None:
-            self.__print(
+            raise ValueError(self.__print(
                 f"[{entry}] Cannot get SettingsLoaderCheck.CHECK_SETTINGS for {check}",
-                'CRITICAL', raise_exception=True
+                'CRITICAL'
-            )
+            ))
-            sys.exit(1)
         # either removes or replaces invalid characters in the list
         if isinstance(setting_value, list):
             # clean up invalid characters
@@ -539,7 +539,7 @@ class SettingsLoader:
         return self.args.get(entry)
 
     # MARK: error print
-    def __print(self, msg: str, level: str, print_error: bool = True, raise_exception: bool = False):
+    def __print(self, msg: str, level: str, print_error: bool = True) -> str:
         """
         print out error, if Log class is set then print to log instead
 
@@ -553,12 +553,11 @@ class SettingsLoader:
         if self.log is not None:
             if not Log.validate_log_level(level):
                 level = 'ERROR'
-            self.log.logger.log(Log.get_log_level_int(level), msg)
+            self.log.logger.log(Log.get_log_level_int(level), msg, stacklevel=2)
         if self.log is None or self.always_print:
             if print_error:
                 print(msg)
-        if raise_exception:
+        return msg
-            raise ValueError(msg)
 
 
 # __END__

src/corelibs/config_handling/settings_loader_handling/settings_loader_check.py

@@ -3,7 +3,9 @@ Class of checks that can be run on value entries
 """
 
 from typing import TypedDict
-from corelibs.check_handling.regex_constants import EMAIL_REGEX_BASIC
+from corelibs.check_handling.regex_constants import (
+    EMAIL_BASIC_REGEX, DOMAIN_WITH_LOCALHOST_REGEX, DOMAIN_WITH_LOCALHOST_PORT_REGEX, DOMAIN_REGEX
+)
 
 
 class SettingsLoaderCheckValue(TypedDict):
@@ -45,10 +47,34 @@ class SettingsLoaderCheck:
         },
         # This does a baisc email check, only alphanumeric with special characters
         "string.email.basic": {
-            "regex": EMAIL_REGEX_BASIC,
+            "regex": EMAIL_BASIC_REGEX,
             "regex_clean": None,
             "replace": "",
         },
+        # Domain check, including localhost no port
+        "string.domain.with-localhost": {
+            "regex": DOMAIN_WITH_LOCALHOST_REGEX,
+            "regex_clean": None,
+            "replace": "",
+        },
+        # Domain check, with localhost and port
+        "string.domain.with-localhost.port": {
+            "regex": DOMAIN_WITH_LOCALHOST_PORT_REGEX,
+            "regex_clean": None,
+            "replace": "",
+        },
+        # Domain check, no pure localhost allowed
+        "string.domain": {
+            "regex": DOMAIN_REGEX,
+            "regex_clean": None,
+            "replace": "",
+        },
+        # Basic date check, does not validate date itself
+        "string.date": {
+            "regex": r"^\d{4}[/-]\d{1,2}[/-]\d{1,2}$",
+            "regex_clean": None,
+            "replace": "",
+        }
     }
 
 

src/corelibs/logging_handling/log.py

@@ -73,9 +73,30 @@ class CustomConsoleFormatter(logging.Formatter):
         message = super().format(record)
         return f"{color}{message}{reset}"
 
+# TODO: add custom handlers for stack_trace, if not set fill with %(filename)s:%(funcName)s:%(lineno)d
+# hasattr(record, 'stack_trace')
 
-# TODO: add custom handlers for stack_correct, if not set fill with %(filename)s:%(funcName)s:%(lineno)d
+
-# hasattr(record, 'stack_correct')
+class CustomHandlerFilter(logging.Filter):
+    """
+    Add a custom handler for filtering
+    """
+    def __init__(self, handler_name: str, filter_exceptions: bool = False):
+        super().__init__(name=handler_name)
+        self.handler_name = handler_name
+        self.filter_exceptions = filter_exceptions
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        # if console and exception do not show
+        if self.handler_name == 'console' and self.filter_exceptions:
+            return record.levelname != "EXCEPTION"
+        # if cnosole entry is true and traget file filter
+        if hasattr(record, 'console') and getattr(record, 'console') is True and self.handler_name == 'file':
+            return False
+        return True
+
+    # def __filter_exceptions(self, record: logging.LogRecord) -> bool:
+    #     return record.levelname != "EXCEPTION"
 
 
 # MARK: Log class
@@ -142,7 +163,7 @@ class Log:
         # in the file writer too, for the ones where color is set BEFORE the format
         # Any is logging.StreamHandler, logging.FileHandler and all logging.handlers.*
         self.handlers: dict[str, Any] = {}
-        self.add_handler('file_handler', self.__create_time_rotating_file_handler(
+        self.add_handler('file_handler', self.__create_timed_rotating_file_handler(
             self.log_settings['log_level_file'], log_path)
         )
         if self.log_settings['console_enabled']:
@@ -211,8 +232,8 @@ class Log:
         default_log_settings['log_queue'] = __setting
         return default_log_settings
 
-    def __filter_exceptions(self, record: logging.LogRecord) -> bool:
+    # def __filter_exceptions(self, record: logging.LogRecord) -> bool:
-        return record.levelname != "EXCEPTION"
+    #     return record.levelname != "EXCEPTION"
 
     # MARK: add a handler
     def add_handler(
@@ -259,16 +280,15 @@ class Log:
             formatter_console = CustomConsoleFormatter(format_string, datefmt=format_date)
         else:
             formatter_console = logging.Formatter(format_string, datefmt=format_date)
-        console_handler.setLevel(log_level_console.name)
         console_handler.set_name('console')
+        console_handler.setLevel(log_level_console.name)
         # do not show exceptions logs on console
-        if filter_exceptions:
+        console_handler.addFilter(CustomHandlerFilter('console', filter_exceptions))
-            console_handler.addFilter(self.__filter_exceptions)
         console_handler.setFormatter(formatter_console)
         return console_handler
 
     # MARK: file handler
-    def __create_time_rotating_file_handler(
+    def __create_timed_rotating_file_handler(
         self, log_level_file: LoggingLevel, log_path: Path,
         when: str = "D", interval: int = 1, backup_count: int = 0
     ) -> logging.handlers.TimedRotatingFileHandler:
@@ -304,6 +324,8 @@ class Log:
         )
         file_handler.set_name('file_timed_rotate')
         file_handler.setLevel(log_level_file.name)
+        # do not show errors flagged with console (they are from exceptions)
+        file_handler.addFilter(CustomHandlerFilter('file'))
         file_handler.setFormatter(formatter_file_handler)
         return file_handler
 
@@ -367,7 +389,7 @@ class Log:
 
         return root_logger
 
-    # FIXME: all below will only work if we add a custom format interface for the stack_correct part
+    # FIXME: we need to add a custom formater to add stack level listing if we want to
     # Important note, although they exist, it is recommended to use self.logger.NAME directly
     # so that the correct filename, method and row number is set
     # for > 50 use logger.log(LoggingLevel.<LEVEL>.value, ...)
@@ -379,8 +401,8 @@ class Log:
             raise ValueError('Logger is not yet initialized')
         if extra is None:
             extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
+        extra['stack_trace'] = traceback_call_str(start=3)
-        self.logger.log(level, msg, *args, extra=extra)
+        self.logger.log(level, msg, *args, extra=extra, stacklevel=2)
 
     # MARK: DEBUG 10
     def debug(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -389,8 +411,8 @@ class Log:
             raise ValueError('Logger is not yet initialized')
         if extra is None:
             extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
+        extra['stack_trace'] = traceback_call_str(start=3)
-        self.logger.debug(msg, *args, extra=extra)
+        self.logger.debug(msg, *args, extra=extra, stacklevel=2)
 
     # MARK: INFO 20
     def info(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -399,8 +421,8 @@ class Log:
             raise ValueError('Logger is not yet initialized')
         if extra is None:
             extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
+        extra['stack_trace'] = traceback_call_str(start=3)
-        self.logger.info(msg, *args, extra=extra)
+        self.logger.info(msg, *args, extra=extra, stacklevel=2)
 
     # MARK: WARNING 30
     def warning(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -409,8 +431,8 @@ class Log:
             raise ValueError('Logger is not yet initialized')
         if extra is None:
             extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
+        extra['stack_trace'] = traceback_call_str(start=3)
-        self.logger.warning(msg, *args, extra=extra)
+        self.logger.warning(msg, *args, extra=extra, stacklevel=2)
 
     # MARK: ERROR 40
     def error(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -419,8 +441,8 @@ class Log:
             raise ValueError('Logger is not yet initialized')
         if extra is None:
             extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
+        extra['stack_trace'] = traceback_call_str(start=3)
-        self.logger.error(msg, *args, extra=extra)
+        self.logger.error(msg, *args, extra=extra, stacklevel=2)
 
     # MARK: CRITICAL 50
     def critical(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -429,8 +451,8 @@ class Log:
             raise ValueError('Logger is not yet initialized')
         if extra is None:
             extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
+        extra['stack_trace'] = traceback_call_str(start=3)
-        self.logger.critical(msg, *args, extra=extra)
+        self.logger.critical(msg, *args, extra=extra, stacklevel=2)
 
     # MARK: ALERT 55
     def alert(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -440,8 +462,8 @@ class Log:
         # extra_dict = dict(extra)
         if extra is None:
             extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
+        extra['stack_trace'] = traceback_call_str(start=3)
-        self.logger.log(LoggingLevel.ALERT.value, msg, *args, extra=extra)
+        self.logger.log(LoggingLevel.ALERT.value, msg, *args, extra=extra, stacklevel=2)
 
     # MARK: EMERGECNY: 60
     def emergency(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -450,11 +472,15 @@ class Log:
             raise ValueError('Logger is not yet initialized')
         if extra is None:
             extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
+        extra['stack_trace'] = traceback_call_str(start=3)
-        self.logger.log(LoggingLevel.EMERGENCY.value, msg, *args, extra=extra)
+        self.logger.log(LoggingLevel.EMERGENCY.value, msg, *args, extra=extra, stacklevel=2)
 
     # MARK: EXCEPTION: 70
-    def exception(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
+    def exception(
+        self,
+        msg: object, *args: object, extra: MutableMapping[str, object] | None = None,
+        log_error: bool = True
+    ) -> None:
         """
         log on exceotion level, this is log.exception, but logs with a new level
 
@@ -462,13 +488,20 @@ class Log:
             msg (object): _description_
             *args (object): arguments for msg
             extra: Mapping[str, object] | None: extra arguments for the formatting if needed
+            log_error: (bool): If set to false will not write additional error message for console (Default True)
         """
         if not hasattr(self, 'logger'):
             raise ValueError('Logger is not yet initialized')
         if extra is None:
             extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
+        extra['stack_trace'] = traceback_call_str(start=3)
-        self.logger.log(LoggingLevel.EXCEPTION.value, msg, *args, exc_info=True, extra=extra)
+        # write to console first with extra flag for filtering in file
+        if log_error:
+            self.logger.log(
+                LoggingLevel.ERROR.value,
+                f"<=EXCEPTION> {msg}", *args, extra=dict(extra) | {'console': True}, stacklevel=2
+            )
+        self.logger.log(LoggingLevel.EXCEPTION.value, msg, *args, exc_info=True, extra=extra, stacklevel=2)
 
     # MARK: break line
     def break_line(self, info: str = "BREAK"):

test-run/config_handling/settings_loader.py

@@ -2,10 +2,12 @@
 Settings loader test
 """
 
+import re
 from pathlib import Path
-from corelibs.iterator_handling.dump_data import dump_data
+from corelibs.debug_handling.dump_data import dump_data
 from corelibs.logging_handling.log import Log
 from corelibs.config_handling.settings_loader import SettingsLoader
+from corelibs.config_handling.settings_loader_handling.settings_loader_check import SettingsLoaderCheck
 
 SCRIPT_PATH: Path = Path(__file__).resolve().parent
 ROOT_PATH: Path = SCRIPT_PATH
@@ -18,6 +20,11 @@ def main():
     Main run
     """
 
+    value = "2025/1/1"
+    regex_c = re.compile(SettingsLoaderCheck.CHECK_SETTINGS['string.date']['regex'], re.VERBOSE)
+    result = regex_c.search(value)
+    print(f"regex {regex_c} check against {value} -> {result}")
+
     # for log testing
     script_path: Path = Path(__file__).resolve().parent
     log = Log(

test-run/logging_handling/log.py

@@ -76,7 +76,7 @@ def main():
         print(f"Divied: {__test}")
     except ZeroDivisionError as e:
         log.logger.critical("Divison through zero: %s", e)
-        log.exception("Divison through zero")
+        log.exception("Divison through zero: %s", e)
 
     for handler in log.logger.handlers:
         print(f"Handler (logger) {handler} -> {handler.level} -> {LoggingLevel.from_any(handler.level)}")

uv.lock

@@ -44,7 +44,7 @@ wheels = [
 
 [[package]]
 name = "corelibs"
-version = "0.12.2"
+version = "0.13.2"
 source = { editable = "." }
 dependencies = [
     { name = "jmespath" },