v0.22.1: Log update with closing queue on exit or abort

to avoid hung threads on errors

ToDo.md

@@ -1,5 +1,7 @@
 # ToDo list
 
-- [ ] stub files .pyi
+- [x] stub files .pyi
 - [ ] Add tests for all, we need 100% test coverate
-- [ ] Log: add custom format for "stack_correct" if set, this will override the normal stack block
+- [x] Log: add custom format for "stack_correct" if set, this will override the normal stack block
+- [ ] Log: add rotate for size based
+- [ ] All folders and file names need to be revisited for naming and content collection

pyproject.toml

@@ -1,7 +1,7 @@
 # MARK: Project info
 [project]
 name = "corelibs"
-version = "0.12.3"
+version = "0.22.1"
 description = "Collection of utils for Python scripts"
 readme = "README.md"
 requires-python = ">=3.13"

src/corelibs/check_handling/regex_constants.py

@@ -5,10 +5,33 @@ List of regex compiled strings that can be used
 import re
 
 
-EMAIL_REGEX_BASIC = r"""
+def compile_re(reg: str) -> re.Pattern[str]:
+    """
+    compile a regex with verbose flag
+
+    Arguments:
+        reg {str} -- _description_
+
+    Returns:
+        re.Pattern[str] -- _description_
+    """
+    return re.compile(reg, re.VERBOSE)
+
+
+# email regex
+EMAIL_BASIC_REGEX: str = r"""
 ^[A-Za-z0-9!#$%&'*+\-\/=?^_`{|}~][A-Za-z0-9!#$%:\(\)&'*+\-\/=?^_`{|}~\.]{0,63}
 @(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[a-zA-Z]{2,6}$
 """
-EMAIL_REGEX_BASIC_COMPILED = re.compile(EMAIL_REGEX_BASIC)
+# Domain regex with localhost
+DOMAIN_WITH_LOCALHOST_REGEX: str = r"""
+^(?:localhost|(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[A-Za-z]{2,})$
+"""
+# domain regex with loclhost and optional port
+DOMAIN_WITH_LOCALHOST_PORT_REGEX: str = r"""
+^(?:localhost|(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[A-Za-z]{2,})(?::\d+)?$
+"""
+# Domain, no localhost
+DOMAIN_REGEX: str = r"^(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[A-Za-z]{2,}$"
 
 # __END__

src/corelibs/config_handling/settings_loader.py

@@ -5,7 +5,6 @@ Additional check for override settings as arguments
 """
 
 import re
-import sys
 import configparser
 from typing import Any, Tuple, Sequence, cast
 from pathlib import Path
@@ -51,11 +50,14 @@ class SettingsLoader:
         self.always_print = always_print
         # entries that have to be split
         self.entry_split_char: dict[str, str] = {}
+        # entries that should be converted
         self.entry_convert: dict[str, str] = {}
+        # default set entries
+        self.entry_set_empty: dict[str, str | None] = {}
         # config parser, load config file first
         self.config_parser: configparser.ConfigParser | None = self.__load_config_file()
         # all settings
-        self.settings: dict[str, dict[str, Any]] | None = None
+        self.settings: dict[str, dict[str, None | str | int | float | bool]] | None = None
         # remove file name and get base path and check
         if not self.config_file.parent.is_dir():
             raise ValueError(f"Cannot find the config folder: {self.config_file.parent}")
@@ -63,7 +65,12 @@ class SettingsLoader:
         self._check_settings_abort: bool = False
 
     # MARK: load settings
-    def load_settings(self, config_id: str, config_validate: dict[str, list[str]]) -> dict[str, str]:
+    def load_settings(
+        self,
+        config_id: str,
+        config_validate: dict[str, list[str]],
+        allow_not_exist: bool = False
+    ) -> dict[str, str]:
         """
         neutral settings loader
 
@@ -79,10 +86,14 @@ class SettingsLoader:
         - in: the right side is another KEY value from the settings where this value must be inside
         - split: character to split entries, if set check:list+ must be set if checks are needed
         - convert: convert to int, float -> if element is number convert, else leave as is
+        - empty: convert empty to, if nothing set on the right side then convert to None type
+
+        TODO: there should be a config/options argument for general settings
 
         Args:
             config_id (str): what block to load
-            config_allowed (list[str]): list of allowed entries sets
+            config_validate (dict[str, list[str]]): list of allowed entries sets
+            allow_not_exist (bool): If set to True, does not throw an error, but returns empty set
 
         Returns:
             dict[str, str]: key = value list
@@ -95,11 +106,12 @@ class SettingsLoader:
                 # load all data as is, validation is done afterwards
                 settings[config_id] = dict(self.config_parser[config_id])
             except KeyError as e:
-                self.__print(
+                if allow_not_exist is True:
+                    return {}
+                raise ValueError(self.__print(
                     f"[!] Cannot read [{config_id}] block in the {self.config_file}: {e}",
-                    'CRITICAL', raise_exception=True
-                )
-                sys.exit(1)
+                    'CRITICAL'
+                )) from e
             try:
                 for key, checks in config_validate.items():
                     skip = True
@@ -110,18 +122,28 @@ class SettingsLoader:
                             try:
                                 [_, convert_to] = check.split(":")
                                 if convert_to not in self.CONVERT_TO_LIST:
-                                    self.__print(
+                                    raise ValueError(self.__print(
                                         f"[!] In [{config_id}] the convert type is invalid {check}: {convert_to}",
-                                        'CRITICAL',
-                                        raise_exception=True
-                                    )
+                                        'CRITICAL'
+                                    ))
+                                self.entry_convert[key] = convert_to
                             except ValueError as e:
-                                self.__print(
+                                raise ValueError(self.__print(
                                     f"[!] In [{config_id}] the convert type setup for entry failed: {check}: {e}",
-                                    'CRITICAL',
-                                    raise_exception=True
-                                )
-                                sys.exit(1)
+                                    'CRITICAL'
+                                )) from e
+                        if check.startswith('empty:'):
+                            try:
+                                [_, empty_set] = check.split(":")
+                                if not empty_set:
+                                    empty_set = None
+                                self.entry_set_empty[key] = empty_set
+                            except ValueError as e:
+                                print(f"VALUE ERROR: {key}")
+                                raise ValueError(self.__print(
+                                    f"[!] In [{config_id}] the empty set type for entry failed: {check}: {e}",
+                                    'CRITICAL'
+                                )) from e
                         # split char, also check to not set it twice, first one only
                         if check.startswith("split:") and not self.entry_split_char.get(key):
                             try:
@@ -138,12 +160,10 @@ class SettingsLoader:
                                 self.entry_split_char[key] = split_char
                                 skip = False
                             except ValueError as e:
-                                self.__print(
+                                raise ValueError(self.__print(
                                     f"[!] In [{config_id}] the split character setup for entry failed: {check}: {e}",
-                                    'CRITICAL',
-                                    raise_exception=True
-                                )
-                                sys.exit(1)
+                                    'CRITICAL'
+                                )) from e
                     if skip:
                         continue
                     settings[config_id][key] = [
@@ -151,16 +171,14 @@ class SettingsLoader:
                         for __value in settings[config_id][key].split(split_char)
                     ]
             except KeyError as e:
-                self.__print(
+                raise ValueError(self.__print(
                     f"[!] Cannot read [{config_id}] block because the entry [{e}] could not be found",
-                    'CRITICAL', raise_exception=True
-                )
-                sys.exit(1)
+                    'CRITICAL'
+                )) from e
         else:
             # ignore error if arguments are set
             if not self.__check_arguments(config_validate, True):
-                self.__print(f"[!] Cannot find file: {self.config_file}", 'CRITICAL', raise_exception=True)
-                sys.exit(1)
+                raise ValueError(self.__print(f"[!] Cannot find file: {self.config_file}", 'CRITICAL'))
             else:
                 # base set
                 settings[config_id] = {}
@@ -240,11 +258,22 @@ class SettingsLoader:
                         self.__build_from_to_equal(entry, check)
                     ):
                         error = True
+                # after post clean up if we have empty entries and we are mandatory
+                if check == "mandatory:yes" and (
+                    not settings[config_id].get(entry) or settings[config_id].get(entry) == ['']
+                ):
+                    error = True
+                    self.__print(f"[!] Missing content entry for: {entry}", 'ERROR')
         if error is True:
-            self.__print("[!] Missing or incorrect settings data. Cannot proceed", 'CRITICAL', raise_exception=True)
-            sys.exit(1)
+            raise ValueError(self.__print("[!] Missing or incorrect settings data. Cannot proceed", 'CRITICAL'))
+        # set empty
+        for [entry, empty_set] in self.entry_set_empty.items():
+            # if set, skip, else set to empty value
+            if settings[config_id].get(entry) or isinstance(settings[config_id].get(entry), list):
+                continue
+            settings[config_id][entry] = empty_set
         # Convert input
-        for [entry, convert_type] in self.entry_convert:
+        for [entry, convert_type] in self.entry_convert.items():
             if convert_type in ["int", "any"] and is_int(settings[config_id][entry]):
                 settings[config_id][entry] = int(settings[config_id][entry])
             elif convert_type in ["float", "any"] and is_float(settings[config_id][entry]):
@@ -263,6 +292,7 @@ class SettingsLoader:
                         'ERROR'
                     )
             # string is always string
+            # TODO: empty and int/float/bool: set to none?
 
         return settings[config_id]
 
@@ -289,22 +319,20 @@ class SettingsLoader:
         try:
             [__from, __to] = check.split('-')
             if (__from and not is_float(__from)) or (__to and not is_float(__to)):
-                self.__print(
+                raise ValueError(self.__print(
                     f"[{entry}] Check value for length is not in: {check}",
-                    'CRITICAL', raise_exception=True
-                )
-                sys.exit(1)
+                    'CRITICAL'
+                ))
             if len(__from) == 0:
                 __from = None
             if len(__to) == 0:
                 __to = None
-        except ValueError:
+        except ValueError as e:
             if not is_float(__equal := check):
-                self.__print(
+                raise ValueError(self.__print(
                     f"[{entry}] Check value for length is not a valid integer: {check}",
-                    'CRITICAL', raise_exception=True
-                )
-                sys.exit(1)
+                    'CRITICAL'
+                )) from e
             if len(__equal) == 0:
                 __equal = None
         # makre sure this is all int or None
@@ -331,6 +359,9 @@ class SettingsLoader:
         (__from, __to, __equal) = check
         valid = True
         for value_raw in convert_to_list(values):
+            # skip no tset values for range check
+            if not value_raw:
+                continue
             value = 0
             error_mark = ''
             if check_type == 'length':
@@ -437,11 +468,10 @@ class SettingsLoader:
         # get the check settings
         __check_settings = SettingsLoaderCheck.CHECK_SETTINGS.get(check)
         if __check_settings is None:
-            self.__print(
+            raise ValueError(self.__print(
                 f"[{entry}] Cannot get SettingsLoaderCheck.CHECK_SETTINGS for {check}",
-                'CRITICAL', raise_exception=True
-            )
-            sys.exit(1)
+                'CRITICAL'
+            ))
         # either removes or replaces invalid characters in the list
         if isinstance(setting_value, list):
             # clean up invalid characters
@@ -509,7 +539,7 @@ class SettingsLoader:
         return self.args.get(entry)
 
     # MARK: error print
-    def __print(self, msg: str, level: str, print_error: bool = True, raise_exception: bool = False):
+    def __print(self, msg: str, level: str, print_error: bool = True) -> str:
         """
         print out error, if Log class is set then print to log instead
 
@@ -523,12 +553,11 @@ class SettingsLoader:
         if self.log is not None:
             if not Log.validate_log_level(level):
                 level = 'ERROR'
-            self.log.logger.log(Log.get_log_level_int(level), msg)
+            self.log.logger.log(Log.get_log_level_int(level), msg, stacklevel=2)
         if self.log is None or self.always_print:
             if print_error:
                 print(msg)
-        if raise_exception:
-            raise ValueError(msg)
+        return msg
 
 
 # __END__

src/corelibs/config_handling/settings_loader_handling/settings_loader_check.py

@@ -3,7 +3,9 @@ Class of checks that can be run on value entries
 """
 
 from typing import TypedDict
-from corelibs.check_handling.regex_constants import EMAIL_REGEX_BASIC
+from corelibs.check_handling.regex_constants import (
+    EMAIL_BASIC_REGEX, DOMAIN_WITH_LOCALHOST_REGEX, DOMAIN_WITH_LOCALHOST_PORT_REGEX, DOMAIN_REGEX
+)
 
 
 class SettingsLoaderCheckValue(TypedDict):
@@ -45,10 +47,34 @@ class SettingsLoaderCheck:
         },
         # This does a baisc email check, only alphanumeric with special characters
         "string.email.basic": {
-            "regex": EMAIL_REGEX_BASIC,
+            "regex": EMAIL_BASIC_REGEX,
             "regex_clean": None,
             "replace": "",
         },
+        # Domain check, including localhost no port
+        "string.domain.with-localhost": {
+            "regex": DOMAIN_WITH_LOCALHOST_REGEX,
+            "regex_clean": None,
+            "replace": "",
+        },
+        # Domain check, with localhost and port
+        "string.domain.with-localhost.port": {
+            "regex": DOMAIN_WITH_LOCALHOST_PORT_REGEX,
+            "regex_clean": None,
+            "replace": "",
+        },
+        # Domain check, no pure localhost allowed
+        "string.domain": {
+            "regex": DOMAIN_REGEX,
+            "regex_clean": None,
+            "replace": "",
+        },
+        # Basic date check, does not validate date itself
+        "string.date": {
+            "regex": r"^\d{4}[/-]\d{1,2}[/-]\d{1,2}$",
+            "regex_clean": None,
+            "replace": "",
+        }
     }
 
 

src/corelibs/debug_handling/debug_helpers.py

@@ -4,30 +4,72 @@ Various debug helpers
 
 import traceback
 import os
+import sys
+from typing import Tuple, Type
+from types import TracebackType
 
+# _typeshed.OptExcInfo
+OptExcInfo = Tuple[None, None, None] | Tuple[Type[BaseException], BaseException, TracebackType]
 
-def traceback_call_str(start: int = 2, depth: int = 1):
+def call_stack(
+    start: int = 0,
+    skip_last: int = -1,
+    separator: str = ' -> ',
+    reset_start_if_empty: bool = False
+) -> str:
     """
     get the trace for the last entry
 
     Keyword Arguments:
-        start {int} -- _description_ (default: {2})
-        depth {int} -- _description_ (default: {1})
+        start {int} -- start, if too might output will empty until reset_start_if_empty is set (default: {0})
+        skip_last {int} -- how many of the last are skipped, defaults to -1 for current method (default: {-1})
+        seperator {str} -- add stack separator, if empty defaults to ' -> ' (default: { -> })
+        reset_start_if_empty {bool} -- if no stack returned because of too high start,
+                                        reset to 0 for full read (default: {False})
 
     Returns:
-        _type_ -- _description_
+        str -- _description_
     """
-    # can't have more than in the stack for depth
-    depth = min(depth, start)
-    depth = start - depth
-    # 0 is full stack length from start
-    if depth == 0:
-        stack = traceback.extract_stack()[-start:]
+    # stack = traceback.extract_stack()[start:depth]
+    # how many of the last entries we skip (so we do not get self), default is -1
+    # start cannot be negative
+    if skip_last > 0:
+        skip_last = skip_last * -1
+    stack = traceback.extract_stack()
+    __stack = stack[start:skip_last]
+    # start possible to high, reset start to 0
+    if not __stack and reset_start_if_empty:
+        start = 0
+        __stack = stack[start:skip_last]
+    if not separator:
+        separator = ' -> '
+    # print(f"* HERE: {dump_data(stack)}")
+    return f"{separator}".join(f"{os.path.basename(f.filename)}:{f.name}:{f.lineno}" for f in __stack)
+
+
+def exception_stack(
+    exc_stack: OptExcInfo | None = None,
+    separator: str = ' -> '
+) -> str:
+    """
+    Exception traceback, if no sys.exc_info is set, run internal
+
+    Keyword Arguments:
+        exc_stack {OptExcInfo | None} -- _description_ (default: {None})
+        separator {str} -- _description_ (default: {' -> '})
+
+    Returns:
+        str -- _description_
+    """
+    if exc_stack is not None:
+        _, _, exc_traceback = exc_stack
     else:
-        stack = traceback.extract_stack()[-start:-depth]
-    return ' -> '.join(
-        f"{os.path.basename(f.filename)}:{f.name}:{f.lineno}"
-        for f in stack
-    )
+        exc_traceback = None
+    _, _, exc_traceback = sys.exc_info()
+    stack = traceback.extract_tb(exc_traceback)
+    if not separator:
+        separator = ' -> '
+    # print(f"* HERE: {dump_data(stack)}")
+    return f"{separator}".join(f"{os.path.basename(f.filename)}:{f.name}:{f.lineno}" for f in stack)
 
 # __END__

src/corelibs/debug_handling/dump_data.py

@@ -6,7 +6,7 @@ import json
 from typing import Any
 
 
-def dump_data(data: dict[Any, Any] | list[Any] | str | None) -> str:
+def dump_data(data: Any) -> str:
     """
     dump formated output from dict/list
 

src/corelibs/exceptions/csv_exceptions.py

@@ -0,0 +1,23 @@
+"""
+Exceptions for csv file reading and processing
+"""
+
+
+class NoCsvReader(Exception):
+    """
+    CSV reader is none
+    """
+
+
+class CsvHeaderDataMissing(Exception):
+    """
+    The csv reader returned None as headers, the header column in the csv file is missing
+    """
+
+
+class CompulsoryCsvHeaderCheckFailed(Exception):
+    """
+    raise if the header is not matching to the excpeted values
+    """
+
+# __END__

src/corelibs/iterator_handling/data_search.py

@@ -2,23 +2,40 @@
 wrapper around search path
 """
 
-from typing import Any
+from typing import Any, TypedDict, NotRequired
+from warnings import deprecated
 
 
+class ArraySearchList(TypedDict):
+    """find in array from list search dict"""
+    key: str
+    value: str | bool | int | float | list[str | None]
+    case_sensitive: NotRequired[bool]
+
+
+@deprecated("Use find_in_array_from_list()")
 def array_search(
-    search_params: list[dict[str, str | bool | list[str | None]]],
+    search_params: list[ArraySearchList],
     data: list[dict[str, Any]],
     return_index: bool = False
+) -> list[dict[str, Any]]:
+    """depreacted, old call order"""
+    return find_in_array_from_list(data, search_params, return_index)
+
+def find_in_array_from_list(
+    data: list[dict[str, Any]],
+    search_params: list[ArraySearchList],
+    return_index: bool = False
 ) -> list[dict[str, Any]]:
     """
-    search in an array of dicts with an array of Key/Value set
+    search in an list of dicts with an list of Key/Value set
     all Key/Value sets must match
     Value set can be list for OR match
     option: case_senstive: default True
 
     Args:
-        search_params (list): List of search params in "Key"/"Value" lists with options
         data (list): data to search in, must be a list
+        search_params (list): List of search params in "key"/"value" lists with options
         return_index (bool): return index of list [default False]
 
     Raises:
@@ -32,18 +49,20 @@ def array_search(
     """
     if not isinstance(search_params, list):  # type: ignore
         raise ValueError("search_params must be a list")
-    keys = []
+    keys: list[str] = []
+    # check that key and value exist and are set
     for search in search_params:
-        if not search.get('Key') or not search.get('Value'):
+        if not search.get('key') or not search.get('value'):
             raise KeyError(
-                f"Either Key '{search.get('Key', '')}' or "
-                f"Value '{search.get('Value', '')}' is missing or empty"
+                f"Either Key '{search.get('key', '')}' or "
+                f"Value '{search.get('value', '')}' is missing or empty"
             )
         # if double key -> abort
-        if search.get("Key") in keys:
+        if search.get("key") in keys:
             raise KeyError(
-                f"Key {search.get('Key', '')} already exists in search_params"
+                f"Key {search.get('key', '')} already exists in search_params"
             )
+        keys.append(str(search['key']))
 
     return_items: list[dict[str, Any]] = []
     for si_idx, search_item in enumerate(data):
@@ -55,20 +74,20 @@ def array_search(
             # lower case left side
             # TODO: allow nested Keys. eg "Key: ["Key a", "key b"]" to be ["Key a"]["key b"]
             if search.get("case_sensitive", True) is False:
-                search_value = search_item.get(str(search['Key']), "").lower()
+                search_value = search_item.get(str(search['key']), "").lower()
             else:
-                search_value = search_item.get(str(search['Key']), "")
+                search_value = search_item.get(str(search['key']), "")
             # lower case right side
-            if isinstance(search['Value'], list):
+            if isinstance(search['value'], list):
                 search_in = [
-                        str(k).lower()
-                        if search.get("case_sensitive", True) is False else k
-                        for k in search['Value']
-                    ]
+                    str(k).lower()
+                    if search.get("case_sensitive", True) is False else k
+                    for k in search['value']
+                ]
             elif search.get("case_sensitive", True) is False:
-                search_in = str(search['Value']).lower()
+                search_in = str(search['value']).lower()
             else:
-                search_in = search['Value']
+                search_in = search['value']
             # compare check
             if (
                 (

src/corelibs/iterator_handling/dict_helpers.py

@@ -3,26 +3,36 @@ Dict helpers
 """
 
 
-from typing import Any
+from typing import TypeAlias, Union, Dict, List, Any, cast
+
+# definitions for the mask run below
+MaskableValue: TypeAlias = Union[str, int, float, bool, None]
+NestedDict: TypeAlias = Dict[str, Union[MaskableValue, List[Any], 'NestedDict']]
+ProcessableValue: TypeAlias = Union[MaskableValue, List[Any], NestedDict]
 
 
 def mask(
-    data_set: dict[str, str],
+    data_set: dict[str, Any],
     mask_keys: list[str] | None = None,
     mask_str: str = "***",
+    mask_str_edges: str = '_',
     skip: bool = False
-) -> dict[str, str]:
+) -> dict[str, Any]:
     """
     mask data for output
     Checks if mask_keys list exist in any key in the data set either from the start or at the end
 
+    Use the mask_str_edges to define how searches inside a string should work. Default it must start
+    and end with '_', remove to search string in string
+
     Arguments:
         data_set {dict[str, str]} -- _description_
 
     Keyword Arguments:
         mask_keys {list[str] | None} -- _description_ (default: {None})
         mask_str {str} -- _description_ (default: {"***"})
-        skip {bool} -- _description_ (default: {False})
+        mask_str_edges {str} -- _description_ (default: {"_"})
+        skip {bool} -- if set to true skip (default: {False})
 
     Returns:
         dict[str, str] -- _description_
@@ -30,29 +40,46 @@ def mask(
     if skip is True:
         return data_set
     if mask_keys is None:
-        mask_keys = ["password", "secret"]
+        mask_keys = ["encryption", "password", "secret"]
+    else:
+        # make sure it is lower case
+        mask_keys = [mask_key.lower() for mask_key in mask_keys]
+
+    def should_mask_key(key: str) -> bool:
+        """Check if a key should be masked"""
+        __key_lower = key.lower()
+        return any(
+            __key_lower.startswith(mask_key) or
+            __key_lower.endswith(mask_key) or
+            f"{mask_str_edges}{mask_key}{mask_str_edges}" in __key_lower
+            for mask_key in mask_keys
+        )
+
+    def mask_recursive(obj: ProcessableValue) -> ProcessableValue:
+        """Recursively mask values in nested structures"""
+        if isinstance(obj, dict):
+            return {
+                key: mask_value(value) if should_mask_key(key) else mask_recursive(value)
+                for key, value in obj.items()
+            }
+        if isinstance(obj, list):
+            return [mask_recursive(item) for item in obj]
+        return obj
+
+    def mask_value(value: Any) -> Any:
+        """Handle masking based on value type"""
+        if isinstance(value, list):
+            # Mask each individual value in the list
+            return [mask_str for _ in cast('list[Any]', value)]
+        if isinstance(value, dict):
+            # Recursively process the dictionary instead of masking the whole thing
+            return mask_recursive(cast('ProcessableValue', value))
+        # Mask primitive values
+        return mask_str
+
     return {
-        key: mask_str
-        if any(key.startswith(mask_key) or key.endswith(mask_key) for mask_key in mask_keys) else value
+        key: mask_value(value) if should_mask_key(key) else mask_recursive(value)
         for key, value in data_set.items()
     }
 
-
-def set_entry(dict_set: dict[str, Any], key: str, value_set: Any) -> dict[str, Any]:
-    """
-    set a new entry in the dict set
-
-    Arguments:
-        key {str} -- _description_
-        dict_set {dict[str, Any]} -- _description_
-        value_set {Any} -- _description_
-
-    Returns:
-        dict[str, Any] -- _description_
-    """
-    if not dict_set.get(key):
-        dict_set[key] = {}
-    dict_set[key] = value_set
-    return dict_set
-
 # __END__

src/corelibs/json_handling/jmespath_helper.py

@@ -32,4 +32,6 @@ def jmespath_search(search_data: dict[Any, Any] | list[Any], search_params: str)
         raise ValueError(f"Type error for search_params: {excp}") from excp
     return search_result
 
+# TODO: compile jmespath setup
+
 # __END__

src/corelibs/logging_handling/log.py

@@ -7,12 +7,14 @@ attach "init_worker_logging" with the set log_queue
 import re
 import logging.handlers
 import logging
+from datetime import datetime
 import time
 from pathlib import Path
+import atexit
 from typing import MutableMapping, TextIO, TypedDict, Any, TYPE_CHECKING, cast
 from corelibs.logging_handling.logging_level_handling.logging_level import LoggingLevel
 from corelibs.string_handling.text_colors import Colors
-from corelibs.debug_handling.debug_helpers import traceback_call_str
+from corelibs.debug_handling.debug_helpers import call_stack, exception_stack
 
 if TYPE_CHECKING:
     from multiprocessing import Queue
@@ -20,14 +22,10 @@ if TYPE_CHECKING:
 
 # MARK: Log settings TypedDict
 class LogSettings(TypedDict):
-    """
-    log settings
-
-    Arguments:
-        TypedDict {_type_} -- _description_
-    """
+    """log settings, for Log setup"""
     log_level_console: LoggingLevel
     log_level_file: LoggingLevel
+    per_run_log: bool
     console_enabled: bool
     console_color_output_enabled: bool
     add_start_info: bool
@@ -35,6 +33,12 @@ class LogSettings(TypedDict):
     log_queue: 'Queue[str] | None'
 
 
+class LoggerInit(TypedDict):
+    """for Logger init"""
+    logger: logging.Logger
+    log_queue: 'Queue[str] | None'
+
+
 # MARK: Custom color filter
 class CustomConsoleFormatter(logging.Formatter):
     """
@@ -73,13 +77,321 @@ class CustomConsoleFormatter(logging.Formatter):
         message = super().format(record)
         return f"{color}{message}{reset}"
 
-
-# TODO: add custom handlers for stack_correct, if not set fill with %(filename)s:%(funcName)s:%(lineno)d
-# hasattr(record, 'stack_correct')
+# TODO: add custom handlers for stack_trace, if not set fill with %(filename)s:%(funcName)s:%(lineno)d
+# hasattr(record, 'stack_trace')
+# also for something like "context" where we add an array of anything to a message
 
 
-# MARK: Log class
-class Log:
+class CustomHandlerFilter(logging.Filter):
+    """
+    Add a custom handler for filtering
+    """
+    HANDLER_NAME_FILTER_EXCEPTION: str = 'console'
+
+    def __init__(self, handler_name: str, filter_exceptions: bool = False):
+        super().__init__(name=handler_name)
+        self.handler_name = handler_name
+        self.filter_exceptions = filter_exceptions
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        # if console and exception do not show
+        if self.handler_name == self.HANDLER_NAME_FILTER_EXCEPTION and self.filter_exceptions:
+            return record.levelname != "EXCEPTION"
+        # if cnosole entry is true and traget file filter
+        if hasattr(record, 'console') and getattr(record, 'console') is True and self.handler_name == 'file':
+            return False
+        return True
+
+    # def __filter_exceptions(self, record: logging.LogRecord) -> bool:
+    #     return record.levelname != "EXCEPTION"
+
+
+# MARK: Parent class
+class LogParent:
+    """
+    Parent class with general methods
+    used by Log and Logger
+    """
+
+    # spacer lenght characters and the character
+    SPACER_CHAR: str = '='
+    SPACER_LENGTH: int = 32
+
+    def __init__(self):
+        self.logger: logging.Logger
+        self.log_queue: 'Queue[str] | None' = None
+        self.handlers: dict[str, Any] = {}
+
+    def __del__(self):
+        self._cleanup()
+
+    # FIXME: we need to add a custom formater to add stack level listing if we want to
+    # Important note, although they exist, it is recommended to use self.logger.NAME directly
+    # so that the correct filename, method and row number is set
+    # for > 50 use logger.log(LoggingLevel.<LEVEL>.value, ...)
+    # for exception logger.log(LoggingLevel.EXCEPTION.value, ..., execInfo=True)
+    # MARK: log message
+    def log(self, level: int, msg: object, *args: object, extra: MutableMapping[str, object] | None = None):
+        """log general"""
+        if not hasattr(self, 'logger'):
+            raise ValueError('Logger is not yet initialized')
+        if extra is None:
+            extra = {}
+        extra['stack_trace'] = call_stack(skip_last=2)
+        self.logger.log(level, msg, *args, extra=extra, stacklevel=2)
+
+    # MARK: DEBUG 10
+    def debug(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
+        """debug"""
+        if not hasattr(self, 'logger'):
+            raise ValueError('Logger is not yet initialized')
+        if extra is None:
+            extra = {}
+        extra['stack_trace'] = call_stack(skip_last=2)
+        self.logger.debug(msg, *args, extra=extra, stacklevel=2)
+
+    # MARK: INFO 20
+    def info(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
+        """info"""
+        if not hasattr(self, 'logger'):
+            raise ValueError('Logger is not yet initialized')
+        if extra is None:
+            extra = {}
+        extra['stack_trace'] = call_stack(skip_last=2)
+        self.logger.info(msg, *args, extra=extra, stacklevel=2)
+
+    # MARK: WARNING 30
+    def warning(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
+        """warning"""
+        if not hasattr(self, 'logger'):
+            raise ValueError('Logger is not yet initialized')
+        if extra is None:
+            extra = {}
+        extra['stack_trace'] = call_stack(skip_last=2)
+        self.logger.warning(msg, *args, extra=extra, stacklevel=2)
+
+    # MARK: ERROR 40
+    def error(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
+        """error"""
+        if not hasattr(self, 'logger'):
+            raise ValueError('Logger is not yet initialized')
+        if extra is None:
+            extra = {}
+        extra['stack_trace'] = call_stack(skip_last=2)
+        self.logger.error(msg, *args, extra=extra, stacklevel=2)
+
+    # MARK: CRITICAL 50
+    def critical(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
+        """critcal"""
+        if not hasattr(self, 'logger'):
+            raise ValueError('Logger is not yet initialized')
+        if extra is None:
+            extra = {}
+        extra['stack_trace'] = call_stack(skip_last=2)
+        self.logger.critical(msg, *args, extra=extra, stacklevel=2)
+
+    # MARK: ALERT 55
+    def alert(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
+        """alert"""
+        if not hasattr(self, 'logger'):
+            raise ValueError('Logger is not yet initialized')
+        # extra_dict = dict(extra)
+        if extra is None:
+            extra = {}
+        extra['stack_trace'] = call_stack(skip_last=2)
+        self.logger.log(LoggingLevel.ALERT.value, msg, *args, extra=extra, stacklevel=2)
+
+    # MARK: EMERGECNY: 60
+    def emergency(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
+        """emergency"""
+        if not hasattr(self, 'logger'):
+            raise ValueError('Logger is not yet initialized')
+        if extra is None:
+            extra = {}
+        extra['stack_trace'] = call_stack(skip_last=2)
+        self.logger.log(LoggingLevel.EMERGENCY.value, msg, *args, extra=extra, stacklevel=2)
+
+    # MARK: EXCEPTION: 70
+    def exception(
+        self,
+        msg: object, *args: object, extra: MutableMapping[str, object] | None = None,
+        log_error: bool = True
+    ) -> None:
+        """
+        log on exceotion level, this is log.exception, but logs with a new level
+
+        Args:
+            msg (object): _description_
+            *args (object): arguments for msg
+            extra: Mapping[str, object] | None: extra arguments for the formatting if needed
+            log_error: (bool): If set to false will not write additional error message for console (Default True)
+        """
+        if not hasattr(self, 'logger'):
+            raise ValueError('Logger is not yet initialized')
+        if extra is None:
+            extra = {}
+        extra['stack_trace'] = call_stack(skip_last=2)
+        extra['exception_trace'] = exception_stack()
+        # write to console first with extra flag for filtering in file
+        if log_error:
+            self.logger.log(
+                LoggingLevel.ERROR.value,
+                f"<=EXCEPTION={extra['exception_trace']}> {msg} [{extra['stack_trace']}]",
+                *args, extra=dict(extra) | {'console': True}, stacklevel=2
+            )
+        self.logger.log(LoggingLevel.EXCEPTION.value, msg, *args, exc_info=True, extra=extra, stacklevel=2)
+
+    def break_line(self, info: str = "BREAK"):
+        """
+        add a break line as info level
+
+        Keyword Arguments:
+            info {str} -- _description_ (default: {"BREAK"})
+        """
+        if not hasattr(self, 'logger'):
+            raise ValueError('Logger is not yet initialized')
+        self.logger.info("[%s] %s>", info, self.SPACER_CHAR * self.SPACER_LENGTH)
+
+    # MARK: queue handling
+    def flush(self, handler_name: str | None = None, timeout: float = 2.0) -> bool:
+        """
+        Flush all pending messages
+
+        Keyword Arguments:
+            handler_name {str | None} -- _description_ (default: {None})
+            timeout {float} -- _description_ (default: {2.0})
+
+        Returns:
+            bool -- _description_
+        """
+        if not self.log_queue:
+            return False
+
+        try:
+            # Wait for queue to be processed
+            start_time = time.time()
+            while not self.log_queue.empty() and (time.time() - start_time) < timeout:
+                time.sleep(0.01)
+
+            # Flush all handlers or handler given
+            if handler_name:
+                try:
+                    self.handlers[handler_name].flush()
+                except IndexError:
+                    pass
+            else:
+                for handler in self.handlers.values():
+                    handler.flush()
+        except OSError:
+            return False
+        return True
+
+    def _cleanup(self):
+        """cleanup for any open queues in case we have an abort"""
+        if not self.log_queue:
+            return
+        self.flush()
+        # Close the queue properly
+        self.log_queue.close()
+        self.log_queue.join_thread()
+
+    # MARK: log level handling
+    def set_log_level(self, handler_name: str, log_level: LoggingLevel) -> bool:
+        """
+        set the logging level for a handler
+
+        Arguments:
+            handler {str} -- _description_
+            log_level {LoggingLevel} -- _description_
+
+        Returns:
+            bool -- _description_
+        """
+        try:
+            # flush queue befoe changing logging level
+            self.flush(handler_name)
+            self.handlers[handler_name].setLevel(log_level.name)
+            return True
+        except IndexError:
+            if self.logger:
+                self.logger.error('Handler %s not found, cannot change log level', handler_name)
+            return False
+        except AttributeError:
+            if self.logger:
+                self.logger.error(
+                    'Cannot change to log level %s for handler %s, log level invalid',
+                    LoggingLevel.name, handler_name
+                )
+            return False
+
+    def get_log_level(self, handler_name: str) -> LoggingLevel:
+        """
+        gettthe logging level for a handler
+
+        Arguments:
+            handler_name {str} -- _description_
+
+        Returns:
+            LoggingLevel -- _description_
+        """
+        try:
+            return LoggingLevel.from_any(self.handlers[handler_name].level)
+        except IndexError:
+            return LoggingLevel.NOTSET
+
+    @staticmethod
+    def validate_log_level(log_level: Any) -> bool:
+        """
+        if the log level is invalid will return false, else return true
+
+        Args:
+            log_level (Any): _description_
+
+        Returns:
+            bool: _description_
+        """
+        try:
+            _ = LoggingLevel.from_any(log_level).value
+            return True
+        except ValueError:
+            return False
+
+    @staticmethod
+    def get_log_level_int(log_level: Any) -> int:
+        """
+        Return log level as INT
+        If invalid returns the default log level
+
+        Arguments:
+            log_level {Any} -- _description_
+
+        Returns:
+            int -- _description_
+        """
+        try:
+            return LoggingLevel.from_any(log_level).value
+        except ValueError:
+            return LoggingLevel.from_string(Log.DEFAULT_LOG_LEVEL.name).value
+
+
+# MARK: Logger
+class Logger(LogParent):
+    """
+    The class we can pass on to other clases without re-init the class itself
+    NOTE: if no queue object is handled over the logging level change might not take immediate effect
+    """
+
+    def __init__(self, logger_settings: LoggerInit):
+        LogParent.__init__(self)
+        self.logger = logger_settings['logger']
+        self.lg = self.logger
+        self.l = self.logger
+        self.handlers = {str(_handler.name): _handler for _handler in self.logger.handlers}
+        self.log_queue = logger_settings['log_queue']
+
+
+# MARK: LogSetup class
+class Log(LogParent):
     """
     logger setup
     """
@@ -93,8 +405,9 @@ class Log:
     DEFAULT_LOG_LEVEL_CONSOLE: LoggingLevel = LoggingLevel.WARNING
     # default settings
     DEFAULT_LOG_SETTINGS: LogSettings = {
-        "log_level_console": LoggingLevel.WARNING,
-        "log_level_file": LoggingLevel.DEBUG,
+        "log_level_console": DEFAULT_LOG_LEVEL_CONSOLE,
+        "log_level_file": DEFAULT_LOG_LEVEL_FILE,
+        "per_run_log": False,
         "console_enabled": True,
         "console_color_output_enabled": True,
         "add_start_info": True,
@@ -110,6 +423,7 @@ class Log:
         log_settings: dict[str, 'LoggingLevel | str | bool | None | Queue[str]'] | LogSettings | None = None,
         other_handlers: dict[str, Any] | None = None
     ):
+        LogParent.__init__(self)
         # add new level for alert, emergecny and exception
         logging.addLevelName(LoggingLevel.ALERT.value, LoggingLevel.ALERT.name)
         logging.addLevelName(LoggingLevel.EMERGENCY.value, LoggingLevel.EMERGENCY.name)
@@ -142,13 +456,13 @@ class Log:
         # in the file writer too, for the ones where color is set BEFORE the format
         # Any is logging.StreamHandler, logging.FileHandler and all logging.handlers.*
         self.handlers: dict[str, Any] = {}
-        self.add_handler('file_handler', self.__create_time_rotating_file_handler(
-            self.log_settings['log_level_file'], log_path)
+        self.add_handler('file_handler', self.__create_file_handler(
+            'file_handler', self.log_settings['log_level_file'], log_path)
         )
         if self.log_settings['console_enabled']:
             # console
             self.add_handler('stream_handler', self.__create_console_handler(
-                self.log_settings['log_level_console'])
+                'stream_handler', self.log_settings['log_level_console'])
             )
         # add other handlers,
         if other_handlers is not None:
@@ -194,6 +508,7 @@ class Log:
             default_log_settings[__log_entry] = LoggingLevel.from_any(__log_level)
         # check bool
         for __log_entry in [
+            "per_run_log",
             "console_enabled",
             "console_color_output_enabled",
             "add_start_info",
@@ -211,8 +526,8 @@ class Log:
         default_log_settings['log_queue'] = __setting
         return default_log_settings
 
-    def __filter_exceptions(self, record: logging.LogRecord) -> bool:
-        return record.levelname != "EXCEPTION"
+    # def __filter_exceptions(self, record: logging.LogRecord) -> bool:
+    #     return record.levelname != "EXCEPTION"
 
     # MARK: add a handler
     def add_handler(
@@ -239,7 +554,8 @@ class Log:
 
     # MARK: console handler
     def __create_console_handler(
-        self, log_level_console: LoggingLevel = LoggingLevel.WARNING, filter_exceptions: bool = True
+        self, handler_name: str,
+        log_level_console: LoggingLevel = LoggingLevel.WARNING, filter_exceptions: bool = True
     ) -> logging.StreamHandler[TextIO]:
         # console logger
         if not self.validate_log_level(log_level_console):
@@ -259,32 +575,43 @@ class Log:
             formatter_console = CustomConsoleFormatter(format_string, datefmt=format_date)
         else:
             formatter_console = logging.Formatter(format_string, datefmt=format_date)
+        console_handler.set_name(handler_name)
         console_handler.setLevel(log_level_console.name)
-        console_handler.set_name('console')
         # do not show exceptions logs on console
-        if filter_exceptions:
-            console_handler.addFilter(self.__filter_exceptions)
+        console_handler.addFilter(CustomHandlerFilter('console', filter_exceptions))
         console_handler.setFormatter(formatter_console)
         return console_handler
 
     # MARK: file handler
-    def __create_time_rotating_file_handler(
-        self, log_level_file: LoggingLevel, log_path: Path,
+    def __create_file_handler(
+        self, handler_name: str,
+        log_level_file: LoggingLevel, log_path: Path,
+        # for TimedRotating, if per_run_log is off
         when: str = "D", interval: int = 1, backup_count: int = 0
-    ) -> logging.handlers.TimedRotatingFileHandler:
+    ) -> logging.handlers.TimedRotatingFileHandler | logging.FileHandler:
         # file logger
         # when: S/M/H/D/W0-W6/midnight
         # interval: how many, 1D = every day
         # backup_count: how many old to keep, 0 = all
         if not self.validate_log_level(log_level_file):
             log_level_file = self.DEFAULT_LOG_LEVEL_FILE
-        file_handler = logging.handlers.TimedRotatingFileHandler(
-            filename=log_path,
-            encoding="utf-8",
-            when=when,
-            interval=interval,
-            backupCount=backup_count
-        )
+        if self.log_settings['per_run_log']:
+            # log path, remove them stem (".log"), then add the datetime and add .log again
+            now = datetime.now()
+            # we add microseconds part to get milli seconds
+            new_stem=f"{log_path.stem}.{now.strftime('%Y-%m-%d_%H-%M-%S')}.{str(now.microsecond)[:3]}"
+            file_handler = logging.FileHandler(
+                filename=log_path.with_name(f"{new_stem}{log_path.suffix}"),
+                encoding="utf-8",
+            )
+        else:
+            file_handler = logging.handlers.TimedRotatingFileHandler(
+                filename=log_path,
+                encoding="utf-8",
+                when=when,
+                interval=interval,
+                backupCount=backup_count
+            )
         formatter_file_handler = logging.Formatter(
             (
                 # time stamp
@@ -302,8 +629,10 @@ class Log:
             ),
             datefmt="%Y-%m-%dT%H:%M:%S",
         )
-        file_handler.set_name('file_timed_rotate')
+        file_handler.set_name(handler_name)
         file_handler.setLevel(log_level_file.name)
+        # do not show errors flagged with console (they are from exceptions)
+        file_handler.addFilter(CustomHandlerFilter('file'))
         file_handler.setFormatter(formatter_file_handler)
         return file_handler
 
@@ -318,6 +647,7 @@ class Log:
         if log_queue is None:
             return
         self.log_queue = log_queue
+        atexit.register(self._cleanup)
         self.listener = logging.handlers.QueueListener(
             self.log_queue,
             *self.handlers.values(),
@@ -325,6 +655,14 @@ class Log:
         )
         self.listener.start()
 
+    def stop_listener(self):
+        """
+        stop the listener
+        """
+        if self.listener is not None:
+            self.flush()
+            self.listener.stop()
+
     # MARK: init main log
     def __init_log(self, log_name: str) -> None:
         """
@@ -367,239 +705,16 @@ class Log:
 
         return root_logger
 
-    # FIXME: all below will only work if we add a custom format interface for the stack_correct part
-    # Important note, although they exist, it is recommended to use self.logger.NAME directly
-    # so that the correct filename, method and row number is set
-    # for > 50 use logger.log(LoggingLevel.<LEVEL>.value, ...)
-    # for exception logger.log(LoggingLevel.EXCEPTION.value, ..., execInfo=True)
-    # MARK: log message
-    def log(self, level: int, msg: object, *args: object, extra: MutableMapping[str, object] | None = None):
-        """log general"""
-        if not hasattr(self, 'logger'):
-            raise ValueError('Logger is not yet initialized')
-        if extra is None:
-            extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
-        self.logger.log(level, msg, *args, extra=extra)
-
-    # MARK: DEBUG 10
-    def debug(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
-        """debug"""
-        if not hasattr(self, 'logger'):
-            raise ValueError('Logger is not yet initialized')
-        if extra is None:
-            extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
-        self.logger.debug(msg, *args, extra=extra)
-
-    # MARK: INFO 20
-    def info(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
-        """info"""
-        if not hasattr(self, 'logger'):
-            raise ValueError('Logger is not yet initialized')
-        if extra is None:
-            extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
-        self.logger.info(msg, *args, extra=extra)
-
-    # MARK: WARNING 30
-    def warning(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
-        """warning"""
-        if not hasattr(self, 'logger'):
-            raise ValueError('Logger is not yet initialized')
-        if extra is None:
-            extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
-        self.logger.warning(msg, *args, extra=extra)
-
-    # MARK: ERROR 40
-    def error(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
-        """error"""
-        if not hasattr(self, 'logger'):
-            raise ValueError('Logger is not yet initialized')
-        if extra is None:
-            extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
-        self.logger.error(msg, *args, extra=extra)
-
-    # MARK: CRITICAL 50
-    def critical(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
-        """critcal"""
-        if not hasattr(self, 'logger'):
-            raise ValueError('Logger is not yet initialized')
-        if extra is None:
-            extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
-        self.logger.critical(msg, *args, extra=extra)
-
-    # MARK: ALERT 55
-    def alert(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
-        """alert"""
-        if not hasattr(self, 'logger'):
-            raise ValueError('Logger is not yet initialized')
-        # extra_dict = dict(extra)
-        if extra is None:
-            extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
-        self.logger.log(LoggingLevel.ALERT.value, msg, *args, extra=extra)
-
-    # MARK: EMERGECNY: 60
-    def emergency(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
-        """emergency"""
-        if not hasattr(self, 'logger'):
-            raise ValueError('Logger is not yet initialized')
-        if extra is None:
-            extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
-        self.logger.log(LoggingLevel.EMERGENCY.value, msg, *args, extra=extra)
-
-    # MARK: EXCEPTION: 70
-    def exception(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
+    def get_logger_settings(self) -> LoggerInit:
         """
-        log on exceotion level, this is log.exception, but logs with a new level
-
-        Args:
-            msg (object): _description_
-            *args (object): arguments for msg
-            extra: Mapping[str, object] | None: extra arguments for the formatting if needed
-        """
-        if not hasattr(self, 'logger'):
-            raise ValueError('Logger is not yet initialized')
-        if extra is None:
-            extra = {}
-        extra['stack_correct'] = traceback_call_str(start=3)
-        self.logger.log(LoggingLevel.EXCEPTION.value, msg, *args, exc_info=True, extra=extra)
-
-    # MARK: break line
-    def break_line(self, info: str = "BREAK"):
-        """
-        add a break line as info level
-
-        Keyword Arguments:
-            info {str} -- _description_ (default: {"BREAK"})
-        """
-        if not hasattr(self, 'logger'):
-            raise ValueError('Logger is not yet initialized')
-        self.logger.info("[%s] %s>", info, self.SPACER_CHAR * self.SPACER_LENGTH)
-
-    # MARK: queue handling
-    def flush(self, handler_name: str | None = None, timeout: float = 2.0) -> bool:
-        """
-        Flush all pending messages
-
-        Keyword Arguments:
-            handler_name {str | None} -- _description_ (default: {None})
-            timeout {float} -- _description_ (default: {2.0})
+        get the logger settings we need to init the Logger class
 
         Returns:
-            bool -- _description_
+            LoggerInit -- _description_
         """
-        if not self.listener or not self.log_queue:
-            return False
-
-        try:
-            # Wait for queue to be processed
-            start_time = time.time()
-            while not self.log_queue.empty() and (time.time() - start_time) < timeout:
-                time.sleep(0.01)
-
-            # Flush all handlers or handler given
-            if handler_name:
-                try:
-                    self.handlers[handler_name].flush()
-                except IndexError:
-                    pass
-            else:
-                for handler in self.handlers.values():
-                    handler.flush()
-        except OSError:
-            return False
-        return True
-
-    def stop_listener(self):
-        """
-        stop the listener
-        """
-        if self.listener is not None:
-            self.flush()
-            self.listener.stop()
-
-    # MARK: log level handling
-    def set_log_level(self, handler_name: str, log_level: LoggingLevel) -> bool:
-        """
-        set the logging level for a handler
-
-        Arguments:
-            handler {str} -- _description_
-            log_level {LoggingLevel} -- _description_
-
-        Returns:
-            bool -- _description_
-        """
-        try:
-            # flush queue befoe changing logging level
-            self.flush(handler_name)
-            self.handlers[handler_name].setLevel(log_level.name)
-            return True
-        except IndexError:
-            if self.logger:
-                self.logger.error('Handler %s not found, cannot change log level', handler_name)
-            return False
-        except AttributeError:
-            if self.logger:
-                self.logger.error(
-                    'Cannot change to log level %s for handler %s, log level invalid',
-                    LoggingLevel.name, handler_name
-                )
-            return False
-
-    def get_log_level(self, handler_name: str) -> LoggingLevel:
-        """
-        gettthe logging level for a handler
-
-        Arguments:
-            handler_name {str} -- _description_
-
-        Returns:
-            LoggingLevel -- _description_
-        """
-        try:
-            return self.handlers[handler_name]
-        except IndexError:
-            return LoggingLevel.NOTSET
-
-    @staticmethod
-    def validate_log_level(log_level: Any) -> bool:
-        """
-        if the log level is invalid will return false, else return true
-
-        Args:
-            log_level (Any): _description_
-
-        Returns:
-            bool: _description_
-        """
-        try:
-            _ = LoggingLevel.from_any(log_level).value
-            return True
-        except ValueError:
-            return False
-
-    @staticmethod
-    def get_log_level_int(log_level: Any) -> int:
-        """
-        Return log level as INT
-        If invalid returns the default log level
-
-        Arguments:
-            log_level {Any} -- _description_
-
-        Returns:
-            int -- _description_
-        """
-        try:
-            return LoggingLevel.from_any(log_level).value
-        except ValueError:
-            return LoggingLevel.from_string(Log.DEFAULT_LOG_LEVEL.name).value
+        return {
+            "logger": self.logger,
+            "log_queue": self.log_queue
+        }
 
 # __END__

src/corelibs/requests_handling/auth_helpers.py

@@ -0,0 +1,20 @@
+"""
+Various HTTP auth helpers
+"""
+
+from base64 import b64encode
+
+
+def basic_auth(username: str, password: str) -> str:
+    """
+    setup basic auth, for debug
+
+    Arguments:
+        username {str} -- _description_
+        password {str} -- _description_
+
+    Returns:
+        str -- _description_
+    """
+    token = b64encode(f"{username}:{password}".encode('utf-8')).decode("ascii")
+    return f'Basic {token}'

src/corelibs/string_handling/timestamp_strings.py

@@ -2,8 +2,18 @@
 Current timestamp strings and time zones
 """
 
+import re
 from datetime import datetime
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+from corelibs.var_handling.var_helpers import is_float
+
+
+class TimeParseError(Exception):
+    """Custom exception for time parsing errors."""
+
+
+class TimeUnitError(Exception):
+    """Custom exception for time parsing errors."""
 
 
 class TimestampStrings:
@@ -24,3 +34,79 @@ class TimestampStrings:
         self.timestamp = self.timestamp_now.strftime("%Y-%m-%d %H:%M:%S")
         self.timestamp_tz = self.timestamp_now_tz.strftime("%Y-%m-%d %H:%M:%S %Z")
         self.timestamp_file = self.timestamp_now.strftime("%Y-%m-%d_%H%M%S")
+
+
+def convert_to_seconds(time_string: str | int | float) -> int:
+    """
+    Conver a string with time units into a seconds string
+    The following units are allowed
+    Y: 365 days
+    M: 30 days
+    d, h, m, s
+
+    Arguments:
+        time_string {str} -- _description_
+
+    Raises:
+        ValueError: _description_
+
+    Returns:
+        int -- _description_
+    """
+
+    # skip out if this is a number of any type
+    # numbers will br made float, rounded and then converted to int
+    if is_float(time_string):
+        return int(round(float(time_string)))
+    time_string = str(time_string)
+
+    # Define time unit conversion factors
+    unit_factors: dict[str, int] = {
+        'Y': 31536000,       # 365 days * 86400 seconds/day
+        'M': 2592000 * 12,   # 1 year in seconds (assuming 365 days per year)
+        'd': 86400,          # 1 day in seconds
+        'h': 3600,           # 1 hour in seconds
+        'm': 60,             # minutes to seconds
+        's': 1               # 1 second in seconds
+    }
+    long_unit_names: dict[str, str] = {
+        'year': 'Y',
+        'years': 'Y',
+        'month': 'M',
+        'months': 'M',
+        'day': 'd',
+        'days': 'd',
+        'hour': 'h',
+        'hours': 'h',
+        'minute': 'm',
+        'minutes': 'm',
+        'min': 'm',
+        'second': 's',
+        'seconds': 's',
+        'sec': 's',
+    }
+
+    total_seconds = 0
+
+    seen_units: list[str] = []  # Track units that have been encountered
+
+    # Use regex to match number and time unit pairs
+    for match in re.finditer(r'(\d+)\s*([a-zA-Z]+)', time_string):
+        value, unit = int(match.group(1)), match.group(2)
+
+        # full name check, fallback to original name
+        unit = long_unit_names.get(unit.lower(), unit)
+
+        # Check for duplicate units
+        if unit in seen_units:
+            raise TimeParseError(f"Unit '{unit}' appears more than once.")
+        # Check invalid unit
+        if unit not in unit_factors:
+            raise TimeUnitError(f"Unit '{unit}' is not a valid unit name.")
+        # Add to total seconds based on the units
+        if unit in unit_factors:
+            total_seconds += value * unit_factors[unit]
+
+        seen_units.append(unit)
+
+    return total_seconds

test-run/config_handling/config/settings.ini

@@ -9,6 +9,8 @@ other_list=a|b|c|d|
 third_list=xy|ab|df|fg
 str_length=foobar
 int_range=20
+int_range_not_set=
+int_range_not_set_empty_set=5
 #
 match_target=foo
 match_target_list=foo,bar,baz

test-run/config_handling/settings_loader.py

@@ -2,10 +2,12 @@
 Settings loader test
 """
 
+import re
 from pathlib import Path
-from corelibs.iterator_handling.dump_data import dump_data
+from corelibs.debug_handling.dump_data import dump_data
 from corelibs.logging_handling.log import Log
 from corelibs.config_handling.settings_loader import SettingsLoader
+from corelibs.config_handling.settings_loader_handling.settings_loader_check import SettingsLoaderCheck
 
 SCRIPT_PATH: Path = Path(__file__).resolve().parent
 ROOT_PATH: Path = SCRIPT_PATH
@@ -18,6 +20,11 @@ def main():
     Main run
     """
 
+    value = "2025/1/1"
+    regex_c = re.compile(SettingsLoaderCheck.CHECK_SETTINGS['string.date']['regex'], re.VERBOSE)
+    result = regex_c.search(value)
+    print(f"regex {regex_c} check against {value} -> {result}")
+
     # for log testing
     script_path: Path = Path(__file__).resolve().parent
     log = Log(
@@ -63,6 +70,12 @@ def main():
                 "int_range": [
                     "range:2-50"
                 ],
+                "int_range_not_set": [
+                    "range:2-50"
+                ],
+                "int_range_not_set_empty_set": [
+                    "empty:"
+                ],
                 "match_target": ["matching:foo"],
                 "match_target_list": ["split:,", "matching:foo|bar|baz",],
                 "match_source_a": ["in:match_target"],

test-run/iterator_handling/data_search.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+"""
+Search data tests
+iterator_handling.data_search
+"""
+
+from corelibs.debug_handling.dump_data import dump_data
+from corelibs.iterator_handling.data_search import find_in_array_from_list, ArraySearchList
+
+
+def main() -> None:
+    """
+    Comment
+    """
+    data = [
+        {
+            "lookup_value_p": "A01",
+            "lookup_value_c": "B01",
+            "replace_value": "R01",
+        },
+        {
+            "lookup_value_p": "A02",
+            "lookup_value_c": "B02",
+            "replace_value": "R02",
+        },
+    ]
+    test_foo = ArraySearchList(
+        key = "lookup_value_p",
+        value = "A01"
+    )
+    print(test_foo)
+    search: list[ArraySearchList] = [
+        {
+            "key": "lookup_value_p",
+            "value": "A01"
+        },
+        {
+            "key": "lookup_value_c",
+            "value": "B01"
+        }
+    ]
+
+    result = find_in_array_from_list(data, search)
+
+    print(f"Search {dump_data(search)} -> {dump_data(result)}")
+
+
+if __name__ == "__main__":
+    main()
+
+# __END__

test-run/iterator_handling/dict_helpers.py

@@ -0,0 +1,106 @@
+"""
+Iterator helper testing
+"""
+
+from corelibs.debug_handling.dump_data import dump_data
+from corelibs.iterator_handling.dict_helpers import mask
+
+
+def __mask():
+    data = {
+        # "user": "john",
+        # "encryption_key": "Secret key",
+        # "ENCRYPTION.TEST": "Secret key test",
+        # "inside_password_test": "Hide this",
+        "password": ["secret1", "secret2"],  # List value gets masked
+        # "config": {
+        #     "db_password": {"primary": "secret", "backup": "secret2"},  # Dict value gets masked
+        #     "api_keys": ["key1", "key2", "key3"]  # List value gets masked
+        # },
+        # "items": [  # List value that doesn't get masked, but gets processed recursively
+        #     {"name": "item1", "secret_key": "itemsecret"},
+        #     {"name": "item2", "passwords": ["pass1", "pass2"]}
+        # ],
+        # "normal_list": ["item1", "item2", "item3"]  # Normal list, not masked
+    }
+    data = {
+        "config": {
+            # "password": ["secret1", "secret2"],
+            # "password_other": {"password": ["secret1", "secret2"]},
+            # "database": {
+            #     "host": "localhost",
+            #     "password": "db_secret",
+            #     "users": [
+            #         {"name": "admin", "password": "admin123"},
+            #         {"name": "user", "secret_key": "user456"}
+            #     ]
+            # },
+            # "api": {
+            #     # "endpoints": ["api1", "api2"],
+            #     "encryption_settings": {
+            #         "enabled": True,
+            #         "secret": "api_secret"
+            #     }
+            # }
+            "secret_key": "normal_value",
+            "api_key": "normal_value",
+            "my_key_value": "normal_value",
+        }
+    }
+    data = {
+        "basic": {
+            "log_level_console": "DEBUG",
+            "log_level_file": "DEBUG",
+            "storage_interface": "sqlite",
+            "content_start_date": "2023-1-1",
+            "encryption_key": "ENCRYPTION_KEY"
+        },
+        "email": {
+            "alert_email": [
+                "test+z-sd@tequila.jp"
+            ]
+        },
+        "poller": {
+            "max_forks": "1",
+            "interface": "Zac"
+        },
+        "pusher": {
+            "max_forks": "3",
+            "interface": "Screendragon"
+        },
+        "api:Zac": {
+            "type": "zac",
+            "client_id": "oro_zac_demo",
+            "client_secret": "CLIENT_SECRET",
+            "username": "zacuser",
+            "password": "ZACuser3",
+            "hostname": "e-gra2.zac.ai",
+            "appname": "e-gra2_api_trial",
+            "api_path": "b/api/v2"
+        },
+        "api:Screendragon": {
+            "type": "screendragon",
+            "client_id": "omniprostaging",
+            "encryption_client": "SOME_SECRET",
+            "client_encryption": "SOME_SECRET",
+            "secret_client": "SOME_SECRET",
+            "client_secret": "SOME_SECRET",
+            "hostname": "omniprostaging.screendragon.com",
+            "appname": "sdapi",
+            "api_path": "api"
+        }
+    }
+    result = mask(data)
+    print(f"** In: {dump_data(data)}")
+    print(f"===> Masked: {dump_data(result)}")
+
+
+def main():
+    """
+    Test: corelibs.string_handling.string_helpers
+    """
+    __mask()
+
+
+if __name__ == "__main__":
+    main()

test-run/json_handling/jmespath_helper.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+"""
+jmes path testing
+"""
+
+from corelibs.debug_handling.dump_data import dump_data
+from corelibs.json_handling.jmespath_helper import jmespath_search
+
+
+def main() -> None:
+    """
+    Comment
+    """
+    __set = {
+        'a': 'b',
+        'foobar': [1, 2, 'a'],
+        'bar': {
+            'a': 1,
+            'b': 'c'
+        },
+        'baz': [
+            {
+                'aa': 1,
+                'ab': 'cc'
+            },
+            {
+                'ba': 2,
+                'bb': 'dd'
+            },
+        ],
+        'foo': {
+            'a': [1, 2, 3],
+            'b': ['a', 'b', 'c']
+        }
+    }
+
+    __get = [
+        'a',
+        'bar.a',
+        'foo.a',
+        'baz[].aa'
+    ]
+    for __jmespath in __get:
+        result = jmespath_search(__set, __jmespath)
+        print(f"GET {__jmespath}: {dump_data(result)}")
+
+
+if __name__ == "__main__":
+    main()
+
+# __END__

test-run/logging_handling/log.py

@@ -3,9 +3,11 @@ Log logging_handling.log testing
 """
 
 # import atexit
+import sys
 from pathlib import Path
 # this is for testing only
-from corelibs.logging_handling.log import Log
+from corelibs.logging_handling.log import Log, Logger
+from corelibs.debug_handling.debug_helpers import exception_stack, call_stack
 from corelibs.logging_handling.logging_level_handling.logging_level import LoggingLevel
 
 
@@ -18,16 +20,20 @@ def main():
         log_path=script_path.joinpath('log', 'test.log'),
         log_name="Test Log",
         log_settings={
-            # "log_level_console": 'DEBUG',
-            "log_level_console": None,
+            "log_level_console": 'DEBUG',
+            # "log_level_console": None,
             "log_level_file": 'DEBUG',
             # "console_color_output_enabled": False,
+            "per_run_log": True
         }
     )
+    logn = Logger(log.get_logger_settings())
 
     log.logger.debug('[NORMAL] Debug test: %s', log.logger.name)
     log.lg.debug('[NORMAL] Debug test: %s', log.logger.name)
     log.debug('[NORMAL-] Debug test: %s', log.logger.name)
+    logn.lg.debug('[NORMAL N] Debug test: %s', log.logger.name)
+    logn.debug('[NORMAL N-] Debug test: %s', log.logger.name)
     log.logger.info('[NORMAL] Info test: %s', log.logger.name)
     log.info('[NORMAL-] Info test: %s', log.logger.name)
     log.logger.warning('[NORMAL] Warning test: %s', log.logger.name)
@@ -75,11 +81,16 @@ def main():
         __test = 5 / 0
         print(f"Divied: {__test}")
     except ZeroDivisionError as e:
+        print(f"** sys.exec_info(): {sys.exc_info()}")
+        print(f"** sys.exec_info(): [{exception_stack()}] | [{exception_stack(sys.exc_info())}] | [{call_stack()}]")
         log.logger.critical("Divison through zero: %s", e)
-        log.exception("Divison through zero")
+        log.exception("Divison through zero: %s", e)
 
     for handler in log.logger.handlers:
-        print(f"Handler (logger) {handler} -> {handler.level} -> {LoggingLevel.from_any(handler.level)}")
+        print(
+            f"** Handler (logger) {handler} [{handler.name}] -> "
+            f"{handler.level} -> {LoggingLevel.from_any(handler.level)}"
+        )
 
     for key, handler in log.handlers.items():
         print(f"Handler (handlers) [{key}] {handler} -> {handler.level} -> {LoggingLevel.from_any(handler.level)}")

test-run/string_handling/timestamp_strings.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+"""
+timestamp string checks
+"""
+
+from corelibs.string_handling.timestamp_strings import convert_to_seconds, TimeParseError, TimeUnitError
+
+
+def main() -> None:
+    """
+    Comment
+    """
+    test_cases = [
+        "5M 6d",           # 5 months, 6 days
+        "2h 30m 45s",      # 2 hours, 30 minutes, 45 seconds
+        "1Y 2M 3d",        # 1 year, 2 months, 3 days
+        "1h",              # 1 hour
+        "30m",             # 30 minutes
+        "2 hours 15 minutes",  # 2 hours, 15 minutes
+        "1d 12h",          # 1 day, 12 hours
+        "3M 2d 4h",        # 3 months, 2 days, 4 hours
+        "45s",             # 45 seconds
+        "1 year 2 months", # 1 year, 2 months
+        "2Y 6M 15d 8h 30m 45s",  # Complex example
+    # ]
+    # invalid_test_cases = [
+        "5M 6d 2M",        # months appears twice
+        "2h 30m 45s 1h",   # hours appears twice
+        "1d 2 days",       # days appears twice (short and long form)
+        "30m 45 minutes",  # minutes appears twice
+        "1Y 2 years",      # years appears twice
+        "1x 2 yrs",        # invalid names
+
+        123,               # int
+        789.12,            # float
+        456.56,            # float, high
+        "4566",            # int as string
+        "5551.12",         # float as string
+        "5551.56",         # float, high as string
+    ]
+
+    for time_string in test_cases:
+        try:
+            result = convert_to_seconds(time_string)
+            print(f"{time_string} => {result}")
+        except (TimeParseError, TimeUnitError) as e:
+            print(f"Error encountered for {time_string}: {type(e).__name__}: {e}")
+
+
+if __name__ == "__main__":
+    main()
+
+# __END__

tests/unit/iterator_handling/test_dict_helpers.py

@@ -0,0 +1,291 @@
+"""
+tests for corelibs.iterator_handling.dict_helpers
+"""
+
+import pytest
+from typing import Any
+from corelibs.iterator_handling.dict_helpers import mask
+
+
+def test_mask_default_behavior():
+    """Test masking with default mask_keys"""
+    data = {
+        "username": "john_doe",
+        "password": "secret123",
+        "email": "john@example.com",
+        "api_secret": "abc123",
+        "encryption_key": "xyz789"
+    }
+
+    result = mask(data)
+
+    assert result["username"] == "john_doe"
+    assert result["password"] == "***"
+    assert result["email"] == "john@example.com"
+    assert result["api_secret"] == "***"
+    assert result["encryption_key"] == "***"
+
+
+def test_mask_custom_keys():
+    """Test masking with custom mask_keys"""
+    data = {
+        "username": "john_doe",
+        "token": "abc123",
+        "api_key": "xyz789",
+        "password": "secret123"
+    }
+
+    result = mask(data, mask_keys=["token", "api"])
+
+    assert result["username"] == "john_doe"
+    assert result["token"] == "***"
+    assert result["api_key"] == "***"
+    assert result["password"] == "secret123"  # Not masked with custom keys
+
+
+def test_mask_custom_mask_string():
+    """Test masking with custom mask string"""
+    data = {"password": "secret123"}
+
+    result = mask(data, mask_str="[HIDDEN]")
+
+    assert result["password"] == "[HIDDEN]"
+
+
+def test_mask_case_insensitive():
+    """Test that masking is case insensitive"""
+    data = {
+        "PASSWORD": "secret123",
+        "Secret_Key": "abc123",
+        "ENCRYPTION_data": "xyz789"
+    }
+
+    result = mask(data)
+
+    assert result["PASSWORD"] == "***"
+    assert result["Secret_Key"] == "***"
+    assert result["ENCRYPTION_data"] == "***"
+
+
+def test_mask_key_patterns():
+    """Test different key matching patterns (start, end, contains)"""
+    data = {
+        "password_hash": "hash123",  # starts with
+        "user_password": "secret123",  # ends with
+        "my_secret_key": "abc123",  # contains with edges
+        "secretvalue": "xyz789",  # contains without edges
+        "startsecretvalue": "xyz123",  # contains without edges
+        "normal_key": "normal_value"
+    }
+
+    result = mask(data)
+
+    assert result["password_hash"] == "***"
+    assert result["user_password"] == "***"
+    assert result["my_secret_key"] == "***"
+    assert result["secretvalue"] == "***"  # will mask beacuse starts with
+    assert result["startsecretvalue"] == "xyz123"  # will not mask
+    assert result["normal_key"] == "normal_value"
+
+
+def test_mask_custom_edges():
+    """Test masking with custom edge characters"""
+    data = {
+        "my-secret-key": "abc123",
+        "my_secret_key": "xyz789"
+    }
+
+    result = mask(data, mask_str_edges="-")
+
+    assert result["my-secret-key"] == "***"
+    assert result["my_secret_key"] == "xyz789"  # Underscore edges don't match
+
+
+def test_mask_empty_edges():
+    """Test masking with empty edge characters (substring matching)"""
+    data = {
+        "secretvalue": "abc123",
+        "mysecretkey": "xyz789",
+        "normal_key": "normal_value"
+    }
+
+    result = mask(data, mask_str_edges="")
+
+    assert result["secretvalue"] == "***"
+    assert result["mysecretkey"] == "***"
+    assert result["normal_key"] == "normal_value"
+
+
+def test_mask_nested_dict():
+    """Test masking nested dictionaries"""
+    data = {
+        "user": {
+            "name": "john",
+            "password": "secret123",
+            "profile": {
+                "email": "john@example.com",
+                "encryption_key": "abc123"
+            }
+        },
+        "api_secret": "xyz789"
+    }
+
+    result = mask(data)
+
+    assert result["user"]["name"] == "john"
+    assert result["user"]["password"] == "***"
+    assert result["user"]["profile"]["email"] == "john@example.com"
+    assert result["user"]["profile"]["encryption_key"] == "***"
+    assert result["api_secret"] == "***"
+
+
+def test_mask_lists():
+    """Test masking lists and nested structures with lists"""
+    data = {
+        "users": [
+            {"name": "john", "password": "secret1"},
+            {"name": "jane", "password": "secret2"}
+        ],
+        "secrets": ["secret1", "secret2", "secret3"]
+    }
+
+    result = mask(data)
+    print(f"R {result['secrets']}")
+
+    assert result["users"][0]["name"] == "john"
+    assert result["users"][0]["password"] == "***"
+    assert result["users"][1]["name"] == "jane"
+    assert result["users"][1]["password"] == "***"
+    assert result["secrets"] == ["***", "***", "***"]
+
+
+def test_mask_mixed_types():
+    """Test masking with different value types"""
+    data = {
+        "password": "string_value",
+        "secret_number": 12345,
+        "encryption_flag": True,
+        "secret_float": 3.14,
+        "password_none": None,
+        "normal_key": "normal_value"
+    }
+
+    result = mask(data)
+
+    assert result["password"] == "***"
+    assert result["secret_number"] == "***"
+    assert result["encryption_flag"] == "***"
+    assert result["secret_float"] == "***"
+    assert result["password_none"] == "***"
+    assert result["normal_key"] == "normal_value"
+
+
+def test_mask_skip_true():
+    """Test that skip=True returns original data unchanged"""
+    data = {
+        "password": "secret123",
+        "encryption_key": "abc123",
+        "normal_key": "normal_value"
+    }
+
+    result = mask(data, skip=True)
+
+    assert result == data
+    assert result is data  # Should return the same object
+
+
+def test_mask_empty_dict():
+    """Test masking empty dictionary"""
+    data: dict[str, Any] = {}
+
+    result = mask(data)
+
+    assert result == {}
+
+
+def test_mask_none_mask_keys():
+    """Test explicit None mask_keys uses defaults"""
+    data = {"password": "secret123", "token": "abc123"}
+
+    result = mask(data, mask_keys=None)
+
+    assert result["password"] == "***"
+    assert result["token"] == "abc123"  # Not in default keys
+
+
+def test_mask_empty_mask_keys():
+    """Test empty mask_keys list"""
+    data = {"password": "secret123", "secret": "abc123"}
+
+    result = mask(data, mask_keys=[])
+
+    assert result["password"] == "secret123"
+    assert result["secret"] == "abc123"
+
+
+def test_mask_complex_nested_structure():
+    """Test masking complex nested structure"""
+    data = {
+        "config": {
+            "database": {
+                "host": "localhost",
+                "password": "db_secret",
+                "users": [
+                    {"name": "admin", "password": "admin123"},
+                    {"name": "user", "secret_key": "user456"}
+                ]
+            },
+            "api": {
+                "endpoints": ["api1", "api2"],
+                "encryption_settings": {
+                    "enabled": True,
+                    "secret": "api_secret"
+                }
+            }
+        }
+    }
+
+    result = mask(data)
+
+    assert result["config"]["database"]["host"] == "localhost"
+    assert result["config"]["database"]["password"] == "***"
+    assert result["config"]["database"]["users"][0]["name"] == "admin"
+    assert result["config"]["database"]["users"][0]["password"] == "***"
+    assert result["config"]["database"]["users"][1]["name"] == "user"
+    assert result["config"]["database"]["users"][1]["secret_key"] == "***"
+    assert result["config"]["api"]["endpoints"] == ["api1", "api2"]
+    assert result["config"]["api"]["encryption_settings"]["enabled"] is True
+    assert result["config"]["api"]["encryption_settings"]["secret"] == "***"
+
+
+def test_mask_preserves_original_data():
+    """Test that original data is not modified"""
+    original_data = {
+        "password": "secret123",
+        "username": "john_doe"
+    }
+    data_copy = original_data.copy()
+
+    result = mask(original_data)
+
+    assert original_data == data_copy  # Original unchanged
+    assert result != original_data  # Result is different
+    assert result["password"] == "***"
+    assert original_data["password"] == "secret123"
+
+
+@pytest.mark.parametrize("mask_key,expected_keys", [
+    (["pass"], ["password", "user_pass", "my_pass_key"]),
+    (["key"], ["api_key", "secret_key", "my_key_value"]),
+    (["token"], ["token", "auth_token", "my_token_here"]),
+])
+def test_mask_parametrized_keys(mask_key: list[str], expected_keys: list[str]):
+    """Parametrized test for different mask key patterns"""
+    data = {key: "value" for key in expected_keys}
+    data["normal_entry"] = "normal_value"
+
+    result = mask(data, mask_keys=mask_key)
+
+    for key in expected_keys:
+        assert result[key] == "***"
+    assert result["normal_entry"] == "normal_value"

uv.lock

@@ -1,5 +1,5 @@
 version = 1
-revision = 2
+revision = 3
 requires-python = ">=3.13"
 
 [[package]]
@@ -44,7 +44,7 @@ wheels = [
 
 [[package]]
 name = "corelibs"
-version = "0.12.2"
+version = "0.22.0"
 source = { editable = "." }
 dependencies = [
     { name = "jmespath" },