v0.12.4: settings loader add set default value for empty

With new empty: block if just like this set to None if not set (empty), can also be any value,
if list, skip setting default

pyproject.toml

@@ -1,7 +1,7 @@
 # MARK: Project info
 [project]
 name = "corelibs"
-version = "0.12.0"
+version = "0.12.4"
 description = "Collection of utils for Python scripts"
 readme = "README.md"
 requires-python = ">=3.13"
@@ -54,7 +54,8 @@ notes-rgx = '(FIXME|TODO)(\((TTD-|#)\[0-9]+\))'
 [tool.flake8]
 max-line-length = 120
 ignore = [
-    "E741" # ignore ambigious variable name
+    "E741", # ignore ambigious variable name
+    "W504"  # Line break occurred after a binary operator [wrong triggered by "or" in if]
 ]
 [tool.pylint.MASTER]
 # this is for the tests/etc folders

src/corelibs/check_handling/regex_constants.py

@@ -0,0 +1,14 @@
+"""
+List of regex compiled strings that can be used
+"""
+
+import re
+
+
+EMAIL_REGEX_BASIC = r"""
+^[A-Za-z0-9!#$%&'*+\-\/=?^_`{|}~][A-Za-z0-9!#$%:\(\)&'*+\-\/=?^_`{|}~\.]{0,63}
+@(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[a-zA-Z]{2,6}$
+"""
+EMAIL_REGEX_BASIC_COMPILED = re.compile(EMAIL_REGEX_BASIC)
+
+# __END__

src/corelibs/config_handling/settings_loader.py

@@ -51,16 +51,21 @@ class SettingsLoader:
         self.always_print = always_print
         # entries that have to be split
         self.entry_split_char: dict[str, str] = {}
+        # entries that should be converted
         self.entry_convert: dict[str, str] = {}
-        # config parser
+        # default set entries
+        self.entry_set_empty: dict[str, str | None] = {}
+        # config parser, load config file first
         self.config_parser: configparser.ConfigParser | None = self.__load_config_file()
         # all settings
-        self.settings: dict[str, dict[str, Any]] | None = None
+        self.settings: dict[str, dict[str, None | str | int | float | bool]] | None = None
         # remove file name and get base path and check
         if not self.config_file.parent.is_dir():
             raise ValueError(f"Cannot find the config folder: {self.config_file.parent}")
-        # load the config file before we parse anything
+        # for check settings, abort flag
+        self._check_settings_abort: bool = False
 
+    # MARK: load settings
     def load_settings(self, config_id: str, config_validate: dict[str, list[str]]) -> dict[str, str]:
         """
         neutral settings loader
@@ -77,6 +82,7 @@ class SettingsLoader:
         - in: the right side is another KEY value from the settings where this value must be inside
         - split: character to split entries, if set check:list+ must be set if checks are needed
         - convert: convert to int, float -> if element is number convert, else leave as is
+        - empty: convert empty to, if nothing set on the right side then convert to None type
 
         Args:
             config_id (str): what block to load
@@ -92,6 +98,13 @@ class SettingsLoader:
             try:
                 # load all data as is, validation is done afterwards
                 settings[config_id] = dict(self.config_parser[config_id])
+            except KeyError as e:
+                self.__print(
+                    f"[!] Cannot read [{config_id}] block in the {self.config_file}: {e}",
+                    'CRITICAL', raise_exception=True
+                )
+                sys.exit(1)
+            try:
                 for key, checks in config_validate.items():
                     skip = True
                     split_char = self.DEFAULT_ELEMENT_SPLIT_CHAR
@@ -106,6 +119,8 @@ class SettingsLoader:
                                         'CRITICAL',
                                         raise_exception=True
                                     )
+                                    sys.exit(1)
+                                self.entry_convert[key] = convert_to
                             except ValueError as e:
                                 self.__print(
                                     f"[!] In [{config_id}] the convert type setup for entry failed: {check}: {e}",
@@ -113,6 +128,20 @@ class SettingsLoader:
                                     raise_exception=True
                                 )
                                 sys.exit(1)
+                        if check.startswith('empty:'):
+                            try:
+                                [_, empty_set] = check.split(":")
+                                if not empty_set:
+                                    empty_set = None
+                                self.entry_set_empty[key] = empty_set
+                            except ValueError as e:
+                                print(f"VALUE ERROR: {key}")
+                                self.__print(
+                                    f"[!] In [{config_id}] the empty set type for entry failed: {check}: {e}",
+                                    'CRITICAL',
+                                    raise_exception=True
+                                )
+                                sys.exit(1)
                         # split char, also check to not set it twice, first one only
                         if check.startswith("split:") and not self.entry_split_char.get(key):
                             try:
@@ -143,7 +172,7 @@ class SettingsLoader:
                     ]
             except KeyError as e:
                 self.__print(
-                    f"[!] Cannot read [{config_id}] block in the {self.config_file}: {e}",
+                    f"[!] Cannot read [{config_id}] block because the entry [{e}] could not be found",
                     'CRITICAL', raise_exception=True
                 )
                 sys.exit(1)
@@ -173,7 +202,9 @@ class SettingsLoader:
                 # - length: for string length
                 # - range: for int/float range check
                 # mandatory check
-                if check == "mandatory:yes" and not settings[config_id].get(entry):
+                if check == "mandatory:yes" and (
+                    not settings[config_id].get(entry) or settings[config_id].get(entry) == ['']
+                ):
                     error = True
                     self.__print(f"[!] Missing content entry for: {entry}", 'ERROR')
                 # skip if empty none
@@ -184,6 +215,8 @@ class SettingsLoader:
                     settings[config_id][entry] = self.__check_settings(
                         check, entry, settings[config_id][entry]
                     )
+                    if self._check_settings_abort is True:
+                        error = True
                 elif check.startswith("matching:"):
                     checks = check.replace("matching:", "").split("|")
                     if __result := is_list_in_list(convert_to_list(settings[config_id][entry]), list(checks)):
@@ -230,8 +263,14 @@ class SettingsLoader:
         if error is True:
             self.__print("[!] Missing or incorrect settings data. Cannot proceed", 'CRITICAL', raise_exception=True)
             sys.exit(1)
+        # set empty
+        for [entry, empty_set] in self.entry_set_empty.items():
+            # if set, skip, else set to empty value
+            if settings[config_id].get(entry) or isinstance(settings[config_id].get(entry), list):
+                continue
+            settings[config_id][entry] = empty_set
         # Convert input
-        for [entry, convert_type] in self.entry_convert:
+        for [entry, convert_type] in self.entry_convert.items():
             if convert_type in ["int", "any"] and is_int(settings[config_id][entry]):
                 settings[config_id][entry] = int(settings[config_id][entry])
             elif convert_type in ["float", "any"] and is_float(settings[config_id][entry]):
@@ -250,9 +289,11 @@ class SettingsLoader:
                         'ERROR'
                     )
             # string is always string
+            # TODO: empty and int/float/bool: set to none?
 
         return settings[config_id]
 
+    # MARK: build from/to/requal logic
     def __build_from_to_equal(
         self, entry: str, check: str, convert_to_int: bool = False
     ) -> Tuple[float | None, float | None, float | None]:
@@ -306,6 +347,7 @@ class SettingsLoader:
             __equal
         )
 
+    # MARK: length/range validation
     def __length_range_validate(
         self,
         entry: str,
@@ -316,6 +358,9 @@ class SettingsLoader:
         (__from, __to, __equal) = check
         valid = True
         for value_raw in convert_to_list(values):
+            # skip no tset values for range check
+            if not value_raw:
+                continue
             value = 0
             error_mark = ''
             if check_type == 'length':
@@ -347,6 +392,7 @@ class SettingsLoader:
                 continue
         return valid
 
+    # MARK: load config file data from file
     def __load_config_file(self) -> configparser.ConfigParser | None:
         """
         load and parse the config file
@@ -358,13 +404,14 @@ class SettingsLoader:
             return config
         return None
 
+    # MARK: regex clean up one
     def __clean_invalid_setting(
         self,
         entry: str,
         validate: str,
         value: str,
         regex: str,
-        regex_clean: str,
+        regex_clean: str | None,
         replace: str = "",
         print_error: bool = True,
     ) -> str:
@@ -380,18 +427,25 @@ class SettingsLoader:
             replace (str): replace with character. Defaults to ''
             print_error (bool): print the error message. Defaults to True
         """
-        check = re.compile(regex)
+        check = re.compile(regex, re.VERBOSE)
-        clean = re.compile(regex_clean)
+        clean: re.Pattern[str] | None = None
-        if not check.search(value):
+        if regex_clean is not None:
+            clean = re.compile(regex_clean, re.VERBOSE)
+        # value must be set if clean is None, else empty value is allowed and will fail
+        if (clean is None and value or clean) and not check.search(value):
             self.__print(
                 f"[!] Invalid content for '{entry}' with check '{validate}' and data: {value}",
                 'ERROR', print_error
             )
-            # clean up
+            # clean up if clean up is not none, else return EMPTY string
-            return clean.sub(replace, value)
+            if clean is not None:
+                return clean.sub(replace, value)
+            self._check_settings_abort = True
+            return ''
         # else return as is
         return value
 
+    # MARK: check settings, regx
     def __check_settings(
         self,
         check: str, entry: str, setting_value: list[str] | str
@@ -439,6 +493,7 @@ class SettingsLoader:
         # return data
         return setting_value
 
+    # MARK: check arguments, for config file load fail
     def __check_arguments(self, arguments: dict[str, list[str]], all_set: bool = False) -> bool:
         """
         check if ast least one argument is set
@@ -468,6 +523,7 @@ class SettingsLoader:
 
         return has_argument
 
+    # MARK: get argument from args dict
     def __get_arg(self, entry: str) -> Any:
         """
         check if an argument entry xists, if None -> returns None else value of argument
@@ -482,6 +538,7 @@ class SettingsLoader:
             return None
         return self.args.get(entry)
 
+    # MARK: error print
     def __print(self, msg: str, level: str, print_error: bool = True, raise_exception: bool = False):
         """
         print out error, if Log class is set then print to log instead

src/corelibs/config_handling/settings_loader_handling/settings_loader_check.py

@@ -3,12 +3,15 @@ Class of checks that can be run on value entries
 """
 
 from typing import TypedDict
+from corelibs.check_handling.regex_constants import EMAIL_REGEX_BASIC
 
 
 class SettingsLoaderCheckValue(TypedDict):
     """Settings check entries"""
+
     regex: str
-    regex_clean: str
+    # if None, then on error we exit, eles we clean up data
+    regex_clean: str | None
     replace: str
 
 
@@ -16,29 +19,37 @@ class SettingsLoaderCheck:
     """
     check:<NAME> or check:list+<NAME>
     """
+
     CHECK_SETTINGS: dict[str, SettingsLoaderCheckValue] = {
         "int": {
             "regex": r"^[0-9]+$",
             "regex_clean": r"[^0-9]",
-            "replace": ""
+            "replace": "",
         },
         "string.alphanumeric": {
             "regex": r"^[a-zA-Z0-9]+$",
             "regex_clean": r"[^a-zA-Z0-9]",
-            "replace": ""
+            "replace": "",
         },
         "string.alphanumeric.lower.dash": {
             "regex": r"^[a-z0-9-]+$",
             "regex_clean": r"[^a-z0-9-]",
-            "replace": ""
+            "replace": "",
         },
         # A-Z a-z 0-9 _ - . ONLY
         # This one does not remove, but replaces with _
         "string.alphanumeric.extended.replace": {
             "regex": r"^[_.a-zA-Z0-9-]+$",
             "regex_clean": r"[^_.a-zA-Z0-9-]",
-            "replace": "_"
+            "replace": "_",
+        },
+        # This does a baisc email check, only alphanumeric with special characters
+        "string.email.basic": {
+            "regex": EMAIL_REGEX_BASIC,
+            "regex_clean": None,
+            "replace": "",
         },
     }
 
+
 # __END__

src/corelibs/iterator_handling/dict_helpers.py

@@ -3,6 +3,9 @@ Dict helpers
 """
 
 
+from typing import Any
+
+
 def mask(
     data_set: dict[str, str],
     mask_keys: list[str] | None = None,
@@ -34,4 +37,22 @@ def mask(
         for key, value in data_set.items()
     }
 
+
+def set_entry(dict_set: dict[str, Any], key: str, value_set: Any) -> dict[str, Any]:
+    """
+    set a new entry in the dict set
+
+    Arguments:
+        key {str} -- _description_
+        dict_set {dict[str, Any]} -- _description_
+        value_set {Any} -- _description_
+
+    Returns:
+        dict[str, Any] -- _description_
+    """
+    if not dict_set.get(key):
+        dict_set[key] = {}
+    dict_set[key] = value_set
+    return dict_set
+
 # __END__

src/corelibs/logging_handling/log.py

@@ -527,7 +527,7 @@ class Log:
     # MARK: log level handling
     def set_log_level(self, handler_name: str, log_level: LoggingLevel) -> bool:
         """
-        set the logging level
+        set the logging level for a handler
 
         Arguments:
             handler {str} -- _description_
@@ -555,7 +555,7 @@ class Log:
 
     def get_log_level(self, handler_name: str) -> LoggingLevel:
         """
-        gett the logging level for a handler
+        gettthe logging level for a handler
 
         Arguments:
             handler_name {str} -- _description_
@@ -571,7 +571,7 @@ class Log:
     @staticmethod
     def validate_log_level(log_level: Any) -> bool:
         """
-        if the log level is invalid, will erturn false
+        if the log level is invalid will return false, else return true
 
         Args:
             log_level (Any): _description_
@@ -589,7 +589,7 @@ class Log:
     def get_log_level_int(log_level: Any) -> int:
         """
         Return log level as INT
-        If invalid return set level in default log level
+        If invalid returns the default log level
 
         Arguments:
             log_level {Any} -- _description_

test-run/config_handling/config/settings.ini

@@ -1,6 +1,7 @@
 [TestA]
 foo=bar
 foobar=1
+bar=st
 some_match=foo
 some_match_list=foo,bar
 test_list=a,b,c,d f, g h
@@ -8,6 +9,8 @@ other_list=a|b|c|d|
 third_list=xy|ab|df|fg
 str_length=foobar
 int_range=20
+int_range_not_set=
+int_range_not_set_empty_set=5
 #
 match_target=foo
 match_target_list=foo,bar,baz
@@ -21,3 +24,6 @@ match_source_list=foo,bar
 element_a=Static energy
 element_b=123.5
 element_c=True
+email=foo@bar.com,other+bar-fee@domain-com.cp,
+email_not_mandatory=
+email_bad=gii@bar.com

test-run/config_handling/settings_loader.py

@@ -42,8 +42,10 @@ def main():
         config_data = sl.load_settings(
             config_load,
             {
+                # "doesnt": ["split:,"],
                 "foo": ["mandatory:yes"],
                 "foobar": ["check:int"],
+                "bar": ["mandatory:yes"],
                 "some_match": ["matching:foo|bar"],
                 "some_match_list": ["split:,", "matching:foo|bar"],
                 "test_list": [
@@ -61,6 +63,12 @@ def main():
                 "int_range": [
                     "range:2-50"
                 ],
+                "int_range_not_set": [
+                    "range:2-50"
+                ],
+                "int_range_not_set_empty_set": [
+                    "empty:"
+                ],
                 "match_target": ["matching:foo"],
                 "match_target_list": ["split:,", "matching:foo|bar|baz",],
                 "match_source_a": ["in:match_target"],
@@ -68,7 +76,33 @@ def main():
                 "match_source_list": ["split:,", "in:match_target_list"],
             }
         )
-        print(f"Load: {config_load} -> {dump_data(config_data)}")
+        print(f"[{config_load}] Load: {config_load} -> {dump_data(config_data)}")
+    except ValueError as e:
+        print(f"Could not load settings: {e}")
+
+    try:
+        config_load = 'TestB'
+        config_data = sl.load_settings(
+            config_load,
+            {
+                "email": [
+                    "split:,",
+                    "mandatory:yes",
+                    "check:string.email.basic"
+                ],
+                "email_not_mandatory": [
+                    "split:,",
+                    # "mandatory:yes",
+                    "check:string.email.basic"
+                ],
+                "email_bad": [
+                    "split:,",
+                    "mandatory:yes",
+                    "check:string.email.basic"
+                ]
+            }
+        )
+        print(f"[{config_load}] Load: {config_load} -> {dump_data(config_data)}")
     except ValueError as e:
         print(f"Could not load settings: {e}")
 

test-run/logging_handling/log.py

@@ -18,7 +18,8 @@ def main():
         log_path=script_path.joinpath('log', 'test.log'),
         log_name="Test Log",
         log_settings={
-            "log_level_console": 'DEBUG',
+            # "log_level_console": 'DEBUG',
+            "log_level_console": None,
             "log_level_file": 'DEBUG',
             # "console_color_output_enabled": False,
         }

uv.lock

@@ -44,7 +44,7 @@ wheels = [
 
 [[package]]
 name = "corelibs"
-version = "0.11.0"
+version = "0.12.2"
 source = { editable = "." }
 dependencies = [
     { name = "jmespath" },