Compare commits

...

23 Commits

Author SHA1 Message Date
Clemens Schwaighofer
da68818d4f Move the domain regex to the regex constant file 2025-07-15 11:13:23 +09:00
Clemens Schwaighofer
db6a3b53c5 v0.13.1: settings loader check additions 2025-07-15 10:33:38 +09:00
Clemens Schwaighofer
82b089498e Merge branch 'development' 2025-07-15 10:33:02 +09:00
Clemens Schwaighofer
948b0dd5e7 Settings loader add more checks
string.domain.with-localhost
string.domain.with-localhost.port
string.domain
string.date
2025-07-15 10:32:19 +09:00
Clemens Schwaighofer
4acc0b51b1 v0.13.0: move the dump data method from the iterator folder to the debug folder 2025-07-15 09:55:25 +09:00
Clemens Schwaighofer
a626b738a9 Move dump_data from iterator folder to debug folder 2025-07-15 09:54:23 +09:00
Clemens Schwaighofer
7119844313 v0.12.6: Settings: exception raised on error point, stacklevel increased for all sub functions in log/settings loader 2025-07-15 09:51:23 +09:00
Clemens Schwaighofer
5763f57830 In settings loader do the raise ValueRror on the error, fix stack level, loggin fix stack level
Settings loader: all errors are thrown where the error happens and not in the print function
The print function if to log will add +1 to the stack level so the error is shown

In the log class in the log wrapper calls add +1 to the stack level to have the error line in the correct place
-> this fixes the stack trace part for now but we still want to have an auto full stack trace simple added
2025-07-15 09:44:29 +09:00
Clemens Schwaighofer
70e8ceecce v0.12.5: settings loader allow empty block 2025-07-14 18:15:59 +09:00
Clemens Schwaighofer
acbe1ac692 Settings load add info for future settings/options argument 2025-07-14 18:15:07 +09:00
Clemens Schwaighofer
99bca2c467 Allow settings block to not exist via call setting 2025-07-14 18:14:33 +09:00
Clemens Schwaighofer
b74ed1f30e v0.12.4: settings loader add set default value for empty 2025-07-14 17:22:03 +09:00
Clemens Schwaighofer
8082ab78a1 Merge branch 'development' 2025-07-14 17:21:28 +09:00
Clemens Schwaighofer
c69076f517 Add set default if empty/not set in settings
With new empty: block if just like this set to None if not set (empty), can also be any value,
if list, skip setting default
2025-07-14 17:21:04 +09:00
Clemens Schwaighofer
648ab001b6 Settings loader fix for not set range check entries
If we have a range or length check and the value is not set, skip, and do not convert either
Not set is None
2025-07-14 17:00:25 +09:00
Clemens Schwaighofer
447034046e v0.12.3: settings loader error message improvement 2025-07-14 16:50:36 +09:00
Clemens Schwaighofer
0770ac0bb4 Better error handling in the settings loader for entry not found in block 2025-07-14 16:49:37 +09:00
Clemens Schwaighofer
aa2fbd4f70 v0.12.2: Fix mandatory for settings loader 2025-07-14 16:25:21 +09:00
Clemens Schwaighofer
58c8447531 Settings loader mandatory fixes
- mandatory empty check if empty list ([''])
- skip regex check if replace value is None -> allowed empty as empty if not mandatory
2025-07-14 16:23:55 +09:00
Clemens Schwaighofer
bcca43d774 v0.12.1: settings loader update, regex constants added 2025-07-14 16:01:54 +09:00
Clemens Schwaighofer
e9ccfe7ad2 Rebame the regex constants file name to not have compiled inside the name 2025-07-14 15:59:34 +09:00
Clemens Schwaighofer
6c2637ad34 Settings loader update with basic email check, and on check abort if not valid
In the settings checker, if a regex_clean is set as None then we will abort the script with error
if the regex is not matching

Add regex check for email basic

Also add a regex_constants list with regex entries (not compiled and compiled)
2025-07-14 15:57:19 +09:00
Clemens Schwaighofer
7183d05dd6 Update log method documentation 2025-07-14 14:29:42 +09:00
11 changed files with 290 additions and 90 deletions

View File

@@ -1,7 +1,7 @@
# MARK: Project info # MARK: Project info
[project] [project]
name = "corelibs" name = "corelibs"
version = "0.12.0" version = "0.13.1"
description = "Collection of utils for Python scripts" description = "Collection of utils for Python scripts"
readme = "README.md" readme = "README.md"
requires-python = ">=3.13" requires-python = ">=3.13"
@@ -54,7 +54,8 @@ notes-rgx = '(FIXME|TODO)(\((TTD-|#)\[0-9]+\))'
[tool.flake8] [tool.flake8]
max-line-length = 120 max-line-length = 120
ignore = [ ignore = [
"E741" # ignore ambigious variable name "E741", # ignore ambigious variable name
"W504" # Line break occurred after a binary operator [wrong triggered by "or" in if]
] ]
[tool.pylint.MASTER] [tool.pylint.MASTER]
# this is for the tests/etc folders # this is for the tests/etc folders

View File

@@ -0,0 +1,37 @@
"""
List of regex compiled strings that can be used
"""
import re
def compile_re(reg: str) -> re.Pattern[str]:
"""
compile a regex with verbose flag
Arguments:
reg {str} -- _description_
Returns:
re.Pattern[str] -- _description_
"""
return re.compile(reg, re.VERBOSE)
# email regex
EMAIL_BASIC_REGEX = r"""
^[A-Za-z0-9!#$%&'*+\-\/=?^_`{|}~][A-Za-z0-9!#$%:\(\)&'*+\-\/=?^_`{|}~\.]{0,63}
@(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[a-zA-Z]{2,6}$
"""
# Domain regex with localhost
DOMAIN_WITH_LOCALHOST_REGEX = r"""
^(?:localhost|(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[A-Za-z]{2,})$
"""
# domain regex with loclhost and optional port
DOMAIN_WITH_LOCALHOST_PORT_REGEX = r"""
^(?:localhost|(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[A-Za-z]{2,})(?::\d+)?$
"""
# Domain, no localhost
DOMAIN_REGEX = r"^(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.[A-Za-z0-9-]{1,63}(?<!-))*\.[A-Za-z]{2,}$"
# __END__

View File

@@ -5,7 +5,6 @@ Additional check for override settings as arguments
""" """
import re import re
import sys
import configparser import configparser
from typing import Any, Tuple, Sequence, cast from typing import Any, Tuple, Sequence, cast
from pathlib import Path from pathlib import Path
@@ -51,17 +50,27 @@ class SettingsLoader:
self.always_print = always_print self.always_print = always_print
# entries that have to be split # entries that have to be split
self.entry_split_char: dict[str, str] = {} self.entry_split_char: dict[str, str] = {}
# entries that should be converted
self.entry_convert: dict[str, str] = {} self.entry_convert: dict[str, str] = {}
# config parser # default set entries
self.entry_set_empty: dict[str, str | None] = {}
# config parser, load config file first
self.config_parser: configparser.ConfigParser | None = self.__load_config_file() self.config_parser: configparser.ConfigParser | None = self.__load_config_file()
# all settings # all settings
self.settings: dict[str, dict[str, Any]] | None = None self.settings: dict[str, dict[str, None | str | int | float | bool]] | None = None
# remove file name and get base path and check # remove file name and get base path and check
if not self.config_file.parent.is_dir(): if not self.config_file.parent.is_dir():
raise ValueError(f"Cannot find the config folder: {self.config_file.parent}") raise ValueError(f"Cannot find the config folder: {self.config_file.parent}")
# load the config file before we parse anything # for check settings, abort flag
self._check_settings_abort: bool = False
def load_settings(self, config_id: str, config_validate: dict[str, list[str]]) -> dict[str, str]: # MARK: load settings
def load_settings(
self,
config_id: str,
config_validate: dict[str, list[str]],
allow_not_exist: bool = False
) -> dict[str, str]:
""" """
neutral settings loader neutral settings loader
@@ -77,10 +86,14 @@ class SettingsLoader:
- in: the right side is another KEY value from the settings where this value must be inside - in: the right side is another KEY value from the settings where this value must be inside
- split: character to split entries, if set check:list+ must be set if checks are needed - split: character to split entries, if set check:list+ must be set if checks are needed
- convert: convert to int, float -> if element is number convert, else leave as is - convert: convert to int, float -> if element is number convert, else leave as is
- empty: convert empty to, if nothing set on the right side then convert to None type
TODO: there should be a config/options argument for general settings
Args: Args:
config_id (str): what block to load config_id (str): what block to load
config_allowed (list[str]): list of allowed entries sets config_validate (dict[str, list[str]]): list of allowed entries sets
allow_not_exist (bool): If set to True, does not throw an error, but returns empty set
Returns: Returns:
dict[str, str]: key = value list dict[str, str]: key = value list
@@ -92,6 +105,14 @@ class SettingsLoader:
try: try:
# load all data as is, validation is done afterwards # load all data as is, validation is done afterwards
settings[config_id] = dict(self.config_parser[config_id]) settings[config_id] = dict(self.config_parser[config_id])
except KeyError as e:
if allow_not_exist is True:
return {}
raise ValueError(self.__print(
f"[!] Cannot read [{config_id}] block in the {self.config_file}: {e}",
'CRITICAL'
)) from e
try:
for key, checks in config_validate.items(): for key, checks in config_validate.items():
skip = True skip = True
split_char = self.DEFAULT_ELEMENT_SPLIT_CHAR split_char = self.DEFAULT_ELEMENT_SPLIT_CHAR
@@ -101,18 +122,28 @@ class SettingsLoader:
try: try:
[_, convert_to] = check.split(":") [_, convert_to] = check.split(":")
if convert_to not in self.CONVERT_TO_LIST: if convert_to not in self.CONVERT_TO_LIST:
self.__print( raise ValueError(self.__print(
f"[!] In [{config_id}] the convert type is invalid {check}: {convert_to}", f"[!] In [{config_id}] the convert type is invalid {check}: {convert_to}",
'CRITICAL', 'CRITICAL'
raise_exception=True ))
) self.entry_convert[key] = convert_to
except ValueError as e: except ValueError as e:
self.__print( raise ValueError(self.__print(
f"[!] In [{config_id}] the convert type setup for entry failed: {check}: {e}", f"[!] In [{config_id}] the convert type setup for entry failed: {check}: {e}",
'CRITICAL', 'CRITICAL'
raise_exception=True )) from e
) if check.startswith('empty:'):
sys.exit(1) try:
[_, empty_set] = check.split(":")
if not empty_set:
empty_set = None
self.entry_set_empty[key] = empty_set
except ValueError as e:
print(f"VALUE ERROR: {key}")
raise ValueError(self.__print(
f"[!] In [{config_id}] the empty set type for entry failed: {check}: {e}",
'CRITICAL'
)) from e
# split char, also check to not set it twice, first one only # split char, also check to not set it twice, first one only
if check.startswith("split:") and not self.entry_split_char.get(key): if check.startswith("split:") and not self.entry_split_char.get(key):
try: try:
@@ -129,12 +160,10 @@ class SettingsLoader:
self.entry_split_char[key] = split_char self.entry_split_char[key] = split_char
skip = False skip = False
except ValueError as e: except ValueError as e:
self.__print( raise ValueError(self.__print(
f"[!] In [{config_id}] the split character setup for entry failed: {check}: {e}", f"[!] In [{config_id}] the split character setup for entry failed: {check}: {e}",
'CRITICAL', 'CRITICAL'
raise_exception=True )) from e
)
sys.exit(1)
if skip: if skip:
continue continue
settings[config_id][key] = [ settings[config_id][key] = [
@@ -142,16 +171,14 @@ class SettingsLoader:
for __value in settings[config_id][key].split(split_char) for __value in settings[config_id][key].split(split_char)
] ]
except KeyError as e: except KeyError as e:
self.__print( raise ValueError(self.__print(
f"[!] Cannot read [{config_id}] block in the {self.config_file}: {e}", f"[!] Cannot read [{config_id}] block because the entry [{e}] could not be found",
'CRITICAL', raise_exception=True 'CRITICAL'
) )) from e
sys.exit(1)
else: else:
# ignore error if arguments are set # ignore error if arguments are set
if not self.__check_arguments(config_validate, True): if not self.__check_arguments(config_validate, True):
self.__print(f"[!] Cannot find file: {self.config_file}", 'CRITICAL', raise_exception=True) raise ValueError(self.__print(f"[!] Cannot find file: {self.config_file}", 'CRITICAL'))
sys.exit(1)
else: else:
# base set # base set
settings[config_id] = {} settings[config_id] = {}
@@ -173,7 +200,9 @@ class SettingsLoader:
# - length: for string length # - length: for string length
# - range: for int/float range check # - range: for int/float range check
# mandatory check # mandatory check
if check == "mandatory:yes" and not settings[config_id].get(entry): if check == "mandatory:yes" and (
not settings[config_id].get(entry) or settings[config_id].get(entry) == ['']
):
error = True error = True
self.__print(f"[!] Missing content entry for: {entry}", 'ERROR') self.__print(f"[!] Missing content entry for: {entry}", 'ERROR')
# skip if empty none # skip if empty none
@@ -184,6 +213,8 @@ class SettingsLoader:
settings[config_id][entry] = self.__check_settings( settings[config_id][entry] = self.__check_settings(
check, entry, settings[config_id][entry] check, entry, settings[config_id][entry]
) )
if self._check_settings_abort is True:
error = True
elif check.startswith("matching:"): elif check.startswith("matching:"):
checks = check.replace("matching:", "").split("|") checks = check.replace("matching:", "").split("|")
if __result := is_list_in_list(convert_to_list(settings[config_id][entry]), list(checks)): if __result := is_list_in_list(convert_to_list(settings[config_id][entry]), list(checks)):
@@ -227,11 +258,22 @@ class SettingsLoader:
self.__build_from_to_equal(entry, check) self.__build_from_to_equal(entry, check)
): ):
error = True error = True
# after post clean up if we have empty entries and we are mandatory
if check == "mandatory:yes" and (
not settings[config_id].get(entry) or settings[config_id].get(entry) == ['']
):
error = True
self.__print(f"[!] Missing content entry for: {entry}", 'ERROR')
if error is True: if error is True:
self.__print("[!] Missing or incorrect settings data. Cannot proceed", 'CRITICAL', raise_exception=True) raise ValueError(self.__print("[!] Missing or incorrect settings data. Cannot proceed", 'CRITICAL'))
sys.exit(1) # set empty
for [entry, empty_set] in self.entry_set_empty.items():
# if set, skip, else set to empty value
if settings[config_id].get(entry) or isinstance(settings[config_id].get(entry), list):
continue
settings[config_id][entry] = empty_set
# Convert input # Convert input
for [entry, convert_type] in self.entry_convert: for [entry, convert_type] in self.entry_convert.items():
if convert_type in ["int", "any"] and is_int(settings[config_id][entry]): if convert_type in ["int", "any"] and is_int(settings[config_id][entry]):
settings[config_id][entry] = int(settings[config_id][entry]) settings[config_id][entry] = int(settings[config_id][entry])
elif convert_type in ["float", "any"] and is_float(settings[config_id][entry]): elif convert_type in ["float", "any"] and is_float(settings[config_id][entry]):
@@ -250,9 +292,11 @@ class SettingsLoader:
'ERROR' 'ERROR'
) )
# string is always string # string is always string
# TODO: empty and int/float/bool: set to none?
return settings[config_id] return settings[config_id]
# MARK: build from/to/requal logic
def __build_from_to_equal( def __build_from_to_equal(
self, entry: str, check: str, convert_to_int: bool = False self, entry: str, check: str, convert_to_int: bool = False
) -> Tuple[float | None, float | None, float | None]: ) -> Tuple[float | None, float | None, float | None]:
@@ -275,22 +319,20 @@ class SettingsLoader:
try: try:
[__from, __to] = check.split('-') [__from, __to] = check.split('-')
if (__from and not is_float(__from)) or (__to and not is_float(__to)): if (__from and not is_float(__from)) or (__to and not is_float(__to)):
self.__print( raise ValueError(self.__print(
f"[{entry}] Check value for length is not in: {check}", f"[{entry}] Check value for length is not in: {check}",
'CRITICAL', raise_exception=True 'CRITICAL'
) ))
sys.exit(1)
if len(__from) == 0: if len(__from) == 0:
__from = None __from = None
if len(__to) == 0: if len(__to) == 0:
__to = None __to = None
except ValueError: except ValueError as e:
if not is_float(__equal := check): if not is_float(__equal := check):
self.__print( raise ValueError(self.__print(
f"[{entry}] Check value for length is not a valid integer: {check}", f"[{entry}] Check value for length is not a valid integer: {check}",
'CRITICAL', raise_exception=True 'CRITICAL'
) )) from e
sys.exit(1)
if len(__equal) == 0: if len(__equal) == 0:
__equal = None __equal = None
# makre sure this is all int or None # makre sure this is all int or None
@@ -306,6 +348,7 @@ class SettingsLoader:
__equal __equal
) )
# MARK: length/range validation
def __length_range_validate( def __length_range_validate(
self, self,
entry: str, entry: str,
@@ -316,6 +359,9 @@ class SettingsLoader:
(__from, __to, __equal) = check (__from, __to, __equal) = check
valid = True valid = True
for value_raw in convert_to_list(values): for value_raw in convert_to_list(values):
# skip no tset values for range check
if not value_raw:
continue
value = 0 value = 0
error_mark = '' error_mark = ''
if check_type == 'length': if check_type == 'length':
@@ -347,6 +393,7 @@ class SettingsLoader:
continue continue
return valid return valid
# MARK: load config file data from file
def __load_config_file(self) -> configparser.ConfigParser | None: def __load_config_file(self) -> configparser.ConfigParser | None:
""" """
load and parse the config file load and parse the config file
@@ -358,13 +405,14 @@ class SettingsLoader:
return config return config
return None return None
# MARK: regex clean up one
def __clean_invalid_setting( def __clean_invalid_setting(
self, self,
entry: str, entry: str,
validate: str, validate: str,
value: str, value: str,
regex: str, regex: str,
regex_clean: str, regex_clean: str | None,
replace: str = "", replace: str = "",
print_error: bool = True, print_error: bool = True,
) -> str: ) -> str:
@@ -380,18 +428,25 @@ class SettingsLoader:
replace (str): replace with character. Defaults to '' replace (str): replace with character. Defaults to ''
print_error (bool): print the error message. Defaults to True print_error (bool): print the error message. Defaults to True
""" """
check = re.compile(regex) check = re.compile(regex, re.VERBOSE)
clean = re.compile(regex_clean) clean: re.Pattern[str] | None = None
if not check.search(value): if regex_clean is not None:
clean = re.compile(regex_clean, re.VERBOSE)
# value must be set if clean is None, else empty value is allowed and will fail
if (clean is None and value or clean) and not check.search(value):
self.__print( self.__print(
f"[!] Invalid content for '{entry}' with check '{validate}' and data: {value}", f"[!] Invalid content for '{entry}' with check '{validate}' and data: {value}",
'ERROR', print_error 'ERROR', print_error
) )
# clean up # clean up if clean up is not none, else return EMPTY string
return clean.sub(replace, value) if clean is not None:
return clean.sub(replace, value)
self._check_settings_abort = True
return ''
# else return as is # else return as is
return value return value
# MARK: check settings, regx
def __check_settings( def __check_settings(
self, self,
check: str, entry: str, setting_value: list[str] | str check: str, entry: str, setting_value: list[str] | str
@@ -413,11 +468,10 @@ class SettingsLoader:
# get the check settings # get the check settings
__check_settings = SettingsLoaderCheck.CHECK_SETTINGS.get(check) __check_settings = SettingsLoaderCheck.CHECK_SETTINGS.get(check)
if __check_settings is None: if __check_settings is None:
self.__print( raise ValueError(self.__print(
f"[{entry}] Cannot get SettingsLoaderCheck.CHECK_SETTINGS for {check}", f"[{entry}] Cannot get SettingsLoaderCheck.CHECK_SETTINGS for {check}",
'CRITICAL', raise_exception=True 'CRITICAL'
) ))
sys.exit(1)
# either removes or replaces invalid characters in the list # either removes or replaces invalid characters in the list
if isinstance(setting_value, list): if isinstance(setting_value, list):
# clean up invalid characters # clean up invalid characters
@@ -439,6 +493,7 @@ class SettingsLoader:
# return data # return data
return setting_value return setting_value
# MARK: check arguments, for config file load fail
def __check_arguments(self, arguments: dict[str, list[str]], all_set: bool = False) -> bool: def __check_arguments(self, arguments: dict[str, list[str]], all_set: bool = False) -> bool:
""" """
check if ast least one argument is set check if ast least one argument is set
@@ -468,6 +523,7 @@ class SettingsLoader:
return has_argument return has_argument
# MARK: get argument from args dict
def __get_arg(self, entry: str) -> Any: def __get_arg(self, entry: str) -> Any:
""" """
check if an argument entry xists, if None -> returns None else value of argument check if an argument entry xists, if None -> returns None else value of argument
@@ -482,7 +538,8 @@ class SettingsLoader:
return None return None
return self.args.get(entry) return self.args.get(entry)
def __print(self, msg: str, level: str, print_error: bool = True, raise_exception: bool = False): # MARK: error print
def __print(self, msg: str, level: str, print_error: bool = True) -> str:
""" """
print out error, if Log class is set then print to log instead print out error, if Log class is set then print to log instead
@@ -496,12 +553,11 @@ class SettingsLoader:
if self.log is not None: if self.log is not None:
if not Log.validate_log_level(level): if not Log.validate_log_level(level):
level = 'ERROR' level = 'ERROR'
self.log.logger.log(Log.get_log_level_int(level), msg) self.log.logger.log(Log.get_log_level_int(level), msg, stacklevel=2)
if self.log is None or self.always_print: if self.log is None or self.always_print:
if print_error: if print_error:
print(msg) print(msg)
if raise_exception: return msg
raise ValueError(msg)
# __END__ # __END__

View File

@@ -3,12 +3,17 @@ Class of checks that can be run on value entries
""" """
from typing import TypedDict from typing import TypedDict
from corelibs.check_handling.regex_constants import (
EMAIL_BASIC_REGEX, DOMAIN_WITH_LOCALHOST_REGEX, DOMAIN_WITH_LOCALHOST_PORT_REGEX, DOMAIN_REGEX
)
class SettingsLoaderCheckValue(TypedDict): class SettingsLoaderCheckValue(TypedDict):
"""Settings check entries""" """Settings check entries"""
regex: str regex: str
regex_clean: str # if None, then on error we exit, eles we clean up data
regex_clean: str | None
replace: str replace: str
@@ -16,29 +21,61 @@ class SettingsLoaderCheck:
""" """
check:<NAME> or check:list+<NAME> check:<NAME> or check:list+<NAME>
""" """
CHECK_SETTINGS: dict[str, SettingsLoaderCheckValue] = { CHECK_SETTINGS: dict[str, SettingsLoaderCheckValue] = {
"int": { "int": {
"regex": r"^[0-9]+$", "regex": r"^[0-9]+$",
"regex_clean": r"[^0-9]", "regex_clean": r"[^0-9]",
"replace": "" "replace": "",
}, },
"string.alphanumeric": { "string.alphanumeric": {
"regex": r"^[a-zA-Z0-9]+$", "regex": r"^[a-zA-Z0-9]+$",
"regex_clean": r"[^a-zA-Z0-9]", "regex_clean": r"[^a-zA-Z0-9]",
"replace": "" "replace": "",
}, },
"string.alphanumeric.lower.dash": { "string.alphanumeric.lower.dash": {
"regex": r"^[a-z0-9-]+$", "regex": r"^[a-z0-9-]+$",
"regex_clean": r"[^a-z0-9-]", "regex_clean": r"[^a-z0-9-]",
"replace": "" "replace": "",
}, },
# A-Z a-z 0-9 _ - . ONLY # A-Z a-z 0-9 _ - . ONLY
# This one does not remove, but replaces with _ # This one does not remove, but replaces with _
"string.alphanumeric.extended.replace": { "string.alphanumeric.extended.replace": {
"regex": r"^[_.a-zA-Z0-9-]+$", "regex": r"^[_.a-zA-Z0-9-]+$",
"regex_clean": r"[^_.a-zA-Z0-9-]", "regex_clean": r"[^_.a-zA-Z0-9-]",
"replace": "_" "replace": "_",
}, },
# This does a baisc email check, only alphanumeric with special characters
"string.email.basic": {
"regex": EMAIL_BASIC_REGEX,
"regex_clean": None,
"replace": "",
},
# Domain check, including localhost no port
"string.domain.with-localhost": {
"regex": DOMAIN_WITH_LOCALHOST_REGEX,
"regex_clean": None,
"replace": "",
},
# Domain check, with localhost and port
"string.domain.with-localhost.port": {
"regex": DOMAIN_WITH_LOCALHOST_PORT_REGEX,
"regex_clean": None,
"replace": "",
},
# Domain check, no pure localhost allowed
"string.domain": {
"regex": DOMAIN_REGEX,
"regex_clean": None,
"replace": "",
},
# Basic date check, does not validate date itself
"string.date": {
"regex": r"^\d{4}[/-]\d{1,2}[/-]\d{1,2}$",
"regex_clean": None,
"replace": "",
}
} }
# __END__ # __END__

View File

@@ -3,6 +3,9 @@ Dict helpers
""" """
from typing import Any
def mask( def mask(
data_set: dict[str, str], data_set: dict[str, str],
mask_keys: list[str] | None = None, mask_keys: list[str] | None = None,
@@ -34,4 +37,22 @@ def mask(
for key, value in data_set.items() for key, value in data_set.items()
} }
def set_entry(dict_set: dict[str, Any], key: str, value_set: Any) -> dict[str, Any]:
"""
set a new entry in the dict set
Arguments:
key {str} -- _description_
dict_set {dict[str, Any]} -- _description_
value_set {Any} -- _description_
Returns:
dict[str, Any] -- _description_
"""
if not dict_set.get(key):
dict_set[key] = {}
dict_set[key] = value_set
return dict_set
# __END__ # __END__

View File

@@ -74,8 +74,8 @@ class CustomConsoleFormatter(logging.Formatter):
return f"{color}{message}{reset}" return f"{color}{message}{reset}"
# TODO: add custom handlers for stack_correct, if not set fill with %(filename)s:%(funcName)s:%(lineno)d # TODO: add custom handlers for stack_trace, if not set fill with %(filename)s:%(funcName)s:%(lineno)d
# hasattr(record, 'stack_correct') # hasattr(record, 'stack_trace')
# MARK: Log class # MARK: Log class
@@ -367,7 +367,7 @@ class Log:
return root_logger return root_logger
# FIXME: all below will only work if we add a custom format interface for the stack_correct part # FIXME: we need to add a custom formater to add stack level listing if we want to
# Important note, although they exist, it is recommended to use self.logger.NAME directly # Important note, although they exist, it is recommended to use self.logger.NAME directly
# so that the correct filename, method and row number is set # so that the correct filename, method and row number is set
# for > 50 use logger.log(LoggingLevel.<LEVEL>.value, ...) # for > 50 use logger.log(LoggingLevel.<LEVEL>.value, ...)
@@ -379,8 +379,8 @@ class Log:
raise ValueError('Logger is not yet initialized') raise ValueError('Logger is not yet initialized')
if extra is None: if extra is None:
extra = {} extra = {}
extra['stack_correct'] = traceback_call_str(start=3) extra['stack_trace'] = traceback_call_str(start=3)
self.logger.log(level, msg, *args, extra=extra) self.logger.log(level, msg, *args, extra=extra, stacklevel=2)
# MARK: DEBUG 10 # MARK: DEBUG 10
def debug(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None: def debug(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -389,8 +389,8 @@ class Log:
raise ValueError('Logger is not yet initialized') raise ValueError('Logger is not yet initialized')
if extra is None: if extra is None:
extra = {} extra = {}
extra['stack_correct'] = traceback_call_str(start=3) extra['stack_trace'] = traceback_call_str(start=3)
self.logger.debug(msg, *args, extra=extra) self.logger.debug(msg, *args, extra=extra, stacklevel=2)
# MARK: INFO 20 # MARK: INFO 20
def info(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None: def info(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -399,8 +399,8 @@ class Log:
raise ValueError('Logger is not yet initialized') raise ValueError('Logger is not yet initialized')
if extra is None: if extra is None:
extra = {} extra = {}
extra['stack_correct'] = traceback_call_str(start=3) extra['stack_trace'] = traceback_call_str(start=3)
self.logger.info(msg, *args, extra=extra) self.logger.info(msg, *args, extra=extra, stacklevel=2)
# MARK: WARNING 30 # MARK: WARNING 30
def warning(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None: def warning(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -409,8 +409,8 @@ class Log:
raise ValueError('Logger is not yet initialized') raise ValueError('Logger is not yet initialized')
if extra is None: if extra is None:
extra = {} extra = {}
extra['stack_correct'] = traceback_call_str(start=3) extra['stack_trace'] = traceback_call_str(start=3)
self.logger.warning(msg, *args, extra=extra) self.logger.warning(msg, *args, extra=extra, stacklevel=2)
# MARK: ERROR 40 # MARK: ERROR 40
def error(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None: def error(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -419,8 +419,8 @@ class Log:
raise ValueError('Logger is not yet initialized') raise ValueError('Logger is not yet initialized')
if extra is None: if extra is None:
extra = {} extra = {}
extra['stack_correct'] = traceback_call_str(start=3) extra['stack_trace'] = traceback_call_str(start=3)
self.logger.error(msg, *args, extra=extra) self.logger.error(msg, *args, extra=extra, stacklevel=2)
# MARK: CRITICAL 50 # MARK: CRITICAL 50
def critical(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None: def critical(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -429,8 +429,8 @@ class Log:
raise ValueError('Logger is not yet initialized') raise ValueError('Logger is not yet initialized')
if extra is None: if extra is None:
extra = {} extra = {}
extra['stack_correct'] = traceback_call_str(start=3) extra['stack_trace'] = traceback_call_str(start=3)
self.logger.critical(msg, *args, extra=extra) self.logger.critical(msg, *args, extra=extra, stacklevel=2)
# MARK: ALERT 55 # MARK: ALERT 55
def alert(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None: def alert(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -440,8 +440,8 @@ class Log:
# extra_dict = dict(extra) # extra_dict = dict(extra)
if extra is None: if extra is None:
extra = {} extra = {}
extra['stack_correct'] = traceback_call_str(start=3) extra['stack_trace'] = traceback_call_str(start=3)
self.logger.log(LoggingLevel.ALERT.value, msg, *args, extra=extra) self.logger.log(LoggingLevel.ALERT.value, msg, *args, extra=extra, stacklevel=2)
# MARK: EMERGECNY: 60 # MARK: EMERGECNY: 60
def emergency(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None: def emergency(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -450,8 +450,8 @@ class Log:
raise ValueError('Logger is not yet initialized') raise ValueError('Logger is not yet initialized')
if extra is None: if extra is None:
extra = {} extra = {}
extra['stack_correct'] = traceback_call_str(start=3) extra['stack_trace'] = traceback_call_str(start=3)
self.logger.log(LoggingLevel.EMERGENCY.value, msg, *args, extra=extra) self.logger.log(LoggingLevel.EMERGENCY.value, msg, *args, extra=extra, stacklevel=2)
# MARK: EXCEPTION: 70 # MARK: EXCEPTION: 70
def exception(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None: def exception(self, msg: object, *args: object, extra: MutableMapping[str, object] | None = None) -> None:
@@ -467,8 +467,8 @@ class Log:
raise ValueError('Logger is not yet initialized') raise ValueError('Logger is not yet initialized')
if extra is None: if extra is None:
extra = {} extra = {}
extra['stack_correct'] = traceback_call_str(start=3) extra['stack_trace'] = traceback_call_str(start=3)
self.logger.log(LoggingLevel.EXCEPTION.value, msg, *args, exc_info=True, extra=extra) self.logger.log(LoggingLevel.EXCEPTION.value, msg, *args, exc_info=True, extra=extra, stacklevel=2)
# MARK: break line # MARK: break line
def break_line(self, info: str = "BREAK"): def break_line(self, info: str = "BREAK"):
@@ -527,7 +527,7 @@ class Log:
# MARK: log level handling # MARK: log level handling
def set_log_level(self, handler_name: str, log_level: LoggingLevel) -> bool: def set_log_level(self, handler_name: str, log_level: LoggingLevel) -> bool:
""" """
set the logging level set the logging level for a handler
Arguments: Arguments:
handler {str} -- _description_ handler {str} -- _description_
@@ -555,7 +555,7 @@ class Log:
def get_log_level(self, handler_name: str) -> LoggingLevel: def get_log_level(self, handler_name: str) -> LoggingLevel:
""" """
gett the logging level for a handler gettthe logging level for a handler
Arguments: Arguments:
handler_name {str} -- _description_ handler_name {str} -- _description_
@@ -571,7 +571,7 @@ class Log:
@staticmethod @staticmethod
def validate_log_level(log_level: Any) -> bool: def validate_log_level(log_level: Any) -> bool:
""" """
if the log level is invalid, will erturn false if the log level is invalid will return false, else return true
Args: Args:
log_level (Any): _description_ log_level (Any): _description_
@@ -589,7 +589,7 @@ class Log:
def get_log_level_int(log_level: Any) -> int: def get_log_level_int(log_level: Any) -> int:
""" """
Return log level as INT Return log level as INT
If invalid return set level in default log level If invalid returns the default log level
Arguments: Arguments:
log_level {Any} -- _description_ log_level {Any} -- _description_

View File

@@ -1,6 +1,7 @@
[TestA] [TestA]
foo=bar foo=bar
foobar=1 foobar=1
bar=st
some_match=foo some_match=foo
some_match_list=foo,bar some_match_list=foo,bar
test_list=a,b,c,d f, g h test_list=a,b,c,d f, g h
@@ -8,6 +9,8 @@ other_list=a|b|c|d|
third_list=xy|ab|df|fg third_list=xy|ab|df|fg
str_length=foobar str_length=foobar
int_range=20 int_range=20
int_range_not_set=
int_range_not_set_empty_set=5
# #
match_target=foo match_target=foo
match_target_list=foo,bar,baz match_target_list=foo,bar,baz
@@ -21,3 +24,6 @@ match_source_list=foo,bar
element_a=Static energy element_a=Static energy
element_b=123.5 element_b=123.5
element_c=True element_c=True
email=foo@bar.com,other+bar-fee@domain-com.cp,
email_not_mandatory=
email_bad=gii@bar.com

View File

@@ -2,10 +2,12 @@
Settings loader test Settings loader test
""" """
import re
from pathlib import Path from pathlib import Path
from corelibs.iterator_handling.dump_data import dump_data from corelibs.debug_handling.dump_data import dump_data
from corelibs.logging_handling.log import Log from corelibs.logging_handling.log import Log
from corelibs.config_handling.settings_loader import SettingsLoader from corelibs.config_handling.settings_loader import SettingsLoader
from corelibs.config_handling.settings_loader_handling.settings_loader_check import SettingsLoaderCheck
SCRIPT_PATH: Path = Path(__file__).resolve().parent SCRIPT_PATH: Path = Path(__file__).resolve().parent
ROOT_PATH: Path = SCRIPT_PATH ROOT_PATH: Path = SCRIPT_PATH
@@ -18,6 +20,11 @@ def main():
Main run Main run
""" """
value = "2025/1/1"
regex_c = re.compile(SettingsLoaderCheck.CHECK_SETTINGS['string.date']['regex'], re.VERBOSE)
result = regex_c.search(value)
print(f"regex {regex_c} check against {value} -> {result}")
# for log testing # for log testing
script_path: Path = Path(__file__).resolve().parent script_path: Path = Path(__file__).resolve().parent
log = Log( log = Log(
@@ -42,8 +49,10 @@ def main():
config_data = sl.load_settings( config_data = sl.load_settings(
config_load, config_load,
{ {
# "doesnt": ["split:,"],
"foo": ["mandatory:yes"], "foo": ["mandatory:yes"],
"foobar": ["check:int"], "foobar": ["check:int"],
"bar": ["mandatory:yes"],
"some_match": ["matching:foo|bar"], "some_match": ["matching:foo|bar"],
"some_match_list": ["split:,", "matching:foo|bar"], "some_match_list": ["split:,", "matching:foo|bar"],
"test_list": [ "test_list": [
@@ -61,6 +70,12 @@ def main():
"int_range": [ "int_range": [
"range:2-50" "range:2-50"
], ],
"int_range_not_set": [
"range:2-50"
],
"int_range_not_set_empty_set": [
"empty:"
],
"match_target": ["matching:foo"], "match_target": ["matching:foo"],
"match_target_list": ["split:,", "matching:foo|bar|baz",], "match_target_list": ["split:,", "matching:foo|bar|baz",],
"match_source_a": ["in:match_target"], "match_source_a": ["in:match_target"],
@@ -68,7 +83,33 @@ def main():
"match_source_list": ["split:,", "in:match_target_list"], "match_source_list": ["split:,", "in:match_target_list"],
} }
) )
print(f"Load: {config_load} -> {dump_data(config_data)}") print(f"[{config_load}] Load: {config_load} -> {dump_data(config_data)}")
except ValueError as e:
print(f"Could not load settings: {e}")
try:
config_load = 'TestB'
config_data = sl.load_settings(
config_load,
{
"email": [
"split:,",
"mandatory:yes",
"check:string.email.basic"
],
"email_not_mandatory": [
"split:,",
# "mandatory:yes",
"check:string.email.basic"
],
"email_bad": [
"split:,",
"mandatory:yes",
"check:string.email.basic"
]
}
)
print(f"[{config_load}] Load: {config_load} -> {dump_data(config_data)}")
except ValueError as e: except ValueError as e:
print(f"Could not load settings: {e}") print(f"Could not load settings: {e}")

View File

@@ -18,7 +18,8 @@ def main():
log_path=script_path.joinpath('log', 'test.log'), log_path=script_path.joinpath('log', 'test.log'),
log_name="Test Log", log_name="Test Log",
log_settings={ log_settings={
"log_level_console": 'DEBUG', # "log_level_console": 'DEBUG',
"log_level_console": None,
"log_level_file": 'DEBUG', "log_level_file": 'DEBUG',
# "console_color_output_enabled": False, # "console_color_output_enabled": False,
} }

2
uv.lock generated
View File

@@ -44,7 +44,7 @@ wheels = [
[[package]] [[package]]
name = "corelibs" name = "corelibs"
version = "0.11.0" version = "0.12.6"
source = { editable = "." } source = { editable = "." }
dependencies = [ dependencies = [
{ name = "jmespath" }, { name = "jmespath" },