v0.46.0: Add unique list helper function

Also add prefix "[SettingsLoader] " to print statements in SettingsLoader if we do not write to log

pyproject.toml

@@ -1,7 +1,7 @@
 # MARK: Project info
 [project]
 name = "corelibs"
-version = "0.45.0"
+version = "0.46.0"
 description = "Collection of utils for Python scripts"
 readme = "README.md"
 requires-python = ">=3.13"

src/corelibs/config_handling/settings_loader.py

@@ -577,7 +577,7 @@ class SettingsLoader:
             self.log.logger.log(Log.get_log_level_int(level), msg, stacklevel=2)
         if self.log is None or self.always_print:
             if print_error:
-                print(msg)
+                print(f"[SettingsLoader] {msg}")
         if level == 'ERROR':
             # remove any prefix [!] for error message list
             self.__error_msg.append(msg.replace('[!] ', '').strip())

src/corelibs/iterator_handling/list_helpers.py

@@ -2,6 +2,7 @@
 List type helpers
 """
 
+import json
 from typing import Any, Sequence
 
 
@@ -44,4 +45,26 @@ def is_list_in_list(
     # Get the difference and extract just the values
     return [item for item, _ in set_a - set_b]
 
+
+def make_unique_list_of_dicts(dict_list: list[Any]) -> list[Any]:
+    """
+    Create a list of unique dictionary entries
+
+    Arguments:
+        dict_list {list[Any]} -- _description_
+
+    Returns:
+        list[Any] -- _description_
+    """
+    try:
+        # try json dumps, can fail with int and str index types
+        return list({json.dumps(d, sort_keys=True, ensure_ascii=True): d for d in dict_list}.values())
+    except TypeError:
+        # Fallback for non-serializable entries, slow but works
+        unique: list[Any] = []
+        for d in dict_list:
+            if d not in unique:
+                unique.append(d)
+        return unique
+
 # __END__

src/corelibs/logging_handling/log.py

@@ -602,9 +602,9 @@ class Log(LogParent):
                 __setting = self.DEFAULT_LOG_SETTINGS.get(__log_entry, True)
             default_log_settings[__log_entry] = __setting
         # check console log type
-        default_log_settings['console_format_type'] = cast('ConsoleFormat', log_settings.get(
-            'console_format_type', self.DEFAULT_LOG_SETTINGS['console_format_type']
-        ))
+        if (console_format_type := log_settings.get('console_format_type')) is None:
+            console_format_type = self.DEFAULT_LOG_SETTINGS['console_format_type']
+        default_log_settings['console_format_type'] = cast('ConsoleFormat', console_format_type)
         # check log queue
         __setting = log_settings.get('log_queue', self.DEFAULT_LOG_SETTINGS['log_queue'])
         if __setting is not None:

test-run/config_handling/config/settings.ini

@@ -17,6 +17,7 @@ str_length=foobar
 int_range=20
 int_range_not_set=
 int_range_not_set_empty_set=5
+bool_var=True
 #
 match_target=foo
 match_target_list=foo,bar,baz

test-run/config_handling/settings_loader.py

@@ -84,6 +84,7 @@ def main():
                 "int_range_not_set_empty_set": [
                     "empty:"
                 ],
+                "bool_var": ["convert:bool"],
                 "match_target": ["matching:foo"],
                 "match_target_list": ["split:,", "matching:foo|bar|baz",],
                 "match_source_a": ["in:match_target"],

test-run/iterator_handling/list_helpers.py

@@ -2,7 +2,9 @@
 test list helpers
 """
 
-from corelibs.iterator_handling.list_helpers import is_list_in_list, convert_to_list
+# from typing import Any
+from corelibs.debug_handling.dump_data import dump_data
+from corelibs.iterator_handling.list_helpers import is_list_in_list, convert_to_list, make_unique_list_of_dicts
 
 
 def __test_is_list_in_list_a():
@@ -18,9 +20,40 @@ def __convert_list():
     print(f"IN: {source} -> {result}")
 
 
+def __make_unique_list_of_dicts():
+    dict_list = [
+        {"a": 1, "b": 2, "nested": {"x": 10, "y": 20}},
+        {"a": 1, "b": 2, "nested": {"x": 10, "y": 20}},
+        {"b": 2, "a": 1, "nested": {"y": 20, "x": 10}},
+        {"b": 2, "a": 1, "nested": {"y": 20, "x": 30}},
+        {"a": 3, "b": 4, "nested": {"x": 30, "y": 40}}
+    ]
+    unique_dicts = make_unique_list_of_dicts(dict_list)
+    print(f"Unique dicts: {dump_data(unique_dicts)}")
+
+    dict_list = [
+        {"a": 1, 1: "one"},
+        {1: "one", "a": 1},
+        {"a": 2, 1: "one"}
+    ]
+    unique_dicts = make_unique_list_of_dicts(dict_list)
+    print(f"Unique dicts: {dump_data(unique_dicts)}")
+
+    dict_list = [
+        {"a": 1, "b": [1, 2, 3]},
+        {"b": [1, 2, 3], "a": 1},
+        {"a": 1, "b": [1, 2, 4]},
+        1, 2, "String", 1, "Foobar"
+    ]
+    unique_dicts = make_unique_list_of_dicts(dict_list)
+    print(f"Unique dicts: {dump_data(unique_dicts)}")
+
+
 def main():
+    """List helpers test runner"""
     __test_is_list_in_list_a()
     __convert_list()
+    __make_unique_list_of_dicts()
 
 
 if __name__ == "__main__":

test-run/logging_handling/log.py

@@ -27,7 +27,8 @@ def main():
             "per_run_log": True,
             # "console_format_type": ConsoleFormatSettings.NONE,
             # "console_format_type": ConsoleFormatSettings.MINIMAL,
-            "console_format_type": ConsoleFormat.TIME_MICROSECONDS | ConsoleFormat.NAME | ConsoleFormat.LEVEL,
+            # "console_format_type": ConsoleFormat.TIME_MICROSECONDS | ConsoleFormat.NAME | ConsoleFormat.LEVEL,
+            "console_format_type": None,
             # "console_format_type": ConsoleFormat.NAME,
             # "console_format_type": (
             #     ConsoleFormat.TIME | ConsoleFormat.TIMEZONE | ConsoleFormat.LINENO | ConsoleFormat.LEVEL

tests/unit/iterator_handling/test_list_helpers.py

@@ -4,7 +4,7 @@ iterator_handling.list_helepr tests
 
 from typing import Any
 import pytest
-from corelibs.iterator_handling.list_helpers import convert_to_list, is_list_in_list
+from corelibs.iterator_handling.list_helpers import convert_to_list, is_list_in_list, make_unique_list_of_dicts
 
 
 class TestConvertToList:
@@ -298,3 +298,225 @@ class TestPerformance:
         # Should still work correctly despite duplicates
         assert set(result) == {1, 3}
         assert isinstance(result, list)
+
+
+class TestMakeUniqueListOfDicts:
+    """Test cases for make_unique_list_of_dicts function"""
+
+    def test_basic_duplicate_removal(self):
+        """Test basic removal of duplicate dictionaries"""
+        dict_list = [
+            {"a": 1, "b": 2},
+            {"a": 1, "b": 2},
+            {"a": 3, "b": 4}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 2
+        assert {"a": 1, "b": 2} in result
+        assert {"a": 3, "b": 4} in result
+
+    def test_order_independent_duplicates(self):
+        """Test that dictionaries with different key orders are treated as duplicates"""
+        dict_list = [
+            {"a": 1, "b": 2},
+            {"b": 2, "a": 1},  # Same content, different order
+            {"a": 3, "b": 4}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 2
+        assert {"a": 1, "b": 2} in result
+        assert {"a": 3, "b": 4} in result
+
+    def test_empty_list(self):
+        """Test with empty list"""
+        result = make_unique_list_of_dicts([])
+        assert result == []
+        assert isinstance(result, list)
+
+    def test_single_dict(self):
+        """Test with single dictionary"""
+        dict_list = [{"a": 1, "b": 2}]
+        result = make_unique_list_of_dicts(dict_list)
+        assert result == [{"a": 1, "b": 2}]
+
+    def test_all_unique(self):
+        """Test when all dictionaries are unique"""
+        dict_list = [
+            {"a": 1},
+            {"b": 2},
+            {"c": 3},
+            {"d": 4}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 4
+        for d in dict_list:
+            assert d in result
+
+    def test_all_duplicates(self):
+        """Test when all dictionaries are duplicates"""
+        dict_list = [
+            {"a": 1, "b": 2},
+            {"a": 1, "b": 2},
+            {"a": 1, "b": 2},
+            {"b": 2, "a": 1}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 1
+        assert result[0] == {"a": 1, "b": 2}
+
+    def test_nested_values(self):
+        """Test with nested structures as values"""
+        dict_list = [
+            {"a": [1, 2], "b": 3},
+            {"a": [1, 2], "b": 3},
+            {"a": [1, 3], "b": 3}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 2
+        assert {"a": [1, 2], "b": 3} in result
+        assert {"a": [1, 3], "b": 3} in result
+
+    def test_different_value_types(self):
+        """Test with different value types"""
+        dict_list = [
+            {"str": "hello", "int": 42, "float": 3.14, "bool": True},
+            {"str": "hello", "int": 42, "float": 3.14, "bool": True},
+            {"str": "world", "int": 99, "float": 2.71, "bool": False}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 2
+
+    def test_empty_dicts(self):
+        """Test with empty dictionaries"""
+        dict_list: list[Any] = [
+            {},
+            {},
+            {"a": 1}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 2
+        assert {} in result
+        assert {"a": 1} in result
+
+    def test_single_key_dicts(self):
+        """Test with single key dictionaries"""
+        dict_list = [
+            {"a": 1},
+            {"a": 1},
+            {"a": 2},
+            {"b": 1}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 3
+        assert {"a": 1} in result
+        assert {"a": 2} in result
+        assert {"b": 1} in result
+
+    def test_many_keys(self):
+        """Test with dictionaries containing many keys"""
+        dict1 = {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5}
+        dict2 = {"e": 5, "d": 4, "c": 3, "b": 2, "a": 1}  # Same, different order
+        dict3 = {"a": 1, "b": 2, "c": 3, "d": 4, "e": 6}  # Different value
+        dict_list = [dict1, dict2, dict3]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 2
+
+    def test_numeric_keys(self):
+        """Test with numeric keys"""
+        dict_list = [
+            {1: "one", 2: "two"},
+            {2: "two", 1: "one"},
+            {1: "one", 2: "three"}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 2
+
+    def test_none_values(self):
+        """Test with None values"""
+        dict_list = [
+            {"a": None, "b": 2},
+            {"a": None, "b": 2},
+            {"a": 1, "b": None}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 2
+        assert {"a": None, "b": 2} in result
+        assert {"a": 1, "b": None} in result
+
+    def test_mixed_key_types(self):
+        """Test with mixed key types (string and numeric)"""
+        dict_list = [
+            {"a": 1, 1: "one"},
+            {1: "one", "a": 1},
+            {"a": 2, 1: "one"}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 2
+
+    @pytest.mark.parametrize("dict_list,expected_length", [
+        ([{"a": 1}, {"a": 1}, {"a": 1}], 1),
+        ([{"a": 1}, {"a": 2}, {"a": 3}], 3),
+        ([{"a": 1, "b": 2}, {"b": 2, "a": 1}], 1),
+        ([{}, {}], 1),
+        ([{"x": [1, 2]}, {"x": [1, 2]}], 1),
+        ([{"a": 1}, {"b": 2}, {"c": 3}], 3),
+    ])  # pyright: ignore[reportUnknownArgumentType]
+    def test_parametrized_unique_dicts(self, dict_list: list[Any], expected_length: int):
+        """Test make_unique_list_of_dicts with various input combinations"""
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == expected_length
+        assert isinstance(result, list)
+
+    def test_large_list(self):
+        """Test with a large list of dictionaries"""
+        dict_list = [{"id": i % 100, "value": f"val_{i % 100}"} for i in range(1000)]
+        result = make_unique_list_of_dicts(dict_list)
+        # Should have 100 unique dicts (0-99)
+        assert len(result) == 100
+
+    def test_preserves_last_occurrence(self):
+        """Test behavior with duplicate entries"""
+        # The function uses dict comprehension, which keeps last occurrence
+        dict_list = [
+            {"a": 1, "b": 2},
+            {"a": 3, "b": 4},
+            {"a": 1, "b": 2}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 2
+        # Just verify correct unique count, order may vary
+
+    def test_nested_dicts(self):
+        """Test with nested dictionaries"""
+        dict_list = [
+            {"outer": {"inner": 1}},
+            {"outer": {"inner": 1}},
+            {"outer": {"inner": 2}}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 2
+
+    def test_string_values_case_sensitive(self):
+        """Test that string values are case-sensitive"""
+        dict_list = [
+            {"name": "John"},
+            {"name": "john"},
+            {"name": "JOHN"},
+            {"name": "John"}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 3
+
+    def test_boolean_values(self):
+        """Test with boolean values"""
+        dict_list = [
+            {"flag": True, "count": 1},
+            {"count": 1, "flag": True},
+            {"flag": False, "count": 1}
+        ]
+        result = make_unique_list_of_dicts(dict_list)
+        assert len(result) == 2
+        assert {"flag": True, "count": 1} in result
+        assert {"flag": False, "count": 1} in result
+
+# __END__

tests/unit/logging_handling/log_testing/test_log_3_custom_console_formatter.py

@@ -28,6 +28,7 @@ def tmp_log_path(tmp_path: Path) -> Path:
 @pytest.fixture
 def basic_log_settings() -> LogSettings:
     """Basic log settings for testing"""
+    # Return a new dict each time to avoid state pollution
     return {
         "log_level_console": LoggingLevel.WARNING,
         "log_level_file": LoggingLevel.DEBUG,
@@ -308,4 +309,54 @@ class TestUpdateConsoleFormatter:
         # Verify message was logged
         assert "Test warning message" in caplog.text
 
+    def test_log_console_format_option_set_to_none(
+        self, tmp_log_path: Path
+    ):
+        """Test that when log_console_format option is set to None, it uses ConsoleFormatSettings.ALL"""
+        # Save the original DEFAULT_LOG_SETTINGS to restore it after test
+        original_default = Log.DEFAULT_LOG_SETTINGS.copy()
+
+        try:
+            # Reset DEFAULT_LOG_SETTINGS to ensure clean state
+            Log.DEFAULT_LOG_SETTINGS = {
+                "log_level_console": Log.DEFAULT_LOG_LEVEL_CONSOLE,
+                "log_level_file": Log.DEFAULT_LOG_LEVEL_FILE,
+                "per_run_log": False,
+                "console_enabled": True,
+                "console_color_output_enabled": True,
+                "console_format_type": ConsoleFormatSettings.ALL,
+                "add_start_info": True,
+                "add_end_info": False,
+                "log_queue": None,
+            }
+
+            # Create a fresh settings dict with console_format_type explicitly set to None
+            settings: LogSettings = {
+                "log_level_console": LoggingLevel.WARNING,
+                "log_level_file": LoggingLevel.DEBUG,
+                "per_run_log": False,
+                "console_enabled": True,
+                "console_color_output_enabled": False,
+                "console_format_type": None,  # type: ignore
+                "add_start_info": False,
+                "add_end_info": False,
+                "log_queue": None,
+            }
+
+            # Verify that None is explicitly set in the input
+            assert settings['console_format_type'] is None
+
+            log = Log(
+                log_path=tmp_log_path,
+                log_name="test_log",
+                log_settings=settings
+            )
+
+            # Verify that None was replaced with ConsoleFormatSettings.ALL
+            # The Log class should replace None with the default value (ALL)
+            assert log.log_settings['console_format_type'] == ConsoleFormatSettings.ALL
+        finally:
+            # Restore original DEFAULT_LOG_SETTINGS
+            Log.DEFAULT_LOG_SETTINGS = original_default
+
 # __END__