Move iterator handling functions to corelibs_iterator, corelibs_hash and corelibs_dump_data modules

Deprecate math helpers in favor of built-in math functions

src/corelibs/file_handling/file_bom_encoding.py

@@ -25,7 +25,7 @@ def is_bom_encoded(file_path: Path) -> bool:
     return is_bom_encoding_ng(file_path)
 
 
-@deprecated("Use corelibs_file.file_bom_encoding.is_bom_encoded_info instead")
+@deprecated("Use corelibs_file.file_bom_encoding.get_bom_encoding_info instead")
 def is_bom_encoded_info(file_path: Path) -> BomEncodingInfo:
     """
     Enhanced BOM detection with additional file analysis

src/corelibs/iterator_handling/data_search.py

@@ -2,27 +2,31 @@
 wrapper around search path
 """
 
-from typing import Any, TypedDict, NotRequired
+from typing import Any
 from warnings import deprecated
+from corelibs_search.data_search import (
+    ArraySearchList as CorelibsArraySearchList,
+    find_in_array_from_list as corelibs_find_in_array_from_list,
+    key_lookup as corelibs_key_lookup,
+    value_lookup as corelibs_value_lookup
+)
 
 
-class ArraySearchList(TypedDict):
+class ArraySearchList(CorelibsArraySearchList):
     """find in array from list search dict"""
-    key: str
-    value: str | bool | int | float | list[str | None]
-    case_sensitive: NotRequired[bool]
 
 
-@deprecated("Use find_in_array_from_list()")
+@deprecated("Use corelibs_search.data_search.find_in_array_from_list instead")
 def array_search(
     search_params: list[ArraySearchList],
     data: list[dict[str, Any]],
     return_index: bool = False
 ) -> list[dict[str, Any]]:
     """depreacted, old call order"""
-    return find_in_array_from_list(data, search_params, return_index)
+    return corelibs_find_in_array_from_list(data, search_params, return_index)
 
 
+@deprecated("Use corelibs_search.data_search.find_in_array_from_list instead")
 def find_in_array_from_list(
     data: list[dict[str, Any]],
     search_params: list[ArraySearchList],
@@ -48,69 +52,14 @@ def find_in_array_from_list(
         list: list of found elements, or if return index
                 list of dics with "index" and "data", where "data" holds the result list
     """
-    if not isinstance(search_params, list):  # type: ignore
-        raise ValueError("search_params must be a list")
-    keys: list[str] = []
-    # check that key and value exist and are set
-    for search in search_params:
-        if not search.get('key') or not search.get('value'):
-            raise KeyError(
-                f"Either Key '{search.get('key', '')}' or "
-                f"Value '{search.get('value', '')}' is missing or empty"
-            )
-        # if double key -> abort
-        if search.get("key") in keys:
-            raise KeyError(
-                f"Key {search.get('key', '')} already exists in search_params"
-            )
-        keys.append(str(search['key']))
-
-    return_items: list[dict[str, Any]] = []
-    for si_idx, search_item in enumerate(data):
-        # for each search entry, all must match
-        matching = 0
-        for search in search_params:
-            # either Value direct or if Value is list then any of those items can match
-            # values are compared in lower case if case senstive is off
-            # lower case left side
-            # TODO: allow nested Keys. eg "Key: ["Key a", "key b"]" to be ["Key a"]["key b"]
-            if search.get("case_sensitive", True) is False:
-                search_value = search_item.get(str(search['key']), "").lower()
-            else:
-                search_value = search_item.get(str(search['key']), "")
-            # lower case right side
-            if isinstance(search['value'], list):
-                search_in = [
-                    str(k).lower()
-                    if search.get("case_sensitive", True) is False else k
-                    for k in search['value']
-                ]
-            elif search.get("case_sensitive", True) is False:
-                search_in = str(search['value']).lower()
-            else:
-                search_in = search['value']
-            # compare check
-            if (
-                (
-                    isinstance(search_in, list) and
-                    search_value in search_in
-                ) or
-                search_value == search_in
-            ):
-                matching += 1
-        if len(search_params) == matching:
-            if return_index is True:
-                # the data is now in "data sub set"
-                return_items.append({
-                    "index": si_idx,
-                    "data": search_item
-                })
-            else:
-                return_items.append(search_item)
-    # return all found or empty list
-    return return_items
+    return corelibs_find_in_array_from_list(
+        data,
+        search_params,
+        return_index
+    )
 
 
+@deprecated("Use corelibs_search.data_search.key_lookup instead")
 def key_lookup(haystack: dict[str, str], key: str) -> str:
     """
     simple key lookup in haystack, erturns empty string if not found
@@ -122,9 +71,10 @@ def key_lookup(haystack: dict[str, str], key: str) -> str:
     Returns:
         str: _description_
     """
-    return haystack.get(key, "")
+    return corelibs_key_lookup(haystack, key)
 
 
+@deprecated("Use corelibs_search.data_search.value_lookup instead")
 def value_lookup(haystack: dict[str, str], value: str, raise_on_many: bool = False) -> str:
     """
     find by value, if not found returns empty, if not raise on many returns the first one
@@ -140,11 +90,6 @@ def value_lookup(haystack: dict[str, str], value: str, raise_on_many: bool = Fal
     Returns:
         str: _description_
     """
-    keys = [__key for __key, __value in haystack.items() if __value == value]
-    if not keys:
-        return ""
-    if raise_on_many is True and len(keys) > 1:
-        raise ValueError("More than one element found with the same name")
-    return keys[0]
+    return corelibs_value_lookup(haystack, value, raise_on_many)
 
 # __END__

src/corelibs/iterator_handling/dict_helpers.py

@@ -2,9 +2,16 @@
 Various helper functions for type data clean up
 """
 
-from typing import Any, cast
+from warnings import deprecated
+from typing import Any
+from corelibs_iterator.dict_support import (
+    delete_keys_from_set as corelibs_delete_keys_from_set,
+    convert_to_dict_type,
+    set_entry as corelibs_set_entry
+)
 
 
+@deprecated("Use corelibs_iterator.dict_support.delete_keys_from_set instead")
 def delete_keys_from_set(
     set_data: dict[str, Any] | list[Any] | str, keys: list[str]
 ) -> dict[str, Any] | list[Any] | Any:
@@ -19,24 +26,10 @@ def delete_keys_from_set(
         dict[str, Any] | list[Any] | None: _description_
     """
     # skip everything if there is no keys list
-    if not keys:
-        return set_data
-    if isinstance(set_data, dict):
-        for key, value in set_data.copy().items():
-            if key in keys:
-                del set_data[key]
-            if isinstance(value, (dict, list)):
-                delete_keys_from_set(value, keys)  # type: ignore Partly unknown
-    elif isinstance(set_data, list):
-        for value in set_data:
-            if isinstance(value, (dict, list)):
-                delete_keys_from_set(value, keys)  # type: ignore Partly unknown
-    else:
-        set_data = [set_data]
-
-    return set_data
+    return corelibs_delete_keys_from_set(set_data, keys)
 
 
+@deprecated("Use corelibs_iterator.dict_support.convert_to_dict_type instead")
 def build_dict(
     any_dict: Any, ignore_entries: list[str] | None = None
 ) -> dict[str, Any | list[Any] | dict[Any, Any]]:
@@ -49,18 +42,10 @@ def build_dict(
     Returns:
         dict[str, Any | list[Any]]: _description_
     """
-    if ignore_entries is None:
-        return cast(dict[str, Any | list[Any] | dict[Any, Any]], any_dict)
-    # ignore entries can be one key or key nested
-    # return {
-    #     key: value for key, value in any_dict.items() if key not in ignore_entries
-    # }
-    return cast(
-        dict[str, Any | list[Any] | dict[Any, Any]],
-        delete_keys_from_set(any_dict, ignore_entries)
-    )
+    return convert_to_dict_type(any_dict, ignore_entries)
 
 
+@deprecated("Use corelibs_iterator.dict_support.set_entry instead")
 def set_entry(dict_set: dict[str, Any], key: str, value_set: Any) -> dict[str, Any]:
     """
     set a new entry in the dict set
@@ -73,9 +58,6 @@ def set_entry(dict_set: dict[str, Any], key: str, value_set: Any) -> dict[str, A
     Returns:
         dict[str, Any] -- _description_
     """
-    if not dict_set.get(key):
-        dict_set[key] = {}
-    dict_set[key] = value_set
-    return dict_set
+    return corelibs_set_entry(dict_set, key, value_set)
 
 # __END__

src/corelibs/iterator_handling/dict_mask.py

@@ -2,8 +2,11 @@
 Dict helpers
 """
 
-
-from typing import TypeAlias, Union, Dict, List, Any, cast
+from warnings import deprecated
+from typing import TypeAlias, Union, Dict, List, Any
+from corelibs_dump_data.dict_mask import (
+    mask as corelibs_mask
+)
 
 # definitions for the mask run below
 MaskableValue: TypeAlias = Union[str, int, float, bool, None]
@@ -11,6 +14,7 @@ NestedDict: TypeAlias = Dict[str, Union[MaskableValue, List[Any], 'NestedDict']]
 ProcessableValue: TypeAlias = Union[MaskableValue, List[Any], NestedDict]
 
 
+@deprecated("use corelibs_dump_data.dict_mask.mask instead")
 def mask(
     data_set: dict[str, Any],
     mask_keys: list[str] | None = None,
@@ -26,7 +30,7 @@ def mask(
     and end with '_', remove to search string in string
 
     Arguments:
-        data_set {dict[str, str]} -- _description_
+        data_set {dict[str, Any]} -- _description_
 
     Keyword Arguments:
         mask_keys {list[str] | None} -- _description_ (default: {None})
@@ -37,49 +41,12 @@ def mask(
     Returns:
         dict[str, str] -- _description_
     """
-    if skip is True:
-        return data_set
-    if mask_keys is None:
-        mask_keys = ["encryption", "password", "secret"]
-    else:
-        # make sure it is lower case
-        mask_keys = [mask_key.lower() for mask_key in mask_keys]
-
-    def should_mask_key(key: str) -> bool:
-        """Check if a key should be masked"""
-        __key_lower = key.lower()
-        return any(
-            __key_lower.startswith(mask_key) or
-            __key_lower.endswith(mask_key) or
-            f"{mask_str_edges}{mask_key}{mask_str_edges}" in __key_lower
-            for mask_key in mask_keys
-        )
-
-    def mask_recursive(obj: ProcessableValue) -> ProcessableValue:
-        """Recursively mask values in nested structures"""
-        if isinstance(obj, dict):
-            return {
-                key: mask_value(value) if should_mask_key(key) else mask_recursive(value)
-                for key, value in obj.items()
-            }
-        if isinstance(obj, list):
-            return [mask_recursive(item) for item in obj]
-        return obj
-
-    def mask_value(value: Any) -> Any:
-        """Handle masking based on value type"""
-        if isinstance(value, list):
-            # Mask each individual value in the list
-            return [mask_str for _ in cast('list[Any]', value)]
-        if isinstance(value, dict):
-            # Recursively process the dictionary instead of masking the whole thing
-            return mask_recursive(cast('ProcessableValue', value))
-        # Mask primitive values
-        return mask_str
-
-    return {
-        key: mask_value(value) if should_mask_key(key) else mask_recursive(value)
-        for key, value in data_set.items()
-    }
+    return corelibs_mask(
+        data_set,
+        mask_keys,
+        mask_str,
+        mask_str_edges,
+        skip
+    )
 
 # __END__

src/corelibs/iterator_handling/fingerprint.py

@@ -2,11 +2,16 @@
 Various dictionary, object and list hashers
 """
 
-import json
-import hashlib
-from typing import Any, cast, Sequence
+from warnings import deprecated
+from typing import Any
+from corelibs_hash.fingerprint import (
+    hash_object as corelibs_hash_object,
+    dict_hash_frozen as corelibs_dict_hash_frozen,
+    dict_hash_crc as corelibs_dict_hash_crc
+)
 
 
+@deprecated("use corelibs_hash.fingerprint.hash_object instead")
 def hash_object(obj: Any) -> str:
     """
     RECOMMENDED for new use
@@ -18,20 +23,10 @@ def hash_object(obj: Any) -> str:
     Returns:
         str -- _description_
     """
-    def normalize(o: Any) -> Any:
-        if isinstance(o, dict):
-            # Sort by repr of keys to handle mixed types (str, int, etc.)
-            o = cast(dict[Any, Any], o)
-            return tuple(sorted((repr(k), normalize(v)) for k, v in o.items()))
-        if isinstance(o, (list, tuple)):
-            o = cast(Sequence[Any], o)
-            return tuple(normalize(item) for item in o)
-        return repr(o)
-
-    normalized = normalize(obj)
-    return hashlib.sha256(str(normalized).encode()).hexdigest()
+    return corelibs_hash_object(obj)
 
 
+@deprecated("use corelibs_hash.fingerprint.hash_object instead")
 def dict_hash_frozen(data: dict[Any, Any]) -> int:
     """
     NOT RECOMMENDED, use dict_hash_crc or hash_object instead
@@ -44,9 +39,10 @@ def dict_hash_frozen(data: dict[Any, Any]) -> int:
     Returns:
         str: _description_
     """
-    return hash(frozenset(data.items()))
+    return corelibs_dict_hash_frozen(data)
 
 
+@deprecated("use corelibs_hash.fingerprint.dict_hash_crc and for new use hash_object instead")
 def dict_hash_crc(data: dict[Any, Any] | list[Any]) -> str:
     """
     LEGACY METHOD, must be kept for fallback, if used by other code, DO NOT CHANGE
@@ -60,14 +56,6 @@ def dict_hash_crc(data: dict[Any, Any] | list[Any]) -> str:
     Returns:
         str: sha256 hash, prefiex with HO_ if fallback used
     """
-    try:
-        return hashlib.sha256(
-            # IT IS IMPORTANT THAT THE BELOW CALL STAYS THE SAME AND DOES NOT CHANGE OR WE WILL GET DIFFERENT HASHES
-            # separators=(',', ':') to get rid of spaces, but if this is used the hash will be different, DO NOT ADD
-            json.dumps(data, sort_keys=True, ensure_ascii=True, default=str).encode('utf-8')
-        ).hexdigest()
-    except TypeError:
-        # Fallback tod different hasher, will return DIFFERENT hash than above, so only usable in int/str key mixes
-        return "HO_" + hash_object(data)
+    return corelibs_dict_hash_crc(data)
 
 # __END__

src/corelibs/iterator_handling/list_helpers.py

@@ -2,10 +2,16 @@
 List type helpers
 """
 
-import json
+from warnings import deprecated
 from typing import Any, Sequence
+from corelibs_iterator.list_support import (
+    convert_to_list as corelibs_convert_to_list,
+    is_list_in_list as corelibs_is_list_in_list,
+    make_unique_list_of_dicts as corelibs_make_unique_list_of_dicts
+)
 
 
+@deprecated("use corelibs_iterator.list_support.convert_to_list instead")
 def convert_to_list(
     entry: str | int | float | bool | Sequence[str | int | float | bool | Sequence[Any]]
 ) -> Sequence[str | int | float | bool | Sequence[Any]]:
@@ -18,11 +24,10 @@ def convert_to_list(
     Returns:
         list[str | int | float | bool] -- _description_
     """
-    if isinstance(entry, list):
-        return entry
-    return [entry]
+    return corelibs_convert_to_list(entry)
 
 
+@deprecated("use corelibs_iterator.list_support.is_list_in_list instead")
 def is_list_in_list(
     list_a: Sequence[str | int | float | bool | Sequence[Any]],
     list_b: Sequence[str | int | float | bool | Sequence[Any]]
@@ -38,14 +43,10 @@ def is_list_in_list(
     Returns:
         list[Any] -- _description_
     """
-    # Create sets of (value, type) tuples
-    set_a = set((item, type(item)) for item in list_a)
-    set_b = set((item, type(item)) for item in list_b)
-
-    # Get the difference and extract just the values
-    return [item for item, _ in set_a - set_b]
+    return corelibs_is_list_in_list(list_a, list_b)
 
 
+@deprecated("use corelibs_iterator.list_support.make_unique_list_of_dicts instead")
 def make_unique_list_of_dicts(dict_list: list[Any]) -> list[Any]:
     """
     Create a list of unique dictionary entries
@@ -56,20 +57,6 @@ def make_unique_list_of_dicts(dict_list: list[Any]) -> list[Any]:
     Returns:
         list[Any] -- _description_
     """
-    try:
-        # try json dumps, can fail with int and str index types
-        return list(
-            {
-                json.dumps(d, sort_keys=True, ensure_ascii=True, separators=(',', ':')): d
-                for d in dict_list
-            }.values()
-        )
-    except TypeError:
-        # Fallback for non-serializable entries, slow but works
-        unique: list[Any] = []
-        for d in dict_list:
-            if d not in unique:
-                unique.append(d)
-        return unique
+    return corelibs_make_unique_list_of_dicts(dict_list)
 
 # __END__

src/corelibs/json_handling/jmespath_helper.py

@@ -4,10 +4,10 @@ helper functions for jmespath interfaces
 
 from warnings import deprecated
 from typing import Any
-from corelibs_json.jmespath_support import jmespath_search as jmespath_search_ng
+from corelibs_search.jmespath_search import jmespath_search as jmespath_search_ng
 
 
-@deprecated("Use corelibs_json.jmespath_support.jmespath_search instead")
+@deprecated("Use corelibs_search.jmespath_search.jmespath_search instead")
 def jmespath_search(search_data: dict[Any, Any] | list[Any], search_params: str) -> Any:
     """
     jmespath search wrapper

src/corelibs/math_handling/math_helpers.py

@@ -2,9 +2,11 @@
 Various math helpers
 """
 
+from warnings import deprecated
 import math
 
 
+@deprecated("Use math.gcd instead")
 def gcd(a: int, b: int):
     """
     Calculate: Greatest Common Divisor
@@ -19,6 +21,7 @@ def gcd(a: int, b: int):
     return math.gcd(a, b)
 
 
+@deprecated("Use math.lcm instead")
 def lcd(a: int, b: int):
     """
     Calculate: Least Common Denominator