PyPI/CoreLibs-PyPI-All
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move all debug handling into their own packages

Browse Source 
dump data: corelibs_dump_data
stack trace: corelibs_stack_trace
profiling, timing, etc: corelibs_debug
This commit is contained in:
Clemens Schwaighofer
2026-02-03 10:48:59 +09:00
parent 28ab7c6f0c
commit 2e0d5aeb51
26 changed files with 92 additions and 3118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/corelibs/db_handling/sql_main.py
View File

@@ -12,7 +12,7 @@ TODO: adapt more CoreLibs DB IO class flow here
"""
from typing import TYPE_CHECKING, Any, Literal
from corelibs.debug_handling.debug_helpers import call_stack
from corelibs_stack_trace.stack import call_stack
from corelibs.db_handling.sqlite_io import SQLiteIO
if TYPE_CHECKING:
from corelibs.logging_handling.log import Logger

2
src/corelibs/db_handling/sqlite_io.py
View File

@@ -8,7 +8,7 @@ also method names are subject to change
from pathlib import Path
from typing import Any, Literal, TYPE_CHECKING
import sqlite3
from corelibs.debug_handling.debug_helpers import call_stack
from corelibs_stack_trace.stack import call_stack
if TYPE_CHECKING:
from corelibs.logging_handling.log import Logger

42
src/corelibs/debug_handling/debug_helpers.py
View File

@@ -2,16 +2,16 @@
Various debug helpers
"""
import traceback
import os
import sys
from warnings import deprecated
from typing import Tuple, Type
from types import TracebackType
from corelibs_stack_trace.stack import call_stack as call_stack_ng, exception_stack as exception_stack_ng
# _typeshed.OptExcInfo
OptExcInfo = Tuple[None, None, None] | Tuple[Type[BaseException], BaseException, TracebackType]
@deprecated("Use corelibs_stack_trace.stack.call_stack instead")
def call_stack(
start: int = 0,
skip_last: int = -1,
@@ -31,23 +31,15 @@ def call_stack(
Returns:
str -- _description_
"""
# stack = traceback.extract_stack()[start:depth]
# how many of the last entries we skip (so we do not get self), default is -1
# start cannot be negative
if skip_last > 0:
skip_last = skip_last * -1
stack = traceback.extract_stack()
__stack = stack[start:skip_last]
# start possible to high, reset start to 0
if not __stack and reset_start_if_empty:
start = 0
__stack = stack[start:skip_last]
if not separator:
separator = ' -> '
# print(f"* HERE: {dump_data(stack)}")
return f"{separator}".join(f"{os.path.basename(f.filename)}:{f.name}:{f.lineno}" for f in __stack)
return call_stack_ng(
start=start,
skip_last=skip_last,
separator=separator,
reset_start_if_empty=reset_start_if_empty
)
@deprecated("Use corelibs_stack_trace.stack.exception_stack instead")
def exception_stack(
exc_stack: OptExcInfo | None = None,
separator: str = ' -> '
@@ -62,15 +54,9 @@ def exception_stack(
Returns:
str -- _description_
"""
if exc_stack is not None:
_, _, exc_traceback = exc_stack
else:
exc_traceback = None
_, _, exc_traceback = sys.exc_info()
stack = traceback.extract_tb(exc_traceback)
if not separator:
separator = ' -> '
# print(f"* HERE: {dump_data(stack)}")
return f"{separator}".join(f"{os.path.basename(f.filename)}:{f.name}:{f.lineno}" for f in stack)
return exception_stack_ng(
exc_stack=exc_stack,
separator=separator
)
# __END__

7
src/corelibs/debug_handling/dump_data.py
View File

@@ -2,10 +2,12 @@
dict dump as JSON formatted
"""
import json
from warnings import deprecated
from typing import Any
from corelibs_dump_data.dump_data import dump_data as dump_data_ng
@deprecated("Use corelibs_dump_data.dump_data.dump_data instead")
def dump_data(data: Any, use_indent: bool = True) -> str:
"""
dump formated output from dict/list
@@ -16,7 +18,6 @@ def dump_data(data: Any, use_indent: bool = True) -> str:
Returns:
str: _description_
"""
indent = 4 if use_indent else None
return json.dumps(data, indent=indent, ensure_ascii=False, default=str)
return dump_data_ng(data=data, use_indent=use_indent)
# __END__

117
src/corelibs/debug_handling/profiling.py
View File

@@ -4,123 +4,40 @@ Profile memory usage in Python
# https://docs.python.org/3/library/tracemalloc.html
import os
import time
import tracemalloc
import linecache
from typing import Tuple
from tracemalloc import Snapshot
import psutil
from warnings import warn, deprecated
from typing import TYPE_CHECKING
from corelibs_debug.profiling import display_top as display_top_ng, display_top_str, Profiling as CoreLibsProfiling
if TYPE_CHECKING:
from tracemalloc import Snapshot
def display_top(snapshot: Snapshot, key_type: str = 'lineno', limit: int = 10) -> str:
@deprecated("Use corelibs_debug.profiling.display_top_str with data from display_top instead")
def display_top(snapshot: 'Snapshot', key_type: str = 'lineno', limit: int = 10) -> str:
"""
Print tracmalloc stats
https://docs.python.org/3/library/tracemalloc.html#pretty-top
Args:
snapshot (Snapshot): _description_
snapshot ('Snapshot'): _description_
key_type (str, optional): _description_. Defaults to 'lineno'.
limit (int, optional): _description_. Defaults to 10.
"""
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
profiler_msg = f"Top {limit} lines"
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
profiler_msg += f"#{index}: {filename}:{frame.lineno}: {(stat.size / 1024):.1f} KiB"
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
profiler_msg += f" {line}"
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
profiler_msg += f"{len(other)} other: {(size / 1024):.1f} KiB"
total = sum(stat.size for stat in top_stats)
profiler_msg += f"Total allocated size: {(total / 1024):.1f} KiB"
return profiler_msg
return display_top_str(
display_top_ng(
snapshot=snapshot,
key_type=key_type,
limit=limit
)
)
class Profiling:
class Profiling(CoreLibsProfiling):
"""
Profile memory usage and elapsed time for some block
Based on: https://stackoverflow.com/a/53301648
"""
def __init__(self):
# profiling id
self.__ident: str = ''
# memory
self.__rss_before: int = 0
self.__vms_before: int = 0
# self.shared_before: int = 0
self.__rss_used: int = 0
self.__vms_used: int = 0
# self.shared_used: int = 0
# time
self.__call_start: float = 0
self.__elapsed = 0
def __get_process_memory(self) -> Tuple[int, int]:
process = psutil.Process(os.getpid())
mi = process.memory_info()
# macos does not have mi.shared
return mi.rss, mi.vms
def __elapsed_since(self) -> str:
elapsed = time.time() - self.__call_start
if elapsed < 1:
return str(round(elapsed * 1000, 2)) + "ms"
if elapsed < 60:
return str(round(elapsed, 2)) + "s"
if elapsed < 3600:
return str(round(elapsed / 60, 2)) + "min"
return str(round(elapsed / 3600, 2)) + "hrs"
def __format_bytes(self, bytes_data: int) -> str:
if abs(bytes_data) < 1000:
return str(bytes_data) + "B"
if abs(bytes_data) < 1e6:
return str(round(bytes_data / 1e3, 2)) + "kB"
if abs(bytes_data) < 1e9:
return str(round(bytes_data / 1e6, 2)) + "MB"
return str(round(bytes_data / 1e9, 2)) + "GB"
def start_profiling(self, ident: str) -> None:
"""
start the profiling
"""
self.__ident = ident
self.__rss_before, self.__vms_before = self.__get_process_memory()
self.__call_start = time.time()
def end_profiling(self) -> None:
"""
end the profiling
"""
if self.__rss_before == 0 and self.__vms_before == 0:
print("start_profile() was not called, output will be negative")
self.__elapsed = self.__elapsed_since()
__rss_after, __vms_after = self.__get_process_memory()
self.__rss_used = __rss_after - self.__rss_before
self.__vms_used = __vms_after - self.__vms_before
def print_profiling(self) -> str:
"""
print the profiling time
"""
return (
f"Profiling: {self.__ident:>20} "
f"RSS: {self.__format_bytes(self.__rss_used):>8} | "
f"VMS: {self.__format_bytes(self.__vms_used):>8} | "
f"time: {self.__elapsed:>8}"
)
warn("Use corelibs_debug.profiling.Profiling instead", DeprecationWarning, stacklevel=2)
# __END__

101
src/corelibs/debug_handling/timer.py
View File

@@ -5,109 +5,16 @@ Returns:
Timer: class timer for basic time run calculations
"""
from datetime import datetime, timedelta
from warnings import warn
from corelibs_debug.timer import Timer as CorelibsTimer
class Timer:
class Timer(CorelibsTimer):
"""
get difference between start and end date/time
"""
def __init__(self):
"""
init new start time and set end time to None
"""
self._overall_start_time = datetime.now()
self._overall_end_time = None
self._overall_run_time = None
self._start_time = datetime.now()
self._end_time = None
self._run_time = None
# MARK: overall run time
def overall_run_time(self) -> timedelta:
"""
overall run time difference from class launch to call of this function
Returns:
timedelta: _description_
"""
self._overall_end_time = datetime.now()
self._overall_run_time = self._overall_end_time - self._overall_start_time
return self._overall_run_time
def get_overall_start_time(self) -> datetime:
"""
get set start time
Returns:
datetime: _description_
"""
return self._overall_start_time
def get_overall_end_time(self) -> datetime | None:
"""
get set end time or None for not set
Returns:
datetime|None: _description_
"""
return self._overall_end_time
def get_overall_run_time(self) -> timedelta | None:
"""
get run time or None if run time was not called
Returns:
datetime|None: _description_
"""
return self._overall_run_time
# MARK: set run time
def run_time(self) -> timedelta:
"""
difference between start time and current time
Returns:
datetime: _description_
"""
self._end_time = datetime.now()
self._run_time = self._end_time - self._start_time
return self._run_time
def reset_run_time(self):
"""
reset start/end and run tine
"""
self._start_time = datetime.now()
self._end_time = None
self._run_time = None
def get_start_time(self) -> datetime:
"""
get set start time
Returns:
datetime: _description_
"""
return self._start_time
def get_end_time(self) -> datetime | None:
"""
get set end time or None for not set
Returns:
datetime|None: _description_
"""
return self._end_time
def get_run_time(self) -> timedelta | None:
"""
get run time or None if run time was not called
Returns:
datetime|None: _description_
"""
return self._run_time
warn("Use corelibs_debug.timer.Timer instead", DeprecationWarning, stacklevel=2)
# __END__

45
src/corelibs/debug_handling/writeline.py
View File

@@ -2,11 +2,18 @@
Various small helpers for data writing
"""
from warnings import deprecated
from typing import TYPE_CHECKING
from corelibs_debug.writeline import (
write_l as write_l_ng, pr_header as pr_header_ng,
pr_title as pr_title_ng, pr_open as pr_open_ng,
pr_close as pr_close_ng, pr_act as pr_act_ng
)
if TYPE_CHECKING:
from io import TextIOWrapper, StringIO
@deprecated("Use corelibs_debug.writeline.write_l instead")
def write_l(line: str, fpl: 'TextIOWrapper | StringIO | None' = None, print_line: bool = False):
"""
Write a line to screen and to output file
@@ -15,23 +22,30 @@ def write_l(line: str, fpl: 'TextIOWrapper | StringIO | None' = None, print_line
line (String): Line to write
fpl (Resource): file handler resource, if none write only to console
"""
if print_line is True:
print(line)
if fpl is not None:
fpl.write(line + "\n")
return write_l_ng(
line=line,
fpl=fpl,
print_line=print_line
)
# progress printers
@deprecated("Use corelibs_debug.writeline.pr_header instead")
def pr_header(tag: str, marker_string: str = '#', width: int = 35):
"""_summary_
Args:
tag (str): _description_
"""
print(f" {marker_string} {tag:^{width}} {marker_string}")
return pr_header_ng(
tag=tag,
marker_string=marker_string,
width=width
)
@deprecated("Use corelibs_debug.writeline.pr_title instead")
def pr_title(tag: str, prefix_string: str = '|', space_filler: str = '.', width: int = 35):
"""_summary_
@@ -39,9 +53,15 @@ def pr_title(tag: str, prefix_string: str = '|', space_filler: str = '.', width:
tag (str): _description_
prefix_string (str, optional): _description_. Defaults to '|'.
"""
print(f" {prefix_string} {tag:{space_filler}<{width}}:", flush=True)
return pr_title_ng(
tag=tag,
prefix_string=prefix_string,
space_filler=space_filler,
width=width
)
@deprecated("Use corelibs_debug.writeline.pr_open instead")
def pr_open(tag: str, prefix_string: str = '|', space_filler: str = '.', width: int = 35):
"""
writen progress open line with tag
@@ -50,9 +70,15 @@ def pr_open(tag: str, prefix_string: str = '|', space_filler: str = '.', width:
tag (str): _description_
prefix_string (str): prefix string. Default: '|'
"""
print(f" {prefix_string} {tag:{space_filler}<{width}} [", end="", flush=True)
return pr_open_ng(
tag=tag,
prefix_string=prefix_string,
space_filler=space_filler,
width=width
)
@deprecated("Use corelibs_debug.writeline.pr_close instead")
def pr_close(tag: str = ''):
"""
write the close tag with new line
@@ -60,9 +86,10 @@ def pr_close(tag: str = ''):
Args:
tag (str, optional): _description_. Defaults to ''.
"""
print(f"{tag}]", flush=True)
return pr_close_ng(tag=tag)
@deprecated("Use corelibs_debug.writeline.pr_act instead")
def pr_act(act: str = "."):
"""
write progress character
@@ -70,6 +97,6 @@ def pr_act(act: str = "."):
Args:
act (str, optional): _description_. Defaults to ".".
"""
print(f"{act}", end="", flush=True)
return pr_act_ng(act=act)
# __EMD__

0
src/corelibs/exceptions/__init__.py Normal file
View File

2
src/corelibs/logging_handling/log.py
View File

@@ -13,9 +13,9 @@ from pathlib import Path
import atexit
from enum import Flag, auto
from typing import MutableMapping, TextIO, TypedDict, Any, TYPE_CHECKING, cast
from corelibs_stack_trace.stack import call_stack, exception_stack
from corelibs_text_colors.text_colors import Colors
from corelibs.logging_handling.logging_level_handling.logging_level import LoggingLevel
from corelibs.debug_handling.debug_helpers import call_stack, exception_stack
if TYPE_CHECKING:
from multiprocessing import Queue