v0.48.0: Update Caller class with better error handling and reporting

- has now ErrorResponse return values instead of None on errors
- changed parameter cafile to ca_file and its position in the init method
- Proxy has ProxyConfig Typed Dict format

Tests updates to reflect those changes

pyproject.toml

@@ -1,7 +1,7 @@
 # MARK: Project info
 [project]
 name = "corelibs"
-version = "0.47.0"
+version = "0.48.0"
 description = "Collection of utils for Python scripts"
 readme = "README.md"
 requires-python = ">=3.13"

src/corelibs/requests_handling/caller.py

@@ -3,32 +3,61 @@ requests lib interface
 V2 call type
 """
 
-from typing import Any
-import warnings
+from typing import Any, TypedDict, cast
 import requests
-# to hide the verfiy warnings because of the bad SSL settings from Netskope, Akamai, etc
-warnings.filterwarnings('ignore', message='Unverified HTTPS request')
+from requests import exceptions
+
+
+class ErrorResponse:
+    """
+    Error response structure. This is returned if a request could not be completed
+    """
+    def __init__(
+        self,
+        code: int,
+        message: str,
+        action: str,
+        url: str,
+        exception: exceptions.InvalidSchema | exceptions.ReadTimeout | exceptions.ConnectionError | None = None
+    ) -> None:
+        self.code = code
+        self.message = message
+        self.action = action
+        self.url = url
+        self.exception_name = type(exception).__name__ if exception is not None else None
+        self.exception_trace = exception if exception is not None else None
+
+
+class ProxyConfig(TypedDict):
+    """
+    Socks proxy settings
+    """
+    type: str
+    host: str
+    port: str
 
 
 class Caller:
-    """_summary_"""
+    """
+    requests lib interface
+    """
 
     def __init__(
         self,
         header: dict[str, str],
-        verify: bool = True,
         timeout: int = 20,
-        proxy: dict[str, str] | None = None,
+        proxy: ProxyConfig | None = None,
+        verify: bool = True,
         ca_file: str | None = None
     ):
         self.headers = header
         self.timeout: int = timeout
-        self.cafile = ca_file
+        self.ca_file = ca_file
         self.verify = verify
-        self.proxy = proxy
+        self.proxy = cast(dict[str, str], proxy) if proxy is not None else None
 
     def __timeout(self, timeout: int | None) -> int:
-        if timeout is not None:
+        if timeout is not None and timeout >= 0:
             return timeout
         return self.timeout
 
@@ -39,7 +68,7 @@ class Caller:
         data: dict[str, Any] | None = None,
         params: dict[str, Any] | None = None,
         timeout: int | None = None
-    ) -> requests.Response | None:
+    ) -> requests.Response | ErrorResponse:
         """
         call wrapper, on error returns None
 
@@ -56,67 +85,96 @@ class Caller:
         if data is None:
             data = {}
         try:
-            response = None
             if action == "get":
-                response = requests.get(
+                return requests.get(
                     url,
                     params=params,
                     headers=self.headers,
                     timeout=self.__timeout(timeout),
                     verify=self.verify,
-                    proxies=self.proxy
+                    proxies=self.proxy,
+                    cert=self.ca_file
                 )
-            elif action == "post":
-                response = requests.post(
+            if action == "post":
+                return requests.post(
                     url,
                     params=params,
                     json=data,
                     headers=self.headers,
                     timeout=self.__timeout(timeout),
                     verify=self.verify,
-                    proxies=self.proxy
+                    proxies=self.proxy,
+                    cert=self.ca_file
                 )
-            elif action == "put":
-                response = requests.put(
+            if action == "put":
+                return requests.put(
                     url,
                     params=params,
                     json=data,
                     headers=self.headers,
                     timeout=self.__timeout(timeout),
                     verify=self.verify,
-                    proxies=self.proxy
+                    proxies=self.proxy,
+                    cert=self.ca_file
                 )
-            elif action == "patch":
-                response = requests.patch(
+            if action == "patch":
+                return requests.patch(
                     url,
                     params=params,
                     json=data,
                     headers=self.headers,
                     timeout=self.__timeout(timeout),
                     verify=self.verify,
-                    proxies=self.proxy
+                    proxies=self.proxy,
+                    cert=self.ca_file
                 )
-            elif action == "delete":
-                response = requests.delete(
+            if action == "delete":
+                return requests.delete(
                     url,
                     params=params,
                     headers=self.headers,
                     timeout=self.__timeout(timeout),
                     verify=self.verify,
-                    proxies=self.proxy
+                    proxies=self.proxy,
+                    cert=self.ca_file
                 )
-            return response
-        except requests.exceptions.InvalidSchema as e:
-            print(f"Invalid URL during '{action}' for {url}:\n\t{e}")
-            return None
-        except requests.exceptions.ReadTimeout as e:
-            print(f"Timeout ({self.timeout}s) during '{action}' for {url}:\n\t{e}")
-            return None
-        except requests.exceptions.ConnectionError as e:
-            print(f"Connection error during '{action}' for {url}:\n\t{e}")
-            return None
+            return ErrorResponse(
+                100,
+                f"Unsupported action '{action}'",
+                action,
+                url
+            )
+        except exceptions.InvalidSchema as e:
+            return ErrorResponse(
+                200,
+                f"Invalid URL during '{action}' for {url}",
+                action,
+                url,
+                e
+            )
+        except exceptions.ReadTimeout as e:
+            return ErrorResponse(
+                300,
+                f"Timeout ({self.timeout}s) during '{action}' for {url}",
+                action,
+                url,
+                e
+            )
+        except exceptions.ConnectionError as e:
+            return ErrorResponse(
+                400,
+                f"Connection error during '{action}' for {url}",
+                action,
+                url,
+                e
+            )
 
-    def get(self, url: str, params: dict[str, Any] | None = None) -> requests.Response | None:
+    def get(
+        self,
+        url: str,
+        params: dict[str, Any] | None = None,
+        timeout: int | None = None
+    ) -> requests.Response | ErrorResponse:
         """
         get data
 
@@ -127,11 +185,15 @@ class Caller:
         Returns:
             requests.Response: _description_
         """
-        return self.__call('get', url, params=params)
+        return self.__call('get', url, params=params, timeout=timeout)
 
     def post(
-        self, url: str, data: dict[str, Any] | None = None, params: dict[str, Any] | None = None
-    ) -> requests.Response | None:
+        self,
+        url: str,
+        data: dict[str, Any] | None = None,
+        params: dict[str, Any] | None = None,
+        timeout: int | None = None
+    ) -> requests.Response | ErrorResponse:
         """
         post data
 
@@ -143,11 +205,15 @@ class Caller:
         Returns:
             requests.Response | None: _description_
         """
-        return self.__call('post', url, data, params)
+        return self.__call('post', url, data, params, timeout=timeout)
 
     def put(
-        self, url: str, data: dict[str, Any] | None = None, params: dict[str, Any] | None = None
-    ) -> requests.Response | None:
+        self,
+        url: str,
+        data: dict[str, Any] | None = None,
+        params: dict[str, Any] | None = None,
+        timeout: int | None = None
+    ) -> requests.Response | ErrorResponse:
         """_summary_
 
         Args:
@@ -158,11 +224,15 @@ class Caller:
         Returns:
             requests.Response | None: _description_
         """
-        return self.__call('put', url, data, params)
+        return self.__call('put', url, data, params, timeout=timeout)
 
     def patch(
-        self, url: str, data: dict[str, Any] | None = None, params: dict[str, Any] | None = None
-    ) -> requests.Response | None:
+        self,
+        url: str,
+        data: dict[str, Any] | None = None,
+        params: dict[str, Any] | None = None,
+        timeout: int | None = None
+    ) -> requests.Response | ErrorResponse:
         """_summary_
 
         Args:
@@ -173,9 +243,14 @@ class Caller:
         Returns:
             requests.Response | None: _description_
         """
-        return self.__call('patch', url, data, params)
+        return self.__call('patch', url, data, params, timeout=timeout)
 
-    def delete(self, url: str, params: dict[str, Any] | None = None) -> requests.Response | None:
+    def delete(
+        self,
+        url: str,
+        params: dict[str, Any] | None = None,
+        timeout: int | None = None
+    ) -> requests.Response | ErrorResponse:
         """
         delete
 
@@ -186,6 +261,6 @@ class Caller:
         Returns:
             requests.Response | None: _description_
         """
-        return self.__call('delete', url, params=params)
+        return self.__call('delete', url, params=params, timeout=timeout)
 
 # __END__

tests/unit/requests_handling/test_caller.py

@@ -2,11 +2,10 @@
 PyTest: requests_handling/caller
 """
 
-from typing import Any
 from unittest.mock import Mock, patch
 import pytest
 import requests
-from corelibs.requests_handling.caller import Caller
+from corelibs.requests_handling.caller import Caller, ErrorResponse, ProxyConfig
 
 
 class TestCallerInit:
@@ -21,13 +20,17 @@ class TestCallerInit:
         assert caller.timeout == 20
         assert caller.verify is True
         assert caller.proxy is None
-        assert caller.cafile is None
+        assert caller.ca_file is None
 
     def test_init_with_all_params(self):
         """Test Caller initialization with all parameters"""
         header = {"Authorization": "Bearer token", "Content-Type": "application/json"}
-        proxy = {"http": "http://proxy.example.com:8080", "https": "https://proxy.example.com:8080"}
-        caller = Caller(header=header, verify=False, timeout=30, proxy=proxy)
+        proxy: ProxyConfig = {
+            "type": "socks5",
+            "host": "proxy.example.com:8080",
+            "port": "8080"
+        }
+        caller = Caller(header=header, timeout=30, proxy=proxy, verify=False)
 
         assert caller.headers == header
         assert caller.timeout == 30
@@ -58,7 +61,7 @@ class TestCallerInit:
         ca_file_path = "/path/to/ca/cert.pem"
         caller = Caller(header={}, ca_file=ca_file_path)
 
-        assert caller.cafile == ca_file_path
+        assert caller.ca_file == ca_file_path
 
 
 class TestCallerGet:
@@ -81,7 +84,8 @@ class TestCallerGet:
             headers={"Authorization": "Bearer token"},
             timeout=20,
             verify=True,
-            proxies=None
+            proxies=None,
+            cert=None
         )
 
     @patch('corelibs.requests_handling.caller.requests.get')
@@ -101,7 +105,8 @@ class TestCallerGet:
             headers={},
             timeout=20,
             verify=True,
-            proxies=None
+            proxies=None,
+            cert=None
         )
 
     @patch('corelibs.requests_handling.caller.requests.get')
@@ -134,7 +139,11 @@ class TestCallerGet:
         mock_response = Mock(spec=requests.Response)
         mock_get.return_value = mock_response
 
-        proxy = {"http": "http://proxy.example.com:8080"}
+        proxy: ProxyConfig = {
+            "type": "socks5",
+            "host": "proxy.example.com:8080",
+            "port": "8080"
+        }
         caller = Caller(header={}, proxy=proxy)
         caller.get("https://api.example.com/data")
 
@@ -142,40 +151,46 @@ class TestCallerGet:
         assert mock_get.call_args[1]["proxies"] == proxy
 
     @patch('corelibs.requests_handling.caller.requests.get')
-    def test_get_invalid_schema_returns_none(self, mock_get: Mock, capsys: Any):
-        """Test GET request with invalid URL schema returns None"""
+    def test_get_invalid_schema_returns_none(self, mock_get: Mock):
+        """Test GET request with invalid URL schema returns ErrorResponse"""
         mock_get.side_effect = requests.exceptions.InvalidSchema("Invalid URL")
 
         caller = Caller(header={})
         response = caller.get("invalid://example.com")
 
-        assert response is None
-        captured = capsys.readouterr()
-        assert "Invalid URL during 'get'" in captured.out
+        assert isinstance(response, ErrorResponse)
+        assert response.code == 200
+        assert "Invalid URL during 'get'" in response.message
+        assert response.action == "get"
+        assert response.url == "invalid://example.com"
 
     @patch('corelibs.requests_handling.caller.requests.get')
-    def test_get_timeout_returns_none(self, mock_get: Mock, capsys: Any):
-        """Test GET request timeout returns None"""
+    def test_get_timeout_returns_none(self, mock_get: Mock):
+        """Test GET request timeout returns ErrorResponse"""
         mock_get.side_effect = requests.exceptions.ReadTimeout("Timeout")
 
         caller = Caller(header={})
         response = caller.get("https://api.example.com/data")
 
-        assert response is None
-        captured = capsys.readouterr()
-        assert "Timeout (20s) during 'get'" in captured.out
+        assert isinstance(response, ErrorResponse)
+        assert response.code == 300
+        assert "Timeout (20s) during 'get'" in response.message
+        assert response.action == "get"
+        assert response.url == "https://api.example.com/data"
 
     @patch('corelibs.requests_handling.caller.requests.get')
-    def test_get_connection_error_returns_none(self, mock_get: Mock, capsys: Any):
-        """Test GET request connection error returns None"""
+    def test_get_connection_error_returns_none(self, mock_get: Mock):
+        """Test GET request connection error returns ErrorResponse"""
         mock_get.side_effect = requests.exceptions.ConnectionError("Connection failed")
 
         caller = Caller(header={})
         response = caller.get("https://api.example.com/data")
 
-        assert response is None
-        captured = capsys.readouterr()
-        assert "Connection error during 'get'" in captured.out
+        assert isinstance(response, ErrorResponse)
+        assert response.code == 400
+        assert "Connection error during 'get'" in response.message
+        assert response.action == "get"
+        assert response.url == "https://api.example.com/data"
 
 
 class TestCallerPost:
@@ -200,7 +215,8 @@ class TestCallerPost:
             headers={"Content-Type": "application/json"},
             timeout=20,
             verify=True,
-            proxies=None
+            proxies=None,
+            cert=None
         )
 
     @patch('corelibs.requests_handling.caller.requests.post')
@@ -234,40 +250,46 @@ class TestCallerPost:
         assert mock_post.call_args[1]["json"] == data
 
     @patch('corelibs.requests_handling.caller.requests.post')
-    def test_post_invalid_schema_returns_none(self, mock_post: Mock, capsys: Any):
-        """Test POST request with invalid URL schema returns None"""
+    def test_post_invalid_schema_returns_none(self, mock_post: Mock):
+        """Test POST request with invalid URL schema returns ErrorResponse"""
         mock_post.side_effect = requests.exceptions.InvalidSchema("Invalid URL")
 
         caller = Caller(header={})
         response = caller.post("invalid://example.com", data={"test": "data"})
 
-        assert response is None
-        captured = capsys.readouterr()
-        assert "Invalid URL during 'post'" in captured.out
+        assert isinstance(response, ErrorResponse)
+        assert response.code == 200
+        assert "Invalid URL during 'post'" in response.message
+        assert response.action == "post"
+        assert response.url == "invalid://example.com"
 
     @patch('corelibs.requests_handling.caller.requests.post')
-    def test_post_timeout_returns_none(self, mock_post: Mock, capsys: Any):
-        """Test POST request timeout returns None"""
+    def test_post_timeout_returns_none(self, mock_post: Mock):
+        """Test POST request timeout returns ErrorResponse"""
         mock_post.side_effect = requests.exceptions.ReadTimeout("Timeout")
 
         caller = Caller(header={})
         response = caller.post("https://api.example.com/data", data={"test": "data"})
 
-        assert response is None
-        captured = capsys.readouterr()
-        assert "Timeout (20s) during 'post'" in captured.out
+        assert isinstance(response, ErrorResponse)
+        assert response.code == 300
+        assert "Timeout (20s) during 'post'" in response.message
+        assert response.action == "post"
+        assert response.url == "https://api.example.com/data"
 
     @patch('corelibs.requests_handling.caller.requests.post')
-    def test_post_connection_error_returns_none(self, mock_post: Mock, capsys: Any):
-        """Test POST request connection error returns None"""
+    def test_post_connection_error_returns_none(self, mock_post: Mock):
+        """Test POST request connection error returns ErrorResponse"""
         mock_post.side_effect = requests.exceptions.ConnectionError("Connection failed")
 
         caller = Caller(header={})
         response = caller.post("https://api.example.com/data", data={"test": "data"})
 
-        assert response is None
-        captured = capsys.readouterr()
-        assert "Connection error during 'post'" in captured.out
+        assert isinstance(response, ErrorResponse)
+        assert response.code == 400
+        assert "Connection error during 'post'" in response.message
+        assert response.action == "post"
+        assert response.url == "https://api.example.com/data"
 
 
 class TestCallerPut:
@@ -292,7 +314,8 @@ class TestCallerPut:
             headers={"Content-Type": "application/json"},
             timeout=20,
             verify=True,
-            proxies=None
+            proxies=None,
+            cert=None
         )
 
     @patch('corelibs.requests_handling.caller.requests.put')
@@ -311,16 +334,18 @@ class TestCallerPut:
         assert mock_put.call_args[1]["params"] == params
 
     @patch('corelibs.requests_handling.caller.requests.put')
-    def test_put_timeout_returns_none(self, mock_put: Mock, capsys: Any):
-        """Test PUT request timeout returns None"""
+    def test_put_timeout_returns_none(self, mock_put: Mock):
+        """Test PUT request timeout returns ErrorResponse"""
         mock_put.side_effect = requests.exceptions.ReadTimeout("Timeout")
 
         caller = Caller(header={})
         response = caller.put("https://api.example.com/data/1", data={"test": "data"})
 
-        assert response is None
-        captured = capsys.readouterr()
-        assert "Timeout (20s) during 'put'" in captured.out
+        assert isinstance(response, ErrorResponse)
+        assert response.code == 300
+        assert "Timeout (20s) during 'put'" in response.message
+        assert response.action == "put"
+        assert response.url == "https://api.example.com/data/1"
 
 
 class TestCallerPatch:
@@ -345,7 +370,8 @@ class TestCallerPatch:
             headers={"Content-Type": "application/json"},
             timeout=20,
             verify=True,
-            proxies=None
+            proxies=None,
+            cert=None
         )
 
     @patch('corelibs.requests_handling.caller.requests.patch')
@@ -364,16 +390,18 @@ class TestCallerPatch:
         assert mock_patch.call_args[1]["params"] == params
 
     @patch('corelibs.requests_handling.caller.requests.patch')
-    def test_patch_connection_error_returns_none(self, mock_patch: Mock, capsys: Any):
-        """Test PATCH request connection error returns None"""
+    def test_patch_connection_error_returns_none(self, mock_patch: Mock):
+        """Test PATCH request connection error returns ErrorResponse"""
         mock_patch.side_effect = requests.exceptions.ConnectionError("Connection failed")
 
         caller = Caller(header={})
         response = caller.patch("https://api.example.com/data/1", data={"test": "data"})
 
-        assert response is None
-        captured = capsys.readouterr()
-        assert "Connection error during 'patch'" in captured.out
+        assert isinstance(response, ErrorResponse)
+        assert response.code == 400
+        assert "Connection error during 'patch'" in response.message
+        assert response.action == "patch"
+        assert response.url == "https://api.example.com/data/1"
 
 
 class TestCallerDelete:
@@ -396,7 +424,8 @@ class TestCallerDelete:
             headers={"Authorization": "Bearer token"},
             timeout=20,
             verify=True,
-            proxies=None
+            proxies=None,
+            cert=None
         )
 
     @patch('corelibs.requests_handling.caller.requests.delete')
@@ -414,16 +443,18 @@ class TestCallerDelete:
         assert mock_delete.call_args[1]["params"] == params
 
     @patch('corelibs.requests_handling.caller.requests.delete')
-    def test_delete_invalid_schema_returns_none(self, mock_delete: Mock, capsys: Any):
-        """Test DELETE request with invalid URL schema returns None"""
+    def test_delete_invalid_schema_returns_none(self, mock_delete: Mock):
+        """Test DELETE request with invalid URL schema returns ErrorResponse"""
         mock_delete.side_effect = requests.exceptions.InvalidSchema("Invalid URL")
 
         caller = Caller(header={})
         response = caller.delete("invalid://example.com/data/1")
 
-        assert response is None
-        captured = capsys.readouterr()
-        assert "Invalid URL during 'delete'" in captured.out
+        assert isinstance(response, ErrorResponse)
+        assert response.code == 200
+        assert "Invalid URL during 'delete'" in response.message
+        assert response.action == "delete"
+        assert response.url == "invalid://example.com/data/1"
 
 
 class TestCallerParametrized:
@@ -492,7 +523,7 @@ class TestCallerParametrized:
     ])
     @patch('corelibs.requests_handling.caller.requests.get')
     def test_exception_handling(
-        self, mock_get: Mock, exception_class: type, expected_message: str, capsys: Any
+        self, mock_get: Mock, exception_class: type, expected_message: str
     ):
         """Test exception handling for all exception types"""
         mock_get.side_effect = exception_class("Test error")
@@ -500,9 +531,8 @@ class TestCallerParametrized:
         caller = Caller(header={})
         response = caller.get("https://api.example.com/data")
 
-        assert response is None
-        captured = capsys.readouterr()
-        assert expected_message in captured.out
+        assert isinstance(response, ErrorResponse)
+        assert expected_message in response.message
 
 
 class TestCallerIntegration:
@@ -599,7 +629,8 @@ class TestCallerEdgeCases:
             headers={},
             timeout=20,
             verify=True,
-            proxies=None
+            proxies=None,
+            cert=None
         )
 
     @patch('corelibs.requests_handling.caller.requests.post')
@@ -659,7 +690,8 @@ class TestCallerEdgeCases:
             headers={},
             timeout=20,
             verify=True,
-            proxies=None
+            proxies=None,
+            cert=None
         )
 
     @patch('corelibs.requests_handling.caller.requests.get')
@@ -679,7 +711,8 @@ class TestCallerEdgeCases:
             headers={},
             timeout=20,
             verify=True,
-            proxies=None
+            proxies=None,
+            cert=None
         )
 
     def test_timeout_zero(self):
@@ -730,9 +763,10 @@ class TestCallerProxyHandling:
         mock_response = Mock(spec=requests.Response)
         mock_get.return_value = mock_response
 
-        proxy = {
-            "http": "http://proxy.example.com:8080",
-            "https": "https://proxy.example.com:8080"
+        proxy: ProxyConfig = {
+            "type": "socks5",
+            "host": "proxy.example.com:8080",
+            "port": "8080"
         }
         caller = Caller(header={}, proxy=proxy)
         caller.get("https://api.example.com/data")
@@ -746,9 +780,10 @@ class TestCallerProxyHandling:
         mock_response = Mock(spec=requests.Response)
         mock_post.return_value = mock_response
 
-        proxy = {
-            "http": "http://user:pass@proxy.example.com:8080",
-            "https": "https://user:pass@proxy.example.com:8080"
+        proxy: ProxyConfig = {
+            "type": "socks5",
+            "host": "proxy.example.com:8080",
+            "port": "8080"
         }
         caller = Caller(header={}, proxy=proxy)
         caller.post("https://api.example.com/data", data={"test": "data"})
@@ -789,7 +824,7 @@ class TestCallerResponseHandling:
         caller = Caller(header={})
         response = caller.get("https://api.example.com/data")
 
-        assert response is not None
+        assert not isinstance(response, ErrorResponse)
         assert response.status_code == 200
         assert response.text == "Success"
         assert response.json() == {"status": "ok"}
@@ -805,7 +840,7 @@ class TestCallerResponseHandling:
             caller = Caller(header={})
             response = caller.get("https://api.example.com/data")
 
-            assert response is not None
+            assert not isinstance(response, ErrorResponse)
             assert response.status_code == status_code