Better errors on http errors (include part of body, decode json)

config/rootfs/debos/scripts/trixie-rt.sh

@@ -17,6 +17,7 @@ BUILD_DEPS="\
     m4 \
     libc6-dev \
     libnuma-dev \
+    python3 \
 "
 
 apt-get install --no-install-recommends -y  ${BUILD_DEPS}
@@ -29,7 +30,11 @@ mkdir -p ${BUILD_DIR} && cd ${BUILD_DIR}
 git config --global http.sslverify false
 
 GIT_URL="https://git.kernel.org/pub/scm/utils/rt-tests/rt-tests.git"
-GIT_SHA=$(git ls-remote ${GIT_URL} | head -n 1 | cut -f 1)
+# Current rt-tests commit used for deterministic builds (main branch HEAD at snapshot time).
+# Update the value below when you intentionally rebuild against a newer rt-tests snapshot.
+# To refresh:
+#   GIT_SHA=$(git ls-remote ${GIT_URL} refs/heads/main | cut -f 1)
+GIT_SHA=4e68b52f0e0c9777c91088948374c6ee3d4a1f6b
 
 echo '    {"name": "rt-tests", "git_url": "'$GIT_URL'", "git_commit": "'$GIT_SHA'" }' >> $BUILDFILE
 echo '  ]}' >> $BUILDFILE

kernelci/api/__init__.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 #
-# Copyright (C) 2023-2024 Collabora Limited
+# Copyright (C) 2023-2026 Collabora Limited
 # Author: Guillaume Tucker <guillaume.tucker@collabora.com>
 # Author: Denys Fedoryshchenko <denys.f@collabora.com>
 
@@ -20,6 +20,79 @@
 
 import kernelci.config.api
 
+HTTP_ERROR_BODY_SNIPPET = 512
+
+
+def _http_error_body_snippet(response, max_length: int = HTTP_ERROR_BODY_SNIPPET):
+    """Return a truncated response body useful for HTTP error logging.
+    Also try to extract a more specific error message from the body if it's a JSON
+    with a "detail", "error" or "message" field.
+    """
+    if response is None or response.content in (None, b''):
+        return None
+
+    content_type = response.headers.get('content-type', '')
+    snippet = None
+
+    if content_type.startswith('application/json'):
+        try:
+            payload = response.json()
+            if isinstance(payload, dict):
+                snippet = (
+                    payload.get('detail') or
+                    payload.get('error') or
+                    payload.get('message')
+                )
+            if snippet is None:
+                snippet = payload
+        except (TypeError, ValueError):
+            pass
+
+    if snippet is None:
+        try:
+            snippet = response.text
+        except (AttributeError, LookupError, RuntimeError, UnicodeError):
+            snippet = None
+
+    if not snippet:
+        return None
+
+    if not isinstance(snippet, str):
+        snippet = json.dumps(snippet)
+    if not snippet:
+        return None
+
+    snippet = snippet.strip()
+    if not snippet:
+        return None
+
+    if len(snippet) > max_length:
+        snippet = snippet[:max_length] + "..."
+    return snippet
+
+
+def _enrich_http_error(error, max_length: int = HTTP_ERROR_BODY_SNIPPET):
+    """Append a small response-body snippet to request exceptions."""
+    status_code = None
+    url = None
+    if hasattr(error, 'response') and error.response is not None:
+        status_code = getattr(error.response, 'status_code', None)
+        url = getattr(error.response, 'url', None)
+        if url is None:
+            request = getattr(error, 'request', None)
+            url = getattr(request, 'url', None)
+
+    message_parts = ['HTTP Error']
+    if status_code:
+        message_parts.append(str(status_code))
+    message_parts.append(f"url={url}" if url else 'url=<unknown>')
+
+    body = _http_error_body_snippet(error.response, max_length=max_length)
+    if body:
+        message_parts.append(f"body={body}")
+
+    error.args = (': '.join(message_parts),)
+
 
 class Data:
     """Convenience class to keep common data in API bindings implementation"""
@@ -81,7 +154,11 @@ def _get(self, path, params=None):
             url, params=params, headers=self.data.headers,
             timeout=self.data.timeout
         )
-        resp.raise_for_status()
+        try:
+            resp.raise_for_status()
+        except requests.exceptions.HTTPError as err:
+            _enrich_http_error(err)
+            raise
         return resp
 
     def _post(self, path, data=None, params=None, json_data=True):
@@ -140,7 +217,11 @@ def _post(self, path, data=None, params=None, json_data=True):
                     url, data, headers=self.data.headers,
                     params=params, timeout=self.data.timeout
                 )
-        resp.raise_for_status()
+        try:
+            resp.raise_for_status()
+        except requests.exceptions.HTTPError as err:
+            _enrich_http_error(err)
+            raise
         return resp
 
     def _put(self, path, data=None, params=None):
@@ -160,7 +241,11 @@ def _put(self, path, data=None, params=None):
             url, json=data, headers=self.data.headers,
             params=params, timeout=self.data.timeout
         )
-        resp.raise_for_status()
+        try:
+            resp.raise_for_status()
+        except requests.exceptions.HTTPError as err:
+            _enrich_http_error(err)
+            raise
         return resp
 
     def _patch(self, path, data=None, params=None):
@@ -180,7 +265,11 @@ def _patch(self, path, data=None, params=None):
             url, json=data, headers=self.data.headers,
             params=params, timeout=self.data.timeout
         )
-        resp.raise_for_status()
+        try:
+            resp.raise_for_status()
+        except requests.exceptions.HTTPError as err:
+            _enrich_http_error(err)
+            raise
         return resp
 
     def _get_paginated(self, input_params, path, offset=None, limit=None):
@@ -231,7 +320,11 @@ def _delete(self, path):
             url, headers=self.data.headers,
             timeout=self.data.timeout
         )
-        resp.raise_for_status()
+        try:
+            resp.raise_for_status()
+        except requests.exceptions.HTTPError as err:
+            _enrich_http_error(err)
+            raise
         return resp