SOF-8099: add client.files to upload, sign and delete account files

.github/workflows/cicd.yml

@@ -9,7 +9,7 @@ concurrency:
 jobs:
 
   run-linter:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         python-version: [3.10.13]
@@ -33,7 +33,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
   run-tests:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         python-version: [3.10.13]
@@ -71,7 +71,7 @@ jobs:
 
   publish:
     needs: [run-linter, run-tests]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     if: github.ref_name == 'main'
 
     steps:

src/py/mat3ra/api_client/__init__.py

@@ -8,6 +8,7 @@
 from .models import Account, APIEnv, AuthContext, AuthEnv
 from .endpoints.bank_materials import BankMaterialEndpoints
 from .endpoints.bank_workflows import BankWorkflowEndpoints
+from .endpoints.files import FileEndpoints
 from .endpoints.jobs import JobEndpoints
 from .endpoints.login import LoginEndpoint
 from .endpoints.logout import LogoutEndpoint

src/py/mat3ra/api_client/client.py

@@ -8,6 +8,7 @@
 from .endpoints.bank_materials import BankMaterialEndpoints
 from .endpoints.bank_workflows import BankWorkflowEndpoints
 from .endpoints.clusters import ClustersEndpoint
+from .endpoints.files import FileEndpoints
 from .endpoints.jobs import JobEndpoints
 from .endpoints.materials import MaterialEndpoints
 from .endpoints.metaproperties import MetaPropertiesEndpoints
@@ -60,14 +61,15 @@ def _init_endpoints(self, timeout_seconds: int) -> None:
         self.bank_materials = BankMaterialEndpoints(*base_args, **base_kwargs)
         self.bank_workflows = BankWorkflowEndpoints(*base_args, **base_kwargs)
         self.clusters = ClustersEndpoint(*base_args, **base_kwargs)
+        self.files = FileEndpoints(*base_args, **base_kwargs)
 
     @staticmethod
     def _resolve_config(
-            host: Optional[str],
-            port: Optional[int],
-            version: Optional[str],
-            secure: Optional[bool],
-            env: APIEnv,
+        host: Optional[str],
+        port: Optional[int],
+        version: Optional[str],
+        secure: Optional[bool],
+        env: APIEnv,
     ) -> Tuple[str, int, str, bool]:
         return (
             host if host is not None else env.host,
@@ -78,11 +80,11 @@ def _resolve_config(
 
     @classmethod
     def _auth_from_env(
-            cls,
-            *,
-            access_token: Optional[str],
-            account_id: Optional[str],
-            auth_token: Optional[str],
+        cls,
+        *,
+        access_token: Optional[str],
+        account_id: Optional[str],
+        auth_token: Optional[str],
     ) -> AuthContext:
         env = cls.auth_env()
         return AuthContext(
@@ -101,16 +103,16 @@ def _validate_auth(auth: AuthContext) -> None:
 
     @classmethod
     def authenticate(
-            cls,
-            *,
-            host: Optional[str] = None,
-            port: Optional[int] = None,
-            version: Optional[str] = None,
-            secure: Optional[bool] = None,
-            access_token: Optional[str] = None,
-            account_id: Optional[str] = None,
-            auth_token: Optional[str] = None,
-            timeout_seconds: int = 60,
+        cls,
+        *,
+        host: Optional[str] = None,
+        port: Optional[int] = None,
+        version: Optional[str] = None,
+        secure: Optional[bool] = None,
+        access_token: Optional[str] = None,
+        account_id: Optional[str] = None,
+        auth_token: Optional[str] = None,
+        timeout_seconds: int = 60,
     ) -> "APIClient":
         host_value, port_value, version_value, secure_value = cls._resolve_config(
             host, port, version, secure, cls.env()
@@ -170,8 +172,9 @@ def get_account(self, name: Optional[str] = None, index: Optional[int] = None) -
 
     def get_default_organization(self) -> Optional[Account]:
         accounts = self._fetch_user_accounts()
-        organizations = [account for account in accounts if
-                         account["entity"].get("type") in ("organization", "enterprise")]
+        organizations = [
+            account for account in accounts if account["entity"].get("type") in ("organization", "enterprise")
+        ]
 
         if not organizations:
             return None

src/py/mat3ra/api_client/endpoints/files.py

@@ -0,0 +1,137 @@
+import json
+import mimetypes
+from typing import Dict, List, Optional, Union
+
+import requests
+
+from . import BaseEndpoint
+from .enums import DEFAULT_API_VERSION, SECURE
+
+DEFAULT_CONTENT_TYPE = "application/octet-stream"
+FileContent = Union[str, bytes]
+
+
+class FileEndpoints(BaseEndpoint):
+    """
+    Files in the account's object storage folder ("Dropbox"). Names are relative to that folder.
+
+    The file bytes never pass through the platform: it signs a URL per file, and the client sends
+    the bytes to storage (or reads them from it) directly.
+
+    Args:
+        host (str): API hostname.
+        port (int): API port number.
+        account_id (str): account ID.
+        auth_token (str): authentication token.
+        version (str): API version.
+        secure (bool): whether to use secure http protocol (https vs http).
+        kwargs (dict): a dictionary of HTTP session options.
+            timeout (int): session timeout in seconds.
+
+    Attributes:
+        name (str): endpoint name.
+        headers (dict): default HTTP headers.
+    """
+
+    def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs):
+        super(FileEndpoints, self).__init__(host, port, version, secure, **kwargs)
+        self.name = "files"
+        self.headers = self.get_headers(account_id, auth_token)
+        self.timeout = kwargs.get("timeout", 60)
+
+    def upload(self, files: Dict[str, FileContent], account_id: Optional[str] = None) -> List[dict]:
+        """
+        Uploads files to the account's folder. Text is uploaded as UTF-8.
+
+        Args:
+            files (dict): file content (str or bytes) by name, relative to the account's folder.
+            account_id (str): account to upload under, the caller's default account if omitted.
+
+        Returns:
+            list[dict]: one record per file, in the order given, with `name`, `key`, `size`,
+                `bucket`, `region` and `provider`.
+        """
+        contents = {name: _to_bytes(content) for name, content in files.items()}
+        uploads = self.get_upload_urls([_describe(name, content) for name, content in contents.items()], account_id)
+        records = []
+        for (name, content), upload in zip(contents.items(), uploads):
+            self._put(upload["signedUrl"], content, upload["headers"])
+            records.append(_to_cloud_file(name, len(content), upload))
+        return records
+
+    def get_upload_urls(self, files: List[dict], account_id: Optional[str] = None) -> List[dict]:
+        """
+        Signs one PUT URL per file. The size and content type are part of the signature.
+
+        Args:
+            files (list[dict]): `name`, `size` (bytes) and `contentType` of each file.
+            account_id (str): account to upload under, the caller's default account if omitted.
+
+        Returns:
+            list[dict]: `key`, `signedUrl`, `headers` (to send with the PUT as given), `bucket`,
+                `region` and `provider` of each file.
+        """
+        return self._sign({"files": files, "operation": "putObject"}, account_id)
+
+    def get_download_urls(self, names: List[str], account_id: Optional[str] = None) -> List[dict]:
+        """
+        Signs one GET URL per file. A name ending in "/" is a folder: every file under it is signed.
+
+        Args:
+            names (list[str]): file names, relative to the account's folder.
+            account_id (str): account the files belong to, the caller's default account if omitted.
+
+        Returns:
+            list[dict]: `key` and `signedUrl` of each file.
+        """
+        return self._sign({"names": names, "operation": "getObject"}, account_id)
+
+    def delete(self, names: List[str], account_id: Optional[str] = None) -> dict:
+        """
+        Deletes files. A name ending in "/" is a folder: everything under it is deleted.
+
+        Args:
+            names (list[str]): file names, relative to the account's folder.
+            account_id (str): account the files belong to, the caller's default account if omitted.
+
+        Returns:
+            dict: `message` confirming the deletion.
+        """
+        data = json.dumps(_with_account({"names": names}, account_id))
+        return self.request("DELETE", self.name, data=data, headers=self.headers)
+
+    def _sign(self, payload: dict, account_id: Optional[str]) -> List[dict]:
+        data = json.dumps(_with_account(payload, account_id))
+        return self.request("POST", f"{self.name}/signed-urls", data=data, headers=self.headers)
+
+    def _put(self, url: str, content: bytes, headers: Dict[str, str]) -> None:
+        """PUTs the bytes to storage. The headers are part of the signature, so they are sent as given."""
+        response = requests.put(url, data=content, headers=headers, timeout=self.timeout)
+        if not response.ok:
+            raise requests.HTTPError(
+                f"Upload to storage failed with HTTP {response.status_code}: {response.text}", response=response
+            )
+
+
+def _to_bytes(content: FileContent) -> bytes:
+    return content.encode("utf-8") if isinstance(content, str) else content
+
+
+def _describe(name: str, content: bytes) -> dict:
+    """What the platform signs for an upload: the name, the exact size and the content type."""
+    return {"name": name, "size": len(content), "contentType": mimetypes.guess_type(name)[0] or DEFAULT_CONTENT_TYPE}
+
+
+def _with_account(payload: dict, account_id: Optional[str]) -> dict:
+    return {**payload, "accountId": account_id} if account_id else payload
+
+
+def _to_cloud_file(name: str, size: int, upload: dict) -> dict:
+    return {
+        "name": name,
+        "key": upload["key"],
+        "size": size,
+        "bucket": upload["bucket"],
+        "region": upload["region"],
+        "provider": upload["provider"],
+    }

tests/py/unit/test_files.py

@@ -0,0 +1,130 @@
+import json
+from unittest import mock
+
+from mat3ra.api_client.endpoints.files import FileEndpoints
+from tests.py.unit import EndpointBaseUnitTest
+
+SIGNED_UPLOAD = {
+    "signedUrl": "https://storage/put?signature",
+    "headers": {"Content-Type": "text/x-python", "x-amz-meta-mode": "33188"},
+    "bucket": "dropbox",
+    "region": "us-west-2",
+    "provider": "aws",
+}
+
+
+class EndpointFilesUnitTest(EndpointBaseUnitTest):
+    """
+    Class for testing files endpoint.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(EndpointFilesUnitTest, self).__init__(*args, **kwargs)
+        self.endpoints = FileEndpoints(self.host, self.port, self.account_id, self.auth_token)
+        self.base_url = f"https://{self.host}:{self.port}/api/{self.version}/files"
+
+    def respond(self, mock_request, data):
+        mock_request.return_value = self.mock_response(json.dumps({"status": "success", "data": data}))
+
+    def sign_each_file(self, mock_request):
+        """The platform answers one signed upload per requested file."""
+        mock_request.side_effect = lambda **request: self.mock_response(
+            json.dumps(
+                {
+                    "status": "success",
+                    "data": [
+                        {**SIGNED_UPLOAD, "key": f"user/{file['name']}"}
+                        for file in json.loads(request["data"])["files"]
+                    ],
+                }
+            )
+        )
+
+    @mock.patch("requests.put")
+    @mock.patch("requests.sessions.Session.request")
+    def test_upload_signs_each_file_with_its_size_and_content_type(self, mock_request, mock_put):
+        self.sign_each_file(mock_request)
+        mock_put.return_value = self.mock_response("")
+
+        self.endpoints.upload({"script.py": "print(1)", "blob.bin": bytes(range(4))}, account_id="organization")
+
+        request = mock_request.call_args[1]
+        self.assertEqual(request["method"], "post")
+        self.assertEqual(request["url"], f"{self.base_url}/signed-urls")
+        self.assertEqual(
+            json.loads(request["data"]),
+            {
+                "files": [
+                    {"name": "script.py", "size": 8, "contentType": "text/x-python"},
+                    {"name": "blob.bin", "size": 4, "contentType": "application/octet-stream"},
+                ],
+                "operation": "putObject",
+                "accountId": "organization",
+            },
+        )
+
+    @mock.patch("requests.put")
+    @mock.patch("requests.sessions.Session.request")
+    def test_upload_puts_the_bytes_with_the_signed_headers(self, mock_request, mock_put):
+        self.sign_each_file(mock_request)
+        mock_put.return_value = self.mock_response("")
+
+        self.endpoints.upload({"script.py": "print('é')", "blob.bin": bytes(range(4))})
+
+        text_put, binary_put = mock_put.call_args_list
+        self.assertEqual(text_put[0][0], SIGNED_UPLOAD["signedUrl"])
+        self.assertEqual(text_put[1]["data"], "print('é')".encode("utf-8"))
+        self.assertEqual(text_put[1]["headers"], SIGNED_UPLOAD["headers"])
+        self.assertEqual(binary_put[1]["data"], bytes(range(4)))
+
+    @mock.patch("requests.put")
+    @mock.patch("requests.sessions.Session.request")
+    def test_upload_returns_a_record_per_file(self, mock_request, mock_put):
+        self.sign_each_file(mock_request)
+        mock_put.return_value = self.mock_response("")
+
+        records = self.endpoints.upload({"script.py": "print(1)"})
+
+        self.assertEqual(
+            records,
+            [
+                {
+                    "name": "script.py",
+                    "key": "user/script.py",
+                    "size": 8,
+                    "bucket": "dropbox",
+                    "region": "us-west-2",
+                    "provider": "aws",
+                }
+            ],
+        )
+        self.assertNotIn("accountId", json.loads(mock_request.call_args[1]["data"]))
+
+    @mock.patch("requests.put")
+    @mock.patch("requests.sessions.Session.request")
+    def test_upload_fails_when_storage_refuses_the_bytes(self, mock_request, mock_put):
+        self.sign_each_file(mock_request)
+        mock_put.return_value = self.mock_response("SignatureDoesNotMatch", status_code=403, reason="Forbidden")
+
+        with self.assertRaisesRegex(Exception, "HTTP 403: SignatureDoesNotMatch"):
+            self.endpoints.upload({"script.py": "print(1)"})
+
+    @mock.patch("requests.sessions.Session.request")
+    def test_get_download_urls(self, mock_request):
+        self.respond(mock_request, [{"key": "user/a.txt", "signedUrl": "https://storage/get"}])
+
+        urls = self.endpoints.get_download_urls(["a.txt"])
+
+        self.assertEqual(urls, [{"key": "user/a.txt", "signedUrl": "https://storage/get"}])
+        self.assertEqual(json.loads(mock_request.call_args[1]["data"]), {"names": ["a.txt"], "operation": "getObject"})
+
+    @mock.patch("requests.sessions.Session.request")
+    def test_delete(self, mock_request):
+        self.respond(mock_request, {"message": "Files a.txt are successfully deleted"})
+
+        self.endpoints.delete(["a.txt"])
+
+        request = mock_request.call_args[1]
+        self.assertEqual(request["method"], "delete")
+        self.assertEqual(request["url"], self.base_url)
+        self.assertEqual(json.loads(request["data"]), {"names": ["a.txt"]})