feat: release v1.0.0-beta.1

2026-07-27 12:44:31 +00:00 · 2026-05-29 17:33:12 +08:00
commit 36b0537676
290 changed files with 72303 additions and 0 deletions
@@ -0,0 +1,461 @@
+"""Memory file reader — direct read of memory pipeline artifacts (persona.md,
+scene_blocks/*.md) from object storage with STS credential management.
+
+The SDK user calls ``client.read_file(path)`` and gets the file content back.
+Under the hood, we:
+  1. Fetch STS temporary credentials from the platform (``POST /v2/cos/secret``)
+  2. Cache credentials until they expire (auto-refresh)
+  3. Sign a COS V5 GET request with the STS credentials
+  4. Return the file content as a string
+
+Storage backend (currently COS) is an implementation detail — the public API
+is intentionally storage-agnostic.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import logging
+import re
+import threading
+import time
+from typing import Any, Dict, Optional
+from urllib.parse import urlparse
+
+import httpx
+
+from .errors import TDAMError
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# COS URL parser
+# ---------------------------------------------------------------------------
+
+def _parse_cos_url(cos_url: str) -> tuple[str, str]:
+    """Parse CosUrl like ``https://bucket.cos.region.myqcloud.com`` → (bucket, region)."""
+    host = urlparse(cos_url).hostname or ""
+    # Pattern: {bucket}.cos.{region}.myqcloud.com
+    m = re.match(r"^(.+?)\.cos\.(.+?)\.myqcloud\.com$", host)
+    if m:
+        return m.group(1), m.group(2)
+    raise TDAMError(
+        code=-1,
+        message=f"Cannot parse CosUrl: {cos_url!r} (expected {{bucket}}.cos.{{region}}.myqcloud.com)",
+    )
+
+
+# ---------------------------------------------------------------------------
+# STS Credential
+# ---------------------------------------------------------------------------
+
+class StsCredential:
+    """Parsed STS credential from ``POST /v2/cos/secret``.
+
+    Platform response format::
+
+        {
+            "CosUrl": "https://{bucket}.cos.{region}.myqcloud.com",
+            "TmpSecretId": "...",
+            "TmpSecretKey": "...",
+            "TmpToken": "...",
+            "ExpirationTime": "2026-05-15T16:44:49+08:00",
+            "PathPrefix": "memory_v2/cos_data/mem-xxx"
+        }
+    """
+
+    __slots__ = (
+        "tmp_secret_id", "tmp_secret_key", "token",
+        "bucket", "region", "prefix", "expires_at_epoch",
+    )
+
+    def __init__(self, data: Dict[str, Any]) -> None:
+        self.tmp_secret_id: str = data["TmpSecretId"]
+        self.tmp_secret_key: str = data["TmpSecretKey"]
+        self.token: str = data.get("TmpToken", "")
+        # Parse bucket + region from CosUrl
+        self.bucket: str
+        self.region: str
+        self.bucket, self.region = _parse_cos_url(data["CosUrl"])
+        # PathPrefix — ensure trailing slash for key concatenation
+        prefix = data.get("PathPrefix", "")
+        self.prefix: str = prefix if prefix.endswith("/") else f"{prefix}/"
+        # Parse ISO 8601 → epoch seconds
+        expires_str = data.get("ExpirationTime", "")
+        if expires_str:
+            from datetime import datetime, timezone
+            try:
+                dt = datetime.fromisoformat(expires_str.replace("Z", "+00:00"))
+                self.expires_at_epoch = dt.timestamp()
+            except Exception:
+                self.expires_at_epoch = time.time() + 1800  # 30 min fallback
+        else:
+            self.expires_at_epoch = time.time() + 1800
+
+    def is_valid(self, buffer_seconds: float = 120) -> bool:
+        """Check if credential is still valid (with 2-minute buffer)."""
+        return time.time() < (self.expires_at_epoch - buffer_seconds)
+
+    @property
+    def cos_host(self) -> str:
+        return f"{self.bucket}.cos.{self.region}.myqcloud.com"
+
+
+# ---------------------------------------------------------------------------
+# STS Credential Manager
+# ---------------------------------------------------------------------------
+
+class StsCredentialManager:
+    """Thread-safe STS credential cache with auto-refresh.
+
+    - Fetches STS from platform ``POST /v2/cos/secret``
+    - Caches until expiry (with 2-minute buffer)
+    - Coalesces concurrent refresh requests
+    """
+
+    def __init__(
+        self,
+        endpoint: str,
+        api_key: str,
+        service_id: str,
+        buffer_seconds: float = 120,
+        timeout: float = 30,
+    ) -> None:
+        self._endpoint = endpoint.rstrip("/")
+        self._api_key = api_key
+        self._service_id = service_id
+        self._buffer = buffer_seconds
+        self._timeout = timeout
+        self._credential: Optional[StsCredential] = None
+        self._lock = threading.Lock()
+        self._client: Optional[httpx.Client] = None
+
+    def get_credential(self) -> StsCredential:
+        """Get a valid STS credential (cached or freshly fetched)."""
+        if self._credential and self._credential.is_valid(self._buffer):
+            return self._credential
+
+        with self._lock:
+            # Double-check after acquiring lock
+            if self._credential and self._credential.is_valid(self._buffer):
+                return self._credential
+            return self._refresh()
+
+    def invalidate(self) -> None:
+        """Force invalidate cached credential (e.g. on 403)."""
+        with self._lock:
+            self._credential = None
+
+    def _refresh(self) -> StsCredential:
+        logger.debug("[cos] Refreshing STS credential via POST /v2/cos/secret ...")
+        if self._client is None:
+            self._client = httpx.Client(timeout=self._timeout)
+
+        url = f"{self._endpoint}/v2/cos/secret"
+        headers = {
+            "Authorization": f"Bearer {self._api_key}",
+            "x-tdai-service-id": self._service_id,
+            "Content-Type": "application/json",
+        }
+        resp = self._client.post(url, json={}, headers=headers)
+        resp.raise_for_status()
+        data = resp.json()
+
+        cred = StsCredential(data)
+        self._credential = cred
+        logger.debug("[cos] STS refreshed: bucket=%s prefix=%s expires=%.0f",
+                     cred.bucket, cred.prefix, cred.expires_at_epoch)
+        return cred
+
+    def close(self) -> None:
+        if self._client is not None:
+            self._client.close()
+
+
+# ---------------------------------------------------------------------------
+# Async STS Credential Manager
+# ---------------------------------------------------------------------------
+
+class AsyncStsCredentialManager:
+    """Async variant of StsCredentialManager."""
+
+    def __init__(
+        self,
+        endpoint: str,
+        api_key: str,
+        service_id: str,
+        buffer_seconds: float = 120,
+        timeout: float = 30,
+    ) -> None:
+        self._endpoint = endpoint.rstrip("/")
+        self._api_key = api_key
+        self._service_id = service_id
+        self._buffer = buffer_seconds
+        self._timeout = timeout
+        self._credential: Optional[StsCredential] = None
+        import asyncio
+        self._lock = asyncio.Lock()
+        self._client: Optional[httpx.AsyncClient] = None
+
+    async def get_credential(self) -> StsCredential:
+        if self._credential and self._credential.is_valid(self._buffer):
+            return self._credential
+
+        async with self._lock:
+            if self._credential and self._credential.is_valid(self._buffer):
+                return self._credential
+            return await self._refresh()
+
+    def invalidate(self) -> None:
+        self._credential = None
+
+    async def _refresh(self) -> StsCredential:
+        logger.debug("[cos] Refreshing STS credential (async) via POST /v2/cos/secret ...")
+        if self._client is None:
+            self._client = httpx.AsyncClient(timeout=self._timeout)
+
+        url = f"{self._endpoint}/v2/cos/secret"
+        headers = {
+            "Authorization": f"Bearer {self._api_key}",
+            "x-tdai-service-id": self._service_id,
+            "Content-Type": "application/json",
+        }
+        resp = await self._client.post(url, json={}, headers=headers)
+        resp.raise_for_status()
+        data = resp.json()
+
+        cred = StsCredential(data)
+        self._credential = cred
+        return cred
+
+    async def close(self) -> None:
+        if self._client is not None:
+            await self._client.aclose()
+
+
+# ---------------------------------------------------------------------------
+# COS V5 Signature
+# ---------------------------------------------------------------------------
+
+def _cos_v5_sign(
+    secret_id: str,
+    secret_key: str,
+    method: str,
+    path: str,
+    host: str,
+    start_time: Optional[int] = None,
+    end_time: Optional[int] = None,
+) -> str:
+    """Generate COS V5 Authorization header value for a GET request.
+
+    References:
+      https://cloud.tencent.com/document/product/436/7778
+    """
+    now = int(time.time())
+    q_sign_time = f"{start_time or (now - 60)};{end_time or (now + 600)}"
+    q_key_time = q_sign_time
+
+    # Step 1: SignKey
+    sign_key = hmac.new(
+        secret_key.encode("utf-8"),
+        q_key_time.encode("utf-8"),
+        hashlib.sha1,
+    ).hexdigest()
+
+    # Step 2: HttpString
+    # For GET with no query params and only host header
+    http_string = f"{method.lower()}\n{path}\n\nhost={host}\n"
+
+    # Step 3: StringToSign
+    sha1_http_string = hashlib.sha1(http_string.encode("utf-8")).hexdigest()
+    string_to_sign = f"sha1\n{q_sign_time}\n{sha1_http_string}\n"
+
+    # Step 4: Signature
+    signature = hmac.new(
+        sign_key.encode("utf-8"),
+        string_to_sign.encode("utf-8"),
+        hashlib.sha1,
+    ).hexdigest()
+
+    return (
+        f"q-sign-algorithm=sha1"
+        f"&q-ak={secret_id}"
+        f"&q-sign-time={q_sign_time}"
+        f"&q-key-time={q_key_time}"
+        f"&q-header-list=host"
+        f"&q-url-param-list="
+        f"&q-signature={signature}"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Memory File Reader (sync)
+# ---------------------------------------------------------------------------
+
+class MemoryFileReader:
+    """Sync memory file reader with STS auto-management.
+
+    Reads memory pipeline artifacts (persona.md, scene_blocks/*.md, …) from
+    object storage. The storage backend is COS today but the public API is
+    storage-agnostic.
+
+    Usage::
+
+        reader = MemoryFileReader(sts_manager)
+        content = reader.read("scene_blocks/cooking-recipes.md")
+    """
+
+    def __init__(
+        self,
+        sts_manager: StsCredentialManager,
+        timeout: float = 30,
+        client: Optional[httpx.Client] = None,
+    ) -> None:
+        self._sts = sts_manager
+        self._client = client or httpx.Client(timeout=timeout)
+
+    def read(self, path: str) -> str:
+        """Read a memory file by relative path.
+
+        The final COS key is: ``{prefix}{path}``
+
+        Returns file content as UTF-8 string.
+        Raises ``TDAMError`` on failure.
+        """
+        cred = self._sts.get_credential()
+        full_key = f"{cred.prefix}{path}"
+        cos_path = f"/{full_key}"
+        host = cred.cos_host
+
+        auth = _cos_v5_sign(
+            secret_id=cred.tmp_secret_id,
+            secret_key=cred.tmp_secret_key,
+            method="GET",
+            path=cos_path,
+            host=host,
+        )
+
+        headers: Dict[str, str] = {
+            "Host": host,
+            "Authorization": auth,
+        }
+        if cred.token:
+            headers["x-cos-security-token"] = cred.token
+
+        url = f"https://{host}{cos_path}"
+        logger.debug("[cos] GET %s", url)
+
+        resp = self._client.get(url, headers=headers)
+
+        if resp.status_code == 403:
+            # Invalidate and retry once
+            logger.warning("[cos] 403 on GET %s — invalidating STS and retrying", path)
+            self._sts.invalidate()
+            cred = self._sts.get_credential()
+            full_key = f"{cred.prefix}{path}"
+            cos_path = f"/{full_key}"
+            host = cred.cos_host
+            auth = _cos_v5_sign(
+                secret_id=cred.tmp_secret_id,
+                secret_key=cred.tmp_secret_key,
+                method="GET",
+                path=cos_path,
+                host=host,
+            )
+            headers = {"Host": host, "Authorization": auth}
+            if cred.token:
+                headers["x-cos-security-token"] = cred.token
+            url = f"https://{host}{cos_path}"
+            resp = self._client.get(url, headers=headers)
+
+        if resp.status_code == 404:
+            raise TDAMError(code=404, message=f"File not found: {path}")
+
+        if resp.status_code != 200:
+            raise TDAMError(
+                code=resp.status_code,
+                message=f"COS GET failed: HTTP {resp.status_code} — {resp.text[:200]}",
+            )
+
+        return resp.text
+
+    def close(self) -> None:
+        if isinstance(self._client, httpx.Client):
+            self._client.close()
+
+
+# ---------------------------------------------------------------------------
+# Async Memory File Reader
+# ---------------------------------------------------------------------------
+
+class AsyncMemoryFileReader:
+    """Async memory file reader with STS auto-management."""
+
+    def __init__(
+        self,
+        sts_manager: AsyncStsCredentialManager,
+        timeout: float = 30,
+        client: Optional[httpx.AsyncClient] = None,
+    ) -> None:
+        self._sts = sts_manager
+        self._client = client or httpx.AsyncClient(timeout=timeout)
+
+    async def read(self, path: str) -> str:
+        cred = await self._sts.get_credential()
+        full_key = f"{cred.prefix}{path}"
+        cos_path = f"/{full_key}"
+        host = cred.cos_host
+
+        auth = _cos_v5_sign(
+            secret_id=cred.tmp_secret_id,
+            secret_key=cred.tmp_secret_key,
+            method="GET",
+            path=cos_path,
+            host=host,
+        )
+
+        headers: Dict[str, str] = {
+            "Host": host,
+            "Authorization": auth,
+        }
+        if cred.token:
+            headers["x-cos-security-token"] = cred.token
+
+        url = f"https://{host}{cos_path}"
+        resp = await self._client.get(url, headers=headers)
+
+        if resp.status_code == 403:
+            self._sts.invalidate()
+            cred = await self._sts.get_credential()
+            full_key = f"{cred.prefix}{path}"
+            cos_path = f"/{full_key}"
+            host = cred.cos_host
+            auth = _cos_v5_sign(
+                secret_id=cred.tmp_secret_id,
+                secret_key=cred.tmp_secret_key,
+                method="GET",
+                path=cos_path,
+                host=host,
+            )
+            headers = {"Host": host, "Authorization": auth}
+            if cred.token:
+                headers["x-cos-security-token"] = cred.token
+            url = f"https://{host}{cos_path}"
+            resp = await self._client.get(url, headers=headers)
+
+        if resp.status_code == 404:
+            raise TDAMError(code=404, message=f"File not found: {path}")
+
+        if resp.status_code != 200:
+            raise TDAMError(
+                code=resp.status_code,
+                message=f"COS GET failed: HTTP {resp.status_code} — {resp.text[:200]}",
+            )
+
+        return resp.text
+
+    async def close(self) -> None:
+        if isinstance(self._client, httpx.AsyncClient):
+            await self._client.aclose()