feat(migration): phase 1 — Session + Tracer + RunConfig factory
Three foundation modules per PLAYBOOK §2.8 / §2.9 / §2.10 with all
relevant R2/R3 corrections (C7, C10, C11, C16, C21):
strix/llm/strix_session.py SessionABC wrapper around the
legacy MemoryCompressor; on any
compression failure, returns
uncompressed history and
permanently disables compression
for the rest of the run (C10 +
Round 3.4 W5/E2).
strix/telemetry/strix_processor.py SDK TracingProcessor that writes
events.jsonl in our schema. All
hooks SYNC per ABC (F3); writes
protected by per-path
threading.Lock (C7); OSError
swallowed and logged (C16); PII
scrubbed via the existing
TelemetrySanitizer.
strix/run_config_factory.py make_run_config() with our
defaults: parallel_tool_calls=
False (C1 Phase-1 safe default),
retry policy explicitly excludes
401/403/400 (C11), reasoning
effort + model_settings_override
merge path (C21).
make_agent_context() returns the
canonical per-agent dict
including is_whitebox/diff_scope/
run_id (C21).
32 new smoke tests (197/197 total). mypy strict + ruff clean. Per-file
ignores added for tests/** S105/PT018 and for the two new src modules'
intentional broad-Exception catches (BLE001).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,153 @@
|
||||
"""Phase 1 smoke tests for StrixSession (memory compression wrapper)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from agents.memory.session import SessionABC
|
||||
|
||||
from strix.llm.strix_session import StrixSession
|
||||
|
||||
|
||||
class _FakeUnderlying(SessionABC):
|
||||
"""In-memory SessionABC used to drive StrixSession in tests."""
|
||||
|
||||
def __init__(self, items: list[Any] | None = None) -> None:
|
||||
self.items: list[Any] = list(items or [])
|
||||
self.session_id = "fake-session"
|
||||
|
||||
async def get_items(self, limit: int | None = None) -> list[Any]:
|
||||
if limit is None:
|
||||
return list(self.items)
|
||||
return list(self.items[-limit:])
|
||||
|
||||
async def add_items(self, items: list[Any]) -> None:
|
||||
self.items.extend(items)
|
||||
|
||||
async def pop_item(self) -> Any | None:
|
||||
return self.items.pop() if self.items else None
|
||||
|
||||
async def clear_session(self) -> None:
|
||||
self.items.clear()
|
||||
|
||||
|
||||
class _CompressorOK:
|
||||
"""Compressor stand-in that compresses by keeping the last item."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
def compress_history(self, messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
self.calls += 1
|
||||
return messages[-1:] if len(messages) > 1 else messages
|
||||
|
||||
|
||||
class _CompressorBoom:
|
||||
"""Compressor stand-in that always raises."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
def compress_history(self, messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
self.calls += 1
|
||||
raise RuntimeError("compressor offline")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def items() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"role": "user", "content": "first"},
|
||||
{"role": "assistant", "content": "second"},
|
||||
{"role": "user", "content": "third"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_items_compresses_when_compressor_ok(items: list[dict[str, Any]]) -> None:
|
||||
underlying = _FakeUnderlying(items)
|
||||
session = StrixSession(underlying, compressor=_CompressorOK())
|
||||
|
||||
out = await session.get_items()
|
||||
assert len(out) == 1
|
||||
assert out[0]["content"] == "third"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_items_returns_empty_without_calling_compressor() -> None:
|
||||
"""If underlying has no items, don't even invoke the compressor."""
|
||||
underlying = _FakeUnderlying([])
|
||||
compressor = _CompressorOK()
|
||||
session = StrixSession(underlying, compressor=compressor)
|
||||
|
||||
out = await session.get_items()
|
||||
assert out == []
|
||||
assert compressor.calls == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_items_falls_back_to_uncompressed_on_exception(
|
||||
items: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""C10 (AUDIT_R2): compressor failure must not tear down the run."""
|
||||
underlying = _FakeUnderlying(items)
|
||||
compressor = _CompressorBoom()
|
||||
session = StrixSession(underlying, compressor=compressor)
|
||||
|
||||
out = await session.get_items()
|
||||
# Uncompressed history returned.
|
||||
assert out == items
|
||||
# Flag set so subsequent calls skip the compressor.
|
||||
assert session.compression_disabled is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compressor_disabled_after_first_failure(items: list[dict[str, Any]]) -> None:
|
||||
"""Round 3.4 §E2 / W5 — once the compressor fails, skip it forever."""
|
||||
underlying = _FakeUnderlying(items)
|
||||
compressor = _CompressorBoom()
|
||||
session = StrixSession(underlying, compressor=compressor)
|
||||
|
||||
# First call: compressor invoked, raises, flag set.
|
||||
await session.get_items()
|
||||
assert compressor.calls == 1
|
||||
assert session.compression_disabled is True
|
||||
|
||||
# Second + third call: compressor short-circuited.
|
||||
await session.get_items()
|
||||
await session.get_items()
|
||||
assert compressor.calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_writes_pass_through(items: list[dict[str, Any]]) -> None:
|
||||
underlying = _FakeUnderlying()
|
||||
session = StrixSession(underlying, compressor=_CompressorOK())
|
||||
|
||||
await session.add_items(items)
|
||||
assert underlying.items == items
|
||||
|
||||
popped = await session.pop_item()
|
||||
assert popped == items[-1]
|
||||
|
||||
await session.clear_session()
|
||||
assert underlying.items == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_id_passes_through() -> None:
|
||||
underlying = _FakeUnderlying()
|
||||
session = StrixSession(underlying, compressor=_CompressorOK())
|
||||
assert session.session_id == "fake-session"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_items_respects_limit(items: list[dict[str, Any]]) -> None:
|
||||
"""``limit`` is forwarded to the underlying session before compression."""
|
||||
underlying = _FakeUnderlying(items)
|
||||
session = StrixSession(underlying, compressor=_CompressorOK())
|
||||
|
||||
out = await session.get_items(limit=2)
|
||||
# Underlying returned last 2 items; compressor kept the last 1.
|
||||
assert len(out) == 1
|
||||
assert out[0]["content"] == "third"
|
||||
@@ -0,0 +1,201 @@
|
||||
"""Phase 1 smoke tests for StrixTracingProcessor."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from strix.telemetry.strix_processor import StrixTracingProcessor
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeSpanData:
|
||||
"""Minimal stand-in for an SDK SpanData class."""
|
||||
|
||||
name: str = "FunctionSpanData" # used by class .__name__
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def export(self) -> dict[str, Any]:
|
||||
return dict(self.payload)
|
||||
|
||||
|
||||
# Concrete SpanData subclasses so the processor's ``_span_kind`` heuristic
|
||||
# (drop ``SpanData`` suffix, lowercase) produces stable event_types.
|
||||
class FunctionSpanData(_FakeSpanData):
|
||||
pass
|
||||
|
||||
|
||||
class GenerationSpanData(_FakeSpanData):
|
||||
pass
|
||||
|
||||
|
||||
class AgentSpanData(_FakeSpanData):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeSpan:
|
||||
span_id: str
|
||||
trace_id: str
|
||||
span_data: _FakeSpanData
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeTrace:
|
||||
trace_id: str
|
||||
name: str = "test-workflow"
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def export(self) -> dict[str, Any]:
|
||||
return {"name": self.name, "metadata": self.metadata}
|
||||
|
||||
|
||||
def _read_events(path: Path) -> list[dict[str, Any]]:
|
||||
return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_dir(tmp_path: Path) -> Path:
|
||||
return tmp_path / "strix_runs" / "test-run"
|
||||
|
||||
|
||||
def test_constructor_creates_run_dir(run_dir: Path) -> None:
|
||||
StrixTracingProcessor(run_dir=run_dir)
|
||||
assert run_dir.exists()
|
||||
|
||||
|
||||
def test_on_trace_start_writes_run_started(run_dir: Path) -> None:
|
||||
p = StrixTracingProcessor(run_dir=run_dir)
|
||||
p.on_trace_start(_FakeTrace(trace_id="t-1", metadata={"scan_id": "abc"}))
|
||||
|
||||
events = _read_events(p.events_path)
|
||||
assert events == [
|
||||
{
|
||||
"event_type": "run.started",
|
||||
"trace_id": "t-1",
|
||||
"metadata": {"name": "test-workflow", "metadata": {"scan_id": "abc"}},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_on_trace_end_writes_run_completed(run_dir: Path) -> None:
|
||||
p = StrixTracingProcessor(run_dir=run_dir)
|
||||
p.on_trace_end(_FakeTrace(trace_id="t-1"))
|
||||
|
||||
events = _read_events(p.events_path)
|
||||
assert events == [{"event_type": "run.completed", "trace_id": "t-1"}]
|
||||
|
||||
|
||||
def test_span_start_and_end_emit_typed_events(run_dir: Path) -> None:
|
||||
"""``GenerationSpanData`` → ``generation.started`` / ``generation.completed``."""
|
||||
p = StrixTracingProcessor(run_dir=run_dir)
|
||||
span = _FakeSpan(
|
||||
span_id="s-1",
|
||||
trace_id="t-1",
|
||||
span_data=GenerationSpanData(payload={"model": "gpt-foo"}),
|
||||
)
|
||||
|
||||
p.on_span_start(span)
|
||||
p.on_span_end(span)
|
||||
|
||||
events = _read_events(p.events_path)
|
||||
assert [e["event_type"] for e in events] == [
|
||||
"generation.started",
|
||||
"generation.completed",
|
||||
]
|
||||
assert events[0]["span_id"] == "s-1"
|
||||
assert events[0]["data"] == {"model": "gpt-foo"}
|
||||
|
||||
|
||||
def test_concurrent_writes_yield_valid_jsonl(run_dir: Path) -> None:
|
||||
"""C7 (AUDIT_R2): per-path lock prevents JSONL corruption under contention."""
|
||||
p = StrixTracingProcessor(run_dir=run_dir)
|
||||
|
||||
def writer(idx: int) -> None:
|
||||
for i in range(50):
|
||||
p._emit({"event_type": "synthetic", "writer": idx, "i": i})
|
||||
|
||||
threads = [threading.Thread(target=writer, args=(i,)) for i in range(10)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
# 10 threads x 50 events = 500 lines; all valid JSON.
|
||||
lines = p.events_path.read_text().splitlines()
|
||||
assert len(lines) == 500
|
||||
for line in lines:
|
||||
json.loads(line) # raises on corrupt line
|
||||
|
||||
|
||||
def test_emit_swallows_oserror_and_logs(
|
||||
run_dir: Path,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""C16 (AUDIT_R3): a write failure must NOT propagate."""
|
||||
p = StrixTracingProcessor(run_dir=run_dir)
|
||||
# Make events_path point to a directory so open(..., "a") raises.
|
||||
p.events_path = run_dir
|
||||
|
||||
with caplog.at_level("ERROR", logger="strix.telemetry.strix_processor"):
|
||||
p._emit({"event_type": "boom"})
|
||||
|
||||
assert any("Failed to append" in rec.message for rec in caplog.records)
|
||||
|
||||
|
||||
def test_span_export_failure_does_not_propagate(run_dir: Path) -> None:
|
||||
"""If span_data.export raises, we still emit an event with data=None."""
|
||||
p = StrixTracingProcessor(run_dir=run_dir)
|
||||
|
||||
class _BoomSpanData:
|
||||
def export(self) -> dict[str, Any]:
|
||||
raise RuntimeError("nope")
|
||||
|
||||
# Reuse the lowercase rule: class name has no "SpanData" suffix → "boomspandata"
|
||||
# would not be ideal; use a properly-named subclass.
|
||||
class FunctionSpanDataBroken(_BoomSpanData):
|
||||
pass
|
||||
|
||||
span = _FakeSpan(span_id="s-1", trace_id="t-1", span_data=FunctionSpanDataBroken())
|
||||
p.on_span_end(span)
|
||||
|
||||
events = _read_events(p.events_path)
|
||||
assert len(events) == 1
|
||||
assert events[0]["data"] is None
|
||||
|
||||
|
||||
def test_pii_scrubbed_via_sanitizer(run_dir: Path) -> None:
|
||||
"""Sanitizer is invoked on every emit before write."""
|
||||
seen: list[Any] = []
|
||||
|
||||
class _StubSanitizer:
|
||||
def sanitize(self, data: Any, key_hint: str | None = None) -> Any:
|
||||
seen.append(data)
|
||||
# Replace any "secret" string with [REDACTED].
|
||||
if isinstance(data, dict):
|
||||
clean = {k: "[REDACTED]" if k == "api_key" else v for k, v in data.items()}
|
||||
if "metadata" in clean and isinstance(clean["metadata"], dict):
|
||||
md = dict(clean["metadata"])
|
||||
md.pop("api_key", None)
|
||||
clean["metadata"] = md
|
||||
return clean
|
||||
return data
|
||||
|
||||
p = StrixTracingProcessor(run_dir=run_dir, sanitizer=_StubSanitizer())
|
||||
p._emit({"event_type": "test", "api_key": "sk-very-secret"})
|
||||
|
||||
events = _read_events(p.events_path)
|
||||
assert events[0]["api_key"] == "[REDACTED]"
|
||||
assert seen and seen[0]["api_key"] == "sk-very-secret"
|
||||
|
||||
|
||||
def test_force_flush_and_shutdown_are_noops(run_dir: Path) -> None:
|
||||
p = StrixTracingProcessor(run_dir=run_dir)
|
||||
# Should not raise.
|
||||
p.force_flush()
|
||||
p.shutdown()
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Phase 1 smoke tests for make_run_config / make_agent_context."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from agents import RunConfig
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.retry import ModelRetryBackoffSettings
|
||||
|
||||
from strix.orchestration.bus import AgentMessageBus
|
||||
from strix.run_config_factory import (
|
||||
_RETRYABLE_HTTP_STATUSES,
|
||||
make_agent_context,
|
||||
make_run_config,
|
||||
)
|
||||
|
||||
|
||||
def test_make_run_config_returns_run_config() -> None:
|
||||
cfg = make_run_config(sandbox_session=None)
|
||||
assert isinstance(cfg, RunConfig)
|
||||
|
||||
|
||||
def test_default_parallel_tool_calls_is_false() -> None:
|
||||
"""C1 (AUDIT.md): Phase 1 default is sequential to match legacy tool server."""
|
||||
cfg = make_run_config(sandbox_session=None)
|
||||
assert cfg.model_settings is not None
|
||||
assert cfg.model_settings.parallel_tool_calls is False
|
||||
|
||||
|
||||
def test_default_tool_choice_is_required() -> None:
|
||||
cfg = make_run_config(sandbox_session=None)
|
||||
assert cfg.model_settings is not None
|
||||
assert cfg.model_settings.tool_choice == "required"
|
||||
|
||||
|
||||
def test_call_model_input_filter_is_wired() -> None:
|
||||
cfg = make_run_config(sandbox_session=None)
|
||||
assert cfg.call_model_input_filter is not None
|
||||
# Wired to inject_messages_filter (validated by name to keep import light).
|
||||
assert cfg.call_model_input_filter.__name__ == "inject_messages_filter"
|
||||
|
||||
|
||||
def test_retry_settings_have_max_retries_5() -> None:
|
||||
cfg = make_run_config(sandbox_session=None)
|
||||
assert cfg.model_settings is not None
|
||||
retry = cfg.model_settings.retry
|
||||
assert retry is not None
|
||||
assert retry.max_retries == 5
|
||||
|
||||
|
||||
def test_retry_backoff_uses_strix_defaults() -> None:
|
||||
"""Mirrors legacy llm.py: min(90, 2*2^n) with initial 2s, max 90s, x2."""
|
||||
cfg = make_run_config(sandbox_session=None)
|
||||
assert cfg.model_settings is not None
|
||||
retry = cfg.model_settings.retry
|
||||
assert retry is not None
|
||||
backoff = retry.backoff
|
||||
assert isinstance(backoff, ModelRetryBackoffSettings)
|
||||
assert backoff.initial_delay == 2.0
|
||||
assert backoff.max_delay == 90.0
|
||||
assert backoff.multiplier == 2.0
|
||||
|
||||
|
||||
def test_retry_http_codes_exclude_401_403_400() -> None:
|
||||
"""C11 (AUDIT_R2): auth/validation errors must NOT be in the retry list."""
|
||||
assert 401 not in _RETRYABLE_HTTP_STATUSES
|
||||
assert 403 not in _RETRYABLE_HTTP_STATUSES
|
||||
assert 400 not in _RETRYABLE_HTTP_STATUSES
|
||||
# And 429 / 5xx must be present.
|
||||
for code in (429, 500, 502, 503, 504):
|
||||
assert code in _RETRYABLE_HTTP_STATUSES
|
||||
|
||||
|
||||
def test_trace_include_sensitive_data_is_false() -> None:
|
||||
cfg = make_run_config(sandbox_session=None)
|
||||
assert cfg.trace_include_sensitive_data is False
|
||||
|
||||
|
||||
def test_model_settings_override_merges() -> None:
|
||||
"""C21 (AUDIT_R3): per-call override path."""
|
||||
override = ModelSettings(tool_choice="auto", parallel_tool_calls=True)
|
||||
cfg = make_run_config(sandbox_session=None, model_settings_override=override)
|
||||
assert cfg.model_settings is not None
|
||||
assert cfg.model_settings.tool_choice == "auto"
|
||||
assert cfg.model_settings.parallel_tool_calls is True
|
||||
# Retry settings (not in override) preserved from base.
|
||||
assert cfg.model_settings.retry is not None
|
||||
assert cfg.model_settings.retry.max_retries == 5
|
||||
|
||||
|
||||
def test_reasoning_effort_propagates() -> None:
|
||||
cfg = make_run_config(sandbox_session=None, reasoning_effort="high")
|
||||
assert cfg.model_settings is not None
|
||||
assert cfg.model_settings.reasoning is not None
|
||||
assert cfg.model_settings.reasoning.effort == "high"
|
||||
|
||||
|
||||
def test_max_turns_default_is_300() -> None:
|
||||
"""Mirrors legacy AgentState.max_iterations=300 (HARNESS_WIKI §5.2)."""
|
||||
# max_turns is RunConfig-level; we default 300 in make_agent_context for
|
||||
# the per-agent context dict. RunConfig itself sets max_turns at run call
|
||||
# time via Runner.run(max_turns=...). Verify our context.
|
||||
bus = AgentMessageBus()
|
||||
ctx = make_agent_context(
|
||||
bus=bus,
|
||||
sandbox_session=None,
|
||||
sandbox_token=None,
|
||||
tool_server_host_port=None,
|
||||
caido_host_port=None,
|
||||
agent_id="root",
|
||||
agent_name="root",
|
||||
parent_id=None,
|
||||
tracer=None,
|
||||
)
|
||||
assert ctx["max_turns"] == 300
|
||||
|
||||
|
||||
def test_make_agent_context_full_shape() -> None:
|
||||
"""C21 — context dict carries every field tools/hooks reach for."""
|
||||
bus = AgentMessageBus()
|
||||
ctx = make_agent_context(
|
||||
bus=bus,
|
||||
sandbox_session=None,
|
||||
sandbox_token="bearer-xyz",
|
||||
tool_server_host_port=48081,
|
||||
caido_host_port=48080,
|
||||
agent_id="agent-1",
|
||||
agent_name="root",
|
||||
parent_id=None,
|
||||
tracer="not-a-real-tracer",
|
||||
is_whitebox=True,
|
||||
diff_scope={"changed_files": ["src/app.py"]},
|
||||
run_id="strix_runs/abc_def",
|
||||
)
|
||||
|
||||
assert ctx["bus"] is bus
|
||||
assert ctx["agent_id"] == "agent-1"
|
||||
assert ctx["parent_id"] is None
|
||||
assert ctx["agent_finish_called"] is False
|
||||
assert ctx["turn_count"] == 0
|
||||
assert ctx["is_whitebox"] is True
|
||||
assert ctx["diff_scope"] == {"changed_files": ["src/app.py"]}
|
||||
assert ctx["run_id"] == "strix_runs/abc_def"
|
||||
assert ctx["sandbox_token"] == "bearer-xyz"
|
||||
assert ctx["tool_server_host_port"] == 48081
|
||||
assert ctx["caido_host_port"] == 48080
|
||||
|
||||
|
||||
def test_make_agent_context_is_whitebox_defaults_false() -> None:
|
||||
bus = AgentMessageBus()
|
||||
ctx = make_agent_context(
|
||||
bus=bus,
|
||||
sandbox_session=None,
|
||||
sandbox_token=None,
|
||||
tool_server_host_port=None,
|
||||
caido_host_port=None,
|
||||
agent_id="r",
|
||||
agent_name="root",
|
||||
parent_id=None,
|
||||
tracer=None,
|
||||
)
|
||||
assert ctx["is_whitebox"] is False
|
||||
assert ctx["diff_scope"] is None
|
||||
|
||||
|
||||
def test_sandbox_config_omitted_when_no_session() -> None:
|
||||
cfg = make_run_config(sandbox_session=None)
|
||||
assert cfg.sandbox is None
|
||||
|
||||
|
||||
def test_model_default_is_strix_claude() -> None:
|
||||
"""Production default per AUDIT/PLAYBOOK convention."""
|
||||
cfg = make_run_config(sandbox_session=None)
|
||||
assert cfg.model == "strix/claude-sonnet-4.6"
|
||||
|
||||
|
||||
def test_multi_provider_is_built() -> None:
|
||||
"""Verify the factory wires our custom MultiProvider, not the SDK default."""
|
||||
cfg = make_run_config(sandbox_session=None)
|
||||
# MultiProvider is opaque, but our build_multi_provider returns
|
||||
# an instance with our prefix routes installed.
|
||||
assert cfg.model_provider is not None
|
||||
Reference in New Issue
Block a user