refactor: lean on SDK for tracing + native session resume; nuke OTEL/Traceloop
The SDK ships its own tracing pipeline (``agents.tracing``) plus ``SQLiteSession`` for native conversation persistence. Strix's custom OTEL bootstrap + Traceloop integration was dead weight — the SDK does not bridge to OpenTelemetry, so all of our adapter code was solving a problem we didn't actually need solved. Telemetry purge: - Drop the ``traceloop-sdk`` and ``opentelemetry-exporter-otlp-proto-http`` runtime deps. ``uv sync`` uninstalls ~30 transitive packages (the OTEL family, ``traceloop-sdk``, ``protobuf``, ``opentelemetry-exporter-otlp-*``, ``deprecated``, ``wrapt``, ``backoff``, etc.) — about 1000 lines off ``uv.lock``. - Delete ``bootstrap_otel`` and ``JsonlSpanExporter`` from ``telemetry/utils.py``; strip the OTEL pruning helpers, ``parse_traceloop_headers``, ``default_resource_attributes``, ``format_trace_id`` / ``format_span_id`` / ``iso_from_unix_ns``. Keep only the sanitizer + JSONL writer + write-lock registry. - Strip ``Tracer._setup_telemetry``, ``_otel_tracer``, ``_remote_export_enabled``, ``_active_events_file_path``, ``_active_run_metadata``, ``_get_events_write_lock``, ``_set_association_properties``. ``_emit_event`` now generates trace/span ids from ``uuid4`` directly. - Drop the ``traceloop_base_url`` / ``traceloop_api_key`` / ``traceloop_headers`` / ``strix_otel_telemetry`` config knobs. - Rename ``is_otel_enabled`` → ``is_telemetry_enabled`` (the gate now controls JSONL emission only). Native session resume: - ``entry.py`` now constructs an ``agents.memory.SQLiteSession`` keyed by ``scan_id`` and persists conversation history at ``strix_runs/<scan_id>/session.db``. A second call to ``run_strix_scan`` with the same ``scan_id`` resumes from where the prior run left off — no manual state plumbing needed. Tracer.agents fix (TUI agent tree was silently empty): - ``StrixOrchestrationHooks.on_agent_start`` now mirrors bus state into ``tracer.agents`` (id / name / parent_id / status), and ``on_agent_end`` flips the entry to ``completed`` / ``crashed``. The TUI now actually shows the agent tree during scans. Tooling: - Drop ``pylint`` from dev deps; ``ruff`` covers everything we used it for. Strip the ``make lint`` pylint step. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -8,7 +8,7 @@ help:
|
||||
@echo ""
|
||||
@echo "Code Quality:"
|
||||
@echo " format - Format code with ruff"
|
||||
@echo " lint - Lint code with ruff and pylint"
|
||||
@echo " lint - Lint code with ruff"
|
||||
@echo " type-check - Run type checking with mypy and pyright"
|
||||
@echo " security - Run security checks with bandit"
|
||||
@echo " check-all - Run all code quality checks"
|
||||
@@ -36,8 +36,6 @@ format:
|
||||
lint:
|
||||
@echo "🔍 Linting code with ruff..."
|
||||
uv run ruff check . --fix
|
||||
@echo "📝 Running additional linting with pylint..."
|
||||
uv run pylint strix/ --score=no --reports=no
|
||||
@echo "✅ Linting complete!"
|
||||
|
||||
type-check:
|
||||
|
||||
@@ -40,8 +40,6 @@ dependencies = [
|
||||
"textual>=6.0.0",
|
||||
"requests>=2.32.0",
|
||||
"cvss>=3.2",
|
||||
"traceloop-sdk>=0.53.0",
|
||||
"opentelemetry-exporter-otlp-proto-http>=1.40.0",
|
||||
"scrubadub>=2.0.1",
|
||||
]
|
||||
|
||||
@@ -64,7 +62,6 @@ dev = [
|
||||
"mypy>=1.16.0",
|
||||
"ruff>=0.11.13",
|
||||
"pyright>=1.1.401",
|
||||
"pylint>=3.3.7",
|
||||
"bandit>=1.8.3",
|
||||
"pre-commit>=4.2.0",
|
||||
"pyinstaller>=6.17.0; python_version >= '3.12' and python_version < '3.15'",
|
||||
@@ -121,9 +118,7 @@ module = [
|
||||
"pyte.*",
|
||||
"libtmux.*",
|
||||
"cvss.*",
|
||||
"opentelemetry.*",
|
||||
"scrubadub.*",
|
||||
"traceloop.*",
|
||||
"docker.*",
|
||||
]
|
||||
ignore_missing_imports = true
|
||||
|
||||
@@ -42,11 +42,7 @@ class Config:
|
||||
|
||||
# Telemetry
|
||||
strix_telemetry = "1"
|
||||
strix_otel_telemetry = None
|
||||
strix_posthog_telemetry = None
|
||||
traceloop_base_url = None
|
||||
traceloop_api_key = None
|
||||
traceloop_headers = None
|
||||
|
||||
# Config file override (set via --config CLI arg)
|
||||
_config_file_override: Path | None = None
|
||||
|
||||
@@ -21,6 +21,7 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
from agents import Runner
|
||||
from agents.memory import SQLiteSession
|
||||
from agents.tracing import add_trace_processor
|
||||
|
||||
from strix.agents.factory import build_strix_agent, make_child_factory
|
||||
@@ -276,10 +277,22 @@ async def run_strix_scan(
|
||||
reasoning_effort=reasoning_effort,
|
||||
)
|
||||
|
||||
# Native SDK session: persists conversation history to
|
||||
# ``strix_runs/<scan_id>/session.db`` so a second invocation
|
||||
# with the same ``scan_id`` resumes from where we left off.
|
||||
session_db = (
|
||||
(tracer.get_run_dir() / "session.db")
|
||||
if tracer is not None and hasattr(tracer, "get_run_dir")
|
||||
else Path.cwd() / "strix_runs" / scan_id / "session.db"
|
||||
)
|
||||
session_db.parent.mkdir(parents=True, exist_ok=True)
|
||||
session = SQLiteSession(session_id=scan_id, db_path=session_db)
|
||||
|
||||
task_text = _build_root_task(scan_config)
|
||||
return await Runner.run(
|
||||
root_agent,
|
||||
input=task_text,
|
||||
session=session,
|
||||
run_config=run_config,
|
||||
context=context,
|
||||
hooks=StrixOrchestrationHooks(),
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from agents.items import ModelResponse
|
||||
@@ -105,10 +106,16 @@ class StrixOrchestrationHooks(RunHooks[Any]):
|
||||
context: AgentHookContext[Any],
|
||||
agent: Any,
|
||||
) -> None:
|
||||
# The CaidoCapability is bound to the sandbox session, not the
|
||||
# Agent (we use plain ``Agent``, not ``SandboxAgent``). We stash
|
||||
# it in the context dict at scan-bring-up time so the hook can
|
||||
# await its healthcheck before the first LLM call.
|
||||
# Two concerns wired together because they fire at the same
|
||||
# moment in the lifecycle:
|
||||
#
|
||||
# 1. CaidoCapability is bound to the sandbox session, not the
|
||||
# Agent (we use plain ``Agent``, not ``SandboxAgent``), so
|
||||
# the healthcheck task gets stashed in the context dict at
|
||||
# scan bring-up and we await it on first agent start.
|
||||
# 2. The TUI reads ``tracer.agents`` to render the agent tree.
|
||||
# We mirror the bus state into the tracer here so the tree
|
||||
# actually shows live and historical agents.
|
||||
del agent
|
||||
try:
|
||||
ctx = context.context
|
||||
@@ -118,6 +125,24 @@ class StrixOrchestrationHooks(RunHooks[Any]):
|
||||
task = getattr(cap, "_healthcheck_task", None)
|
||||
if task is not None:
|
||||
await task
|
||||
|
||||
tracer = ctx.get("tracer")
|
||||
bus = ctx.get("bus")
|
||||
me = ctx.get("agent_id")
|
||||
if tracer is None or bus is None or me is None:
|
||||
return
|
||||
now = datetime.now(UTC).isoformat()
|
||||
tracer.agents.setdefault(
|
||||
me,
|
||||
{
|
||||
"id": me,
|
||||
"name": bus.names.get(me, me),
|
||||
"parent_id": bus.parent_of.get(me),
|
||||
"status": bus.statuses.get(me, "running"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("on_agent_start failed")
|
||||
|
||||
@@ -136,6 +161,12 @@ class StrixOrchestrationHooks(RunHooks[Any]):
|
||||
if bus is None or me is None:
|
||||
return
|
||||
crashed = (output is None) or not ctx.get("agent_finish_called", False)
|
||||
final_status = "crashed" if crashed else "completed"
|
||||
|
||||
tracer = ctx.get("tracer")
|
||||
if tracer is not None and me in tracer.agents:
|
||||
tracer.agents[me]["status"] = final_status
|
||||
tracer.agents[me]["updated_at"] = datetime.now(UTC).isoformat()
|
||||
parent = bus.parent_of.get(me)
|
||||
if crashed and parent is not None:
|
||||
await bus.send(
|
||||
@@ -150,7 +181,7 @@ class StrixOrchestrationHooks(RunHooks[Any]):
|
||||
"type": "crash",
|
||||
},
|
||||
)
|
||||
await bus.finalize(me, "crashed" if crashed else "completed")
|
||||
await bus.finalize(me, final_status)
|
||||
except Exception:
|
||||
logger.exception("on_agent_end failed")
|
||||
|
||||
|
||||
@@ -9,10 +9,8 @@ def _is_enabled(raw_value: str | None, default: str = "1") -> bool:
|
||||
return value not in _DISABLED_VALUES
|
||||
|
||||
|
||||
def is_otel_enabled() -> bool:
|
||||
explicit = Config.get("strix_otel_telemetry")
|
||||
if explicit is not None:
|
||||
return _is_enabled(explicit)
|
||||
def is_telemetry_enabled() -> bool:
|
||||
"""Master gate — controls JSONL event emission and posthog."""
|
||||
return _is_enabled(Config.get("strix_telemetry"), default="1")
|
||||
|
||||
|
||||
@@ -20,4 +18,4 @@ def is_posthog_enabled() -> bool:
|
||||
explicit = Config.get("strix_posthog_telemetry")
|
||||
if explicit is not None:
|
||||
return _is_enabled(explicit)
|
||||
return _is_enabled(Config.get("strix_telemetry"), default="1")
|
||||
return is_telemetry_enabled()
|
||||
|
||||
+5
-150
@@ -1,42 +1,19 @@
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.trace import SpanContext, SpanKind
|
||||
|
||||
from strix.config import Config
|
||||
from strix.telemetry import posthog
|
||||
from strix.telemetry.flags import is_otel_enabled
|
||||
from strix.telemetry.utils import (
|
||||
TelemetrySanitizer,
|
||||
append_jsonl_record,
|
||||
bootstrap_otel,
|
||||
format_span_id,
|
||||
format_trace_id,
|
||||
get_events_write_lock,
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
from traceloop.sdk import Traceloop
|
||||
except ImportError: # pragma: no cover - exercised when dependency is absent
|
||||
Traceloop = None # type: ignore[assignment,unused-ignore]
|
||||
from strix.telemetry.flags import is_telemetry_enabled
|
||||
from strix.telemetry.utils import TelemetrySanitizer, append_jsonl_record
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_global_tracer: Optional["Tracer"] = None
|
||||
|
||||
_OTEL_BOOTSTRAP_LOCK = threading.Lock()
|
||||
_OTEL_BOOTSTRAPPED = False
|
||||
_OTEL_REMOTE_ENABLED = False
|
||||
|
||||
|
||||
def get_global_tracer() -> Optional["Tracer"]:
|
||||
return _global_tracer
|
||||
@@ -86,16 +63,12 @@ class Tracer:
|
||||
self._next_message_id = 1
|
||||
self._saved_vuln_ids: set[str] = set()
|
||||
self._run_completed_emitted = False
|
||||
self._telemetry_enabled = is_otel_enabled()
|
||||
self._telemetry_enabled = is_telemetry_enabled()
|
||||
self._sanitizer = TelemetrySanitizer()
|
||||
|
||||
self._otel_tracer: Any = None
|
||||
self._remote_export_enabled = False
|
||||
|
||||
self.caido_url: str | None = None
|
||||
self.vulnerability_found_callback: Callable[[dict[str, Any]], None] | None = None
|
||||
|
||||
self._setup_telemetry()
|
||||
self._emit_run_started_event()
|
||||
|
||||
@property
|
||||
@@ -104,65 +77,6 @@ class Tracer:
|
||||
self._events_file_path = self.get_run_dir() / "events.jsonl"
|
||||
return self._events_file_path
|
||||
|
||||
def _active_events_file_path(self) -> Path:
|
||||
active = get_global_tracer()
|
||||
if active and active._events_file_path is not None:
|
||||
return active._events_file_path
|
||||
return self.events_file_path
|
||||
|
||||
def _get_events_write_lock(self, output_path: Path | None = None) -> threading.Lock:
|
||||
path = output_path or self.events_file_path
|
||||
return get_events_write_lock(path)
|
||||
|
||||
def _active_run_metadata(self) -> dict[str, Any]:
|
||||
active = get_global_tracer()
|
||||
if active:
|
||||
return active.run_metadata
|
||||
return self.run_metadata
|
||||
|
||||
def _setup_telemetry(self) -> None:
|
||||
global _OTEL_BOOTSTRAPPED, _OTEL_REMOTE_ENABLED
|
||||
|
||||
if not self._telemetry_enabled:
|
||||
self._otel_tracer = None
|
||||
self._remote_export_enabled = False
|
||||
return
|
||||
|
||||
run_dir = self.get_run_dir()
|
||||
self._events_file_path = run_dir / "events.jsonl"
|
||||
base_url = (Config.get("traceloop_base_url") or "").strip()
|
||||
api_key = (Config.get("traceloop_api_key") or "").strip()
|
||||
headers_raw = Config.get("traceloop_headers") or ""
|
||||
|
||||
(
|
||||
self._otel_tracer,
|
||||
self._remote_export_enabled,
|
||||
_OTEL_BOOTSTRAPPED,
|
||||
_OTEL_REMOTE_ENABLED,
|
||||
) = bootstrap_otel(
|
||||
bootstrapped=_OTEL_BOOTSTRAPPED,
|
||||
remote_enabled_state=_OTEL_REMOTE_ENABLED,
|
||||
bootstrap_lock=_OTEL_BOOTSTRAP_LOCK,
|
||||
traceloop=Traceloop,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
headers_raw=headers_raw,
|
||||
output_path_getter=self._active_events_file_path,
|
||||
run_metadata_getter=self._active_run_metadata,
|
||||
sanitizer=self._sanitize_data,
|
||||
write_lock_getter=self._get_events_write_lock,
|
||||
tracer_name="strix.telemetry.tracer",
|
||||
)
|
||||
|
||||
def _set_association_properties(self, properties: dict[str, Any]) -> None:
|
||||
if Traceloop is None:
|
||||
return
|
||||
sanitized = self._sanitize_data(properties)
|
||||
try:
|
||||
Traceloop.set_association_properties(sanitized)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("Failed to set Traceloop association properties")
|
||||
|
||||
def _sanitize_data(self, data: Any, key_hint: str | None = None) -> Any:
|
||||
return self._sanitizer.sanitize(data, key_hint=key_hint)
|
||||
|
||||
@@ -209,61 +123,12 @@ class Tracer:
|
||||
sanitized_payload = self._sanitize_data(payload) if payload is not None else None
|
||||
sanitized_error = self._sanitize_data(error) if error is not None else None
|
||||
|
||||
trace_id: str | None = None
|
||||
span_id: str | None = None
|
||||
parent_span_id: str | None = None
|
||||
|
||||
current_context = trace.get_current_span().get_span_context()
|
||||
if isinstance(current_context, SpanContext) and current_context.is_valid:
|
||||
parent_span_id = format_span_id(current_context.span_id)
|
||||
|
||||
if self._otel_tracer is not None:
|
||||
try:
|
||||
with self._otel_tracer.start_as_current_span(
|
||||
f"strix.{event_type}",
|
||||
kind=SpanKind.INTERNAL,
|
||||
) as span:
|
||||
span_context = span.get_span_context()
|
||||
trace_id = format_trace_id(span_context.trace_id)
|
||||
span_id = format_span_id(span_context.span_id)
|
||||
|
||||
span.set_attribute("strix.event_type", event_type)
|
||||
span.set_attribute("strix.source", source)
|
||||
span.set_attribute("strix.run_id", self.run_id)
|
||||
span.set_attribute("strix.run_name", self.run_name or "")
|
||||
|
||||
if status:
|
||||
span.set_attribute("strix.status", status)
|
||||
if sanitized_actor is not None:
|
||||
span.set_attribute(
|
||||
"strix.actor",
|
||||
json.dumps(sanitized_actor, ensure_ascii=False),
|
||||
)
|
||||
if sanitized_payload is not None:
|
||||
span.set_attribute(
|
||||
"strix.payload",
|
||||
json.dumps(sanitized_payload, ensure_ascii=False),
|
||||
)
|
||||
if sanitized_error is not None:
|
||||
span.set_attribute(
|
||||
"strix.error",
|
||||
json.dumps(sanitized_error, ensure_ascii=False),
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("Failed to create OTEL span for event type '%s'", event_type)
|
||||
|
||||
if trace_id is None:
|
||||
trace_id = format_trace_id(uuid4().int & ((1 << 128) - 1)) or uuid4().hex
|
||||
if span_id is None:
|
||||
span_id = format_span_id(uuid4().int & ((1 << 64) - 1)) or uuid4().hex[:16]
|
||||
|
||||
record = {
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"event_type": event_type,
|
||||
"run_id": self.run_id,
|
||||
"trace_id": trace_id,
|
||||
"span_id": span_id,
|
||||
"parent_span_id": parent_span_id,
|
||||
"trace_id": uuid4().hex,
|
||||
"span_id": uuid4().hex[:16],
|
||||
"actor": sanitized_actor,
|
||||
"payload": sanitized_payload,
|
||||
"status": status,
|
||||
@@ -282,7 +147,6 @@ class Tracer:
|
||||
self._run_dir = None
|
||||
self._events_file_path = None
|
||||
self._run_completed_emitted = False
|
||||
self._set_association_properties({"run_id": self.run_id, "run_name": self.run_name or ""})
|
||||
self._emit_run_started_event()
|
||||
|
||||
def _emit_run_started_event(self) -> None:
|
||||
@@ -295,7 +159,6 @@ class Tracer:
|
||||
"run_name": self.run_name,
|
||||
"start_time": self.start_time,
|
||||
"local_jsonl_path": str(self.events_file_path),
|
||||
"remote_export_enabled": self._remote_export_enabled,
|
||||
},
|
||||
status="running",
|
||||
include_run_metadata=True,
|
||||
@@ -472,14 +335,6 @@ class Tracer:
|
||||
"max_iterations": config.get("max_iterations", 200),
|
||||
}
|
||||
)
|
||||
self._set_association_properties(
|
||||
{
|
||||
"run_id": self.run_id,
|
||||
"run_name": self.run_name or "",
|
||||
"targets": config.get("targets", []),
|
||||
"max_iterations": config.get("max_iterations", 200),
|
||||
}
|
||||
)
|
||||
self._emit_event(
|
||||
"run.configured",
|
||||
payload={"scan_config": config},
|
||||
|
||||
@@ -2,19 +2,9 @@ import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Callable, Sequence
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
|
||||
from opentelemetry.sdk.trace.export import (
|
||||
BatchSpanProcessor,
|
||||
SimpleSpanProcessor,
|
||||
SpanExporter,
|
||||
SpanExportResult,
|
||||
)
|
||||
from scrubadub import Scrubber
|
||||
from scrubadub.detectors import RegexDetector
|
||||
from scrubadub.filth import Filth
|
||||
@@ -40,18 +30,6 @@ _SENSITIVE_TOKEN_PATTERN = re.compile(
|
||||
_SCRUBADUB_PLACEHOLDER_PATTERN = re.compile(r"\{\{[^}]+\}\}")
|
||||
_EVENTS_FILE_LOCKS_LOCK = threading.Lock()
|
||||
_EVENTS_FILE_LOCKS: dict[str, threading.Lock] = {}
|
||||
_NOISY_OTEL_CONTENT_PREFIXES = (
|
||||
"gen_ai.prompt.",
|
||||
"gen_ai.completion.",
|
||||
"llm.input_messages.",
|
||||
"llm.output_messages.",
|
||||
)
|
||||
_NOISY_OTEL_EXACT_KEYS = {
|
||||
"llm.input",
|
||||
"llm.output",
|
||||
"llm.prompt",
|
||||
"llm.completion",
|
||||
}
|
||||
|
||||
|
||||
class _SecretFilth(Filth): # type: ignore[misc]
|
||||
@@ -103,27 +81,6 @@ class TelemetrySanitizer:
|
||||
return str(data)
|
||||
|
||||
|
||||
def format_trace_id(trace_id: int | None) -> str | None:
|
||||
if trace_id is None or trace_id == 0:
|
||||
return None
|
||||
return f"{trace_id:032x}"
|
||||
|
||||
|
||||
def format_span_id(span_id: int | None) -> str | None:
|
||||
if span_id is None or span_id == 0:
|
||||
return None
|
||||
return f"{span_id:016x}"
|
||||
|
||||
|
||||
def iso_from_unix_ns(unix_ns: int | None) -> str | None:
|
||||
if unix_ns is None:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromtimestamp(unix_ns / 1_000_000_000, tz=UTC).isoformat()
|
||||
except (OSError, OverflowError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def get_events_write_lock(output_path: Path) -> threading.Lock:
|
||||
path_key = str(output_path.resolve(strict=False))
|
||||
with _EVENTS_FILE_LOCKS_LOCK:
|
||||
@@ -143,270 +100,3 @@ def append_jsonl_record(output_path: Path, record: dict[str, Any]) -> None:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with get_events_write_lock(output_path), output_path.open("a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def default_resource_attributes() -> dict[str, str]:
|
||||
return {
|
||||
"service.name": "strix-agent",
|
||||
"service.namespace": "strix",
|
||||
}
|
||||
|
||||
|
||||
def parse_traceloop_headers(raw_headers: str) -> dict[str, str]:
|
||||
headers = raw_headers.strip()
|
||||
if not headers:
|
||||
return {}
|
||||
|
||||
if headers.startswith("{"):
|
||||
try:
|
||||
parsed = json.loads(headers)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("Invalid TRACELOOP_HEADERS JSON, ignoring custom headers")
|
||||
return {}
|
||||
if isinstance(parsed, dict):
|
||||
return {str(key): str(value) for key, value in parsed.items() if value is not None}
|
||||
logger.warning("TRACELOOP_HEADERS JSON must be an object, ignoring custom headers")
|
||||
return {}
|
||||
|
||||
result: dict[str, str] = {}
|
||||
for part in headers.split(","):
|
||||
key, sep, value = part.partition("=")
|
||||
if not sep:
|
||||
continue
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
if key and value:
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def prune_otel_span_attributes(attributes: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Drop high-volume LLM payload attributes to keep JSONL event files compact."""
|
||||
filtered: dict[str, Any] = {}
|
||||
filtered_count = 0
|
||||
|
||||
for key, value in attributes.items():
|
||||
key_str = str(key)
|
||||
if key_str in _NOISY_OTEL_EXACT_KEYS:
|
||||
filtered_count += 1
|
||||
continue
|
||||
|
||||
if key_str.endswith(".content") and key_str.startswith(_NOISY_OTEL_CONTENT_PREFIXES):
|
||||
filtered_count += 1
|
||||
continue
|
||||
|
||||
filtered[key_str] = value
|
||||
|
||||
if filtered_count:
|
||||
filtered["strix.filtered_attributes_count"] = filtered_count
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
class JsonlSpanExporter(SpanExporter): # type: ignore[misc]
|
||||
"""Append OTEL spans to JSONL for local run artifacts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
output_path_getter: Callable[[], Path],
|
||||
run_metadata_getter: Callable[[], dict[str, Any]],
|
||||
sanitizer: Callable[[Any], Any],
|
||||
write_lock_getter: Callable[[Path], threading.Lock],
|
||||
):
|
||||
self._output_path_getter = output_path_getter
|
||||
self._run_metadata_getter = run_metadata_getter
|
||||
self._sanitize = sanitizer
|
||||
self._write_lock_getter = write_lock_getter
|
||||
|
||||
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
||||
records: list[dict[str, Any]] = []
|
||||
for span in spans:
|
||||
attributes = prune_otel_span_attributes(dict(span.attributes or {}))
|
||||
if "strix.event_type" in attributes:
|
||||
# Tracer events are written directly in Tracer._emit_event.
|
||||
continue
|
||||
records.append(self._span_to_record(span, attributes))
|
||||
|
||||
if not records:
|
||||
return SpanExportResult.SUCCESS
|
||||
|
||||
try:
|
||||
output_path = self._output_path_getter()
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self._write_lock_getter(output_path), output_path.open("a", encoding="utf-8") as f:
|
||||
for record in records:
|
||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
except OSError:
|
||||
logger.exception("Failed to write OTEL span records to JSONL")
|
||||
return SpanExportResult.FAILURE
|
||||
|
||||
return SpanExportResult.SUCCESS
|
||||
|
||||
def shutdown(self) -> None:
|
||||
return None
|
||||
|
||||
def force_flush(self, timeout_millis: int = 30_000) -> bool: # noqa: ARG002
|
||||
return True
|
||||
|
||||
def _span_to_record(
|
||||
self,
|
||||
span: ReadableSpan,
|
||||
attributes: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
span_context = span.get_span_context()
|
||||
parent_context = span.parent
|
||||
|
||||
status = None
|
||||
if span.status and span.status.status_code:
|
||||
status = span.status.status_code.name.lower()
|
||||
|
||||
event_type = str(attributes.get("gen_ai.operation.name", span.name))
|
||||
run_metadata = self._run_metadata_getter()
|
||||
run_id_attr = (
|
||||
attributes.get("strix.run_id")
|
||||
or attributes.get("strix_run_id")
|
||||
or run_metadata.get("run_id")
|
||||
or span.resource.attributes.get("strix.run_id")
|
||||
)
|
||||
|
||||
record: dict[str, Any] = {
|
||||
"timestamp": iso_from_unix_ns(span.end_time) or datetime.now(UTC).isoformat(),
|
||||
"event_type": event_type,
|
||||
"run_id": str(run_id_attr or run_metadata.get("run_id") or ""),
|
||||
"trace_id": format_trace_id(span_context.trace_id),
|
||||
"span_id": format_span_id(span_context.span_id),
|
||||
"parent_span_id": format_span_id(parent_context.span_id if parent_context else None),
|
||||
"actor": None,
|
||||
"payload": None,
|
||||
"status": status,
|
||||
"error": None,
|
||||
"source": "otel.span",
|
||||
"span_name": span.name,
|
||||
"span_kind": span.kind.name.lower(),
|
||||
"attributes": self._sanitize(attributes),
|
||||
}
|
||||
|
||||
if span.events:
|
||||
record["otel_events"] = self._sanitize(
|
||||
[
|
||||
{
|
||||
"name": event.name,
|
||||
"timestamp": iso_from_unix_ns(event.timestamp),
|
||||
"attributes": dict(event.attributes or {}),
|
||||
}
|
||||
for event in span.events
|
||||
]
|
||||
)
|
||||
|
||||
return record
|
||||
|
||||
|
||||
def bootstrap_otel(
|
||||
*,
|
||||
bootstrapped: bool,
|
||||
remote_enabled_state: bool,
|
||||
bootstrap_lock: threading.Lock,
|
||||
traceloop: Any,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
headers_raw: str,
|
||||
output_path_getter: Callable[[], Path],
|
||||
run_metadata_getter: Callable[[], dict[str, Any]],
|
||||
sanitizer: Callable[[Any], Any],
|
||||
write_lock_getter: Callable[[Path], threading.Lock],
|
||||
tracer_name: str = "strix.telemetry.tracer",
|
||||
) -> tuple[Any, bool, bool, bool]:
|
||||
with bootstrap_lock:
|
||||
if bootstrapped:
|
||||
return (
|
||||
trace.get_tracer(tracer_name),
|
||||
remote_enabled_state,
|
||||
bootstrapped,
|
||||
remote_enabled_state,
|
||||
)
|
||||
|
||||
local_exporter = JsonlSpanExporter(
|
||||
output_path_getter=output_path_getter,
|
||||
run_metadata_getter=run_metadata_getter,
|
||||
sanitizer=sanitizer,
|
||||
write_lock_getter=write_lock_getter,
|
||||
)
|
||||
local_processor = SimpleSpanProcessor(local_exporter)
|
||||
|
||||
headers = parse_traceloop_headers(headers_raw)
|
||||
remote_enabled = bool(base_url and api_key)
|
||||
otlp_headers = headers
|
||||
if remote_enabled:
|
||||
otlp_headers = {"Authorization": f"Bearer {api_key}"}
|
||||
otlp_headers.update(headers)
|
||||
|
||||
otel_init_ok = False
|
||||
if traceloop:
|
||||
try:
|
||||
from traceloop.sdk.instruments import Instruments
|
||||
|
||||
init_kwargs: dict[str, Any] = {
|
||||
"app_name": "strix-agent",
|
||||
"processor": local_processor,
|
||||
"telemetry_enabled": False,
|
||||
"resource_attributes": default_resource_attributes(),
|
||||
"block_instruments": {
|
||||
Instruments.URLLIB3,
|
||||
Instruments.REQUESTS,
|
||||
},
|
||||
}
|
||||
if remote_enabled:
|
||||
init_kwargs.update(
|
||||
{
|
||||
"api_endpoint": base_url,
|
||||
"api_key": api_key,
|
||||
"headers": headers,
|
||||
}
|
||||
)
|
||||
import io
|
||||
import sys
|
||||
|
||||
_stdout = sys.stdout
|
||||
sys.stdout = io.StringIO()
|
||||
try:
|
||||
traceloop.init(**init_kwargs)
|
||||
finally:
|
||||
sys.stdout = _stdout
|
||||
otel_init_ok = True
|
||||
except Exception:
|
||||
logger.exception("Failed to initialize Traceloop/OpenLLMetry")
|
||||
remote_enabled = False
|
||||
|
||||
if not otel_init_ok:
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
|
||||
provider = TracerProvider(resource=Resource.create(default_resource_attributes()))
|
||||
provider.add_span_processor(local_processor)
|
||||
if remote_enabled:
|
||||
try:
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
|
||||
OTLPSpanExporter,
|
||||
)
|
||||
|
||||
endpoint = base_url.rstrip("/") + "/v1/traces"
|
||||
provider.add_span_processor(
|
||||
BatchSpanProcessor(
|
||||
OTLPSpanExporter(endpoint=endpoint, headers=otlp_headers)
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to configure OTLP HTTP exporter")
|
||||
remote_enabled = False
|
||||
|
||||
try:
|
||||
trace.set_tracer_provider(provider)
|
||||
otel_init_ok = True
|
||||
except Exception:
|
||||
logger.exception("Failed to set OpenTelemetry tracer provider")
|
||||
remote_enabled = False
|
||||
|
||||
otel_tracer = trace.get_tracer(tracer_name)
|
||||
if otel_init_ok:
|
||||
return otel_tracer, remote_enabled, True, remote_enabled
|
||||
|
||||
return otel_tracer, remote_enabled, bootstrapped, remote_enabled_state
|
||||
|
||||
Reference in New Issue
Block a user