refactor: delete orphaned dirs, dead streaming infra, unused session/compressor
Orphaned files/dirs: - ``strix/agents/StrixAgent/`` — empty, only ``__pycache__``. - ``strix/tools/browser/litellm/`` — empty, only ``__pycache__``. - ``strix/strix_runs/`` — runtime output left in the working tree. - ``strix/prompts/`` — single Jinja template that nothing renders. Dead streaming pipeline (was never wired in the SDK migration): - Delete ``strix/interface/streaming_parser.py`` (XML tool-call parser for an output format the SDK doesn't produce). - Strip ``streaming_content`` / ``interrupted_content`` dicts and five unused methods from ``Tracer``. - Strip the streaming-render path + ``interrupted`` branch from TUI. - Trim ``strix/llm/utils.py``: drop ``normalize_tool_format``, ``parse_tool_invocations``, ``format_tool_call``, ``fix_incomplete_tool_call`` and the XML-stripping in ``clean_content``. Keep only the inter-agent-XML scrub. Unwired session compression: - Delete ``strix/llm/strix_session.py`` and ``strix/llm/memory_compressor.py``. ``Runner.run`` was never called with a ``session=``, so the compressor never ran. Drop the matching test file and the ``strix_memory_compressor_timeout`` config knob. Tracer cleanup: - Remove ``log_agent_creation``, ``log_tool_execution_start``, ``update_tool_execution``, ``update_agent_status``, ``get_agent_tools`` — none had production callers. - Rewrite the redaction + correlation tests against ``log_chat_message`` (which still emits events). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
0xallam
@@ -1,12 +1,8 @@
|
||||
"""Jinja-based system-prompt renderer.
|
||||
|
||||
Loads ``strix/agents/prompts/system_prompt.jinja`` (508 lines — the
|
||||
multi-section production prompt with skills, tools, scan modes, etc.)
|
||||
and renders it with the caller's per-run context (scan mode, whitebox,
|
||||
interactive, scope authorization block).
|
||||
|
||||
References:
|
||||
- HARNESS_WIKI.md §4.1 (system prompt assembly)
|
||||
Loads ``strix/agents/prompts/system_prompt.jinja`` and renders it with
|
||||
the caller's per-run context (skills, scan mode, whitebox flag,
|
||||
interactive flag, scope authorization block).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -17,7 +17,6 @@ class Config:
|
||||
ollama_api_base = None
|
||||
strix_reasoning_effort = "high"
|
||||
strix_llm_max_retries = "5"
|
||||
strix_memory_compressor_timeout = "30"
|
||||
llm_timeout = "300"
|
||||
_LLM_CANONICAL_NAMES = (
|
||||
"strix_llm",
|
||||
@@ -28,7 +27,6 @@ class Config:
|
||||
"ollama_api_base",
|
||||
"strix_reasoning_effort",
|
||||
"strix_llm_max_retries",
|
||||
"strix_memory_compressor_timeout",
|
||||
"llm_timeout",
|
||||
)
|
||||
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
import html
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
from strix.llm.utils import normalize_tool_format
|
||||
|
||||
|
||||
_FUNCTION_TAG_PREFIX = "<function="
|
||||
_INVOKE_TAG_PREFIX = "<invoke "
|
||||
|
||||
_FUNC_PATTERN = re.compile(r"<function=([^>]+)>")
|
||||
_FUNC_END_PATTERN = re.compile(r"</function>")
|
||||
_COMPLETE_PARAM_PATTERN = re.compile(r"<parameter=([^>]+)>(.*?)</parameter>", re.DOTALL)
|
||||
_INCOMPLETE_PARAM_PATTERN = re.compile(r"<parameter=([^>]+)>(.*)$", re.DOTALL)
|
||||
|
||||
|
||||
def _get_safe_content(content: str) -> tuple[str, str]:
|
||||
if not content:
|
||||
return "", ""
|
||||
|
||||
last_lt = content.rfind("<")
|
||||
if last_lt == -1:
|
||||
return content, ""
|
||||
|
||||
suffix = content[last_lt:]
|
||||
|
||||
if _FUNCTION_TAG_PREFIX.startswith(suffix) or _INVOKE_TAG_PREFIX.startswith(suffix):
|
||||
return content[:last_lt], suffix
|
||||
|
||||
return content, ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamSegment:
|
||||
type: Literal["text", "tool"]
|
||||
content: str
|
||||
tool_name: str | None = None
|
||||
args: dict[str, str] | None = None
|
||||
is_complete: bool = False
|
||||
|
||||
|
||||
def parse_streaming_content(content: str) -> list[StreamSegment]:
|
||||
if not content:
|
||||
return []
|
||||
|
||||
content = normalize_tool_format(content)
|
||||
|
||||
segments: list[StreamSegment] = []
|
||||
|
||||
func_matches = list(_FUNC_PATTERN.finditer(content))
|
||||
|
||||
if not func_matches:
|
||||
safe_content, _ = _get_safe_content(content)
|
||||
text = safe_content.strip()
|
||||
if text:
|
||||
segments.append(StreamSegment(type="text", content=text))
|
||||
return segments
|
||||
|
||||
first_func_start = func_matches[0].start()
|
||||
if first_func_start > 0:
|
||||
text_before = content[:first_func_start].strip()
|
||||
if text_before:
|
||||
segments.append(StreamSegment(type="text", content=text_before))
|
||||
|
||||
for i, match in enumerate(func_matches):
|
||||
tool_name = match.group(1)
|
||||
func_start = match.end()
|
||||
|
||||
func_end_match = _FUNC_END_PATTERN.search(content, func_start)
|
||||
|
||||
if func_end_match:
|
||||
func_body = content[func_start : func_end_match.start()]
|
||||
is_complete = True
|
||||
end_pos = func_end_match.end()
|
||||
else:
|
||||
if i + 1 < len(func_matches):
|
||||
next_func_start = func_matches[i + 1].start()
|
||||
func_body = content[func_start:next_func_start]
|
||||
else:
|
||||
func_body = content[func_start:]
|
||||
is_complete = False
|
||||
end_pos = len(content)
|
||||
|
||||
args = _parse_streaming_params(func_body)
|
||||
|
||||
segments.append(
|
||||
StreamSegment(
|
||||
type="tool",
|
||||
content=func_body,
|
||||
tool_name=tool_name,
|
||||
args=args,
|
||||
is_complete=is_complete,
|
||||
)
|
||||
)
|
||||
|
||||
if is_complete and i + 1 < len(func_matches):
|
||||
next_start = func_matches[i + 1].start()
|
||||
text_between = content[end_pos:next_start].strip()
|
||||
if text_between:
|
||||
segments.append(StreamSegment(type="text", content=text_between))
|
||||
|
||||
return segments
|
||||
|
||||
|
||||
def _parse_streaming_params(func_body: str) -> dict[str, str]:
|
||||
args: dict[str, str] = {}
|
||||
|
||||
complete_matches = list(_COMPLETE_PARAM_PATTERN.finditer(func_body))
|
||||
complete_end_pos = 0
|
||||
|
||||
for match in complete_matches:
|
||||
param_name = match.group(1)
|
||||
param_value = html.unescape(match.group(2).strip())
|
||||
args[param_name] = param_value
|
||||
complete_end_pos = max(complete_end_pos, match.end())
|
||||
|
||||
remaining = func_body[complete_end_pos:]
|
||||
incomplete_match = _INCOMPLETE_PARAM_PATTERN.search(remaining)
|
||||
if incomplete_match:
|
||||
param_name = incomplete_match.group(1)
|
||||
param_value = html.unescape(incomplete_match.group(2).strip())
|
||||
args[param_name] = param_value
|
||||
|
||||
return args
|
||||
+4
-126
@@ -33,7 +33,6 @@ from textual.widgets.tree import TreeNode
|
||||
|
||||
from strix.config import Config
|
||||
from strix.entry import run_strix_scan
|
||||
from strix.interface.streaming_parser import parse_streaming_content
|
||||
from strix.interface.tool_components.agent_message_renderer import AgentMessageRenderer
|
||||
from strix.interface.tool_components.registry import get_tool_renderer
|
||||
from strix.interface.tool_components.user_message_renderer import UserMessageRenderer
|
||||
@@ -723,9 +722,6 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
self._displayed_agents: set[str] = set()
|
||||
self._displayed_events: list[str] = []
|
||||
|
||||
self._streaming_render_cache: dict[str, tuple[int, Any]] = {}
|
||||
self._last_streaming_len: dict[str, int] = {}
|
||||
|
||||
self._scan_thread: threading.Thread | None = None
|
||||
self._scan_loop: asyncio.AbstractEventLoop | None = None
|
||||
self._scan_stop_event = threading.Event()
|
||||
@@ -979,25 +975,17 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
)
|
||||
|
||||
events = self._gather_agent_events(self.selected_agent_id)
|
||||
streaming = self.tracer.get_streaming_content(self.selected_agent_id)
|
||||
|
||||
if not events and not streaming:
|
||||
if not events:
|
||||
return self._get_chat_placeholder_content(
|
||||
"Starting agent...", "placeholder-no-activity"
|
||||
)
|
||||
|
||||
current_event_ids = [e["id"] for e in events]
|
||||
current_streaming_len = len(streaming) if streaming else 0
|
||||
last_streaming_len = self._last_streaming_len.get(self.selected_agent_id, 0)
|
||||
|
||||
if (
|
||||
current_event_ids == self._displayed_events
|
||||
and current_streaming_len == last_streaming_len
|
||||
):
|
||||
if current_event_ids == self._displayed_events:
|
||||
return None, None
|
||||
|
||||
self._displayed_events = current_event_ids
|
||||
self._last_streaming_len[self.selected_agent_id] = current_streaming_len
|
||||
return self._get_rendered_events_content(events), "chat-content"
|
||||
|
||||
def _update_chat_view(self) -> None:
|
||||
@@ -1106,15 +1094,6 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
renderables.append(Text(""))
|
||||
renderables.append(content)
|
||||
|
||||
if self.selected_agent_id:
|
||||
streaming = self.tracer.get_streaming_content(self.selected_agent_id)
|
||||
if streaming:
|
||||
streaming_text = self._render_streaming_content(streaming)
|
||||
if streaming_text:
|
||||
if renderables:
|
||||
renderables.append(Text(""))
|
||||
renderables.append(streaming_text)
|
||||
|
||||
if not renderables:
|
||||
return Text()
|
||||
|
||||
@@ -1123,85 +1102,6 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
|
||||
return self._merge_renderables(renderables)
|
||||
|
||||
def _render_streaming_content(self, content: str, agent_id: str | None = None) -> Any:
|
||||
cache_key = agent_id or self.selected_agent_id or ""
|
||||
content_len = len(content)
|
||||
|
||||
if cache_key in self._streaming_render_cache:
|
||||
cached_len, cached_output = self._streaming_render_cache[cache_key]
|
||||
if cached_len == content_len:
|
||||
return cached_output
|
||||
|
||||
renderables: list[Any] = []
|
||||
segments = parse_streaming_content(content)
|
||||
|
||||
for segment in segments:
|
||||
if segment.type == "text":
|
||||
text_content = AgentMessageRenderer.render_simple(segment.content)
|
||||
if renderables:
|
||||
renderables.append(Text(""))
|
||||
renderables.append(text_content)
|
||||
|
||||
elif segment.type == "tool":
|
||||
tool_renderable = self._render_streaming_tool(
|
||||
segment.tool_name or "unknown",
|
||||
segment.args or {},
|
||||
segment.is_complete,
|
||||
)
|
||||
if renderables:
|
||||
renderables.append(Text(""))
|
||||
renderables.append(tool_renderable)
|
||||
|
||||
if not renderables:
|
||||
result = Text()
|
||||
elif len(renderables) == 1 and isinstance(renderables[0], Text):
|
||||
result = self._sanitize_text(renderables[0])
|
||||
else:
|
||||
result = self._merge_renderables(renderables)
|
||||
|
||||
self._streaming_render_cache[cache_key] = (content_len, result)
|
||||
return result
|
||||
|
||||
def _render_streaming_tool(
|
||||
self, tool_name: str, args: dict[str, str], is_complete: bool
|
||||
) -> Any:
|
||||
tool_data = {
|
||||
"tool_name": tool_name,
|
||||
"args": args,
|
||||
"status": "completed" if is_complete else "running",
|
||||
"result": None,
|
||||
}
|
||||
|
||||
renderer = get_tool_renderer(tool_name)
|
||||
if renderer:
|
||||
widget = renderer.render(tool_data)
|
||||
return widget.content
|
||||
|
||||
return self._render_default_streaming_tool(tool_name, args, is_complete)
|
||||
|
||||
def _render_default_streaming_tool(
|
||||
self, tool_name: str, args: dict[str, str], is_complete: bool
|
||||
) -> Text:
|
||||
text = Text()
|
||||
|
||||
if is_complete:
|
||||
text.append("✓ ", style="green")
|
||||
else:
|
||||
text.append("● ", style="yellow")
|
||||
|
||||
text.append("Using tool ", style="dim")
|
||||
text.append(tool_name, style="bold blue")
|
||||
|
||||
if args:
|
||||
for key, value in list(args.items())[:3]:
|
||||
text.append("\n ")
|
||||
text.append(key, style="dim")
|
||||
text.append(": ")
|
||||
display_value = value if len(value) <= 100 else value[:97] + "..."
|
||||
text.append(display_value, style="italic" if not is_complete else None)
|
||||
|
||||
return text
|
||||
|
||||
def _get_status_display_content(
|
||||
self, agent_id: str, agent_data: dict[str, Any]
|
||||
) -> tuple[Text | None, Text, bool]:
|
||||
@@ -1442,8 +1342,7 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
if tool_name not in initial_tools:
|
||||
return True
|
||||
|
||||
streaming = self.tracer.get_streaming_content(agent_id)
|
||||
return bool(streaming and streaming.strip())
|
||||
return False
|
||||
|
||||
def _agent_vulnerability_count(self, agent_id: str) -> int:
|
||||
count = 0
|
||||
@@ -1493,8 +1392,6 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
return
|
||||
|
||||
self._displayed_events.clear()
|
||||
self._streaming_render_cache.clear()
|
||||
self._last_streaming_len.clear()
|
||||
|
||||
self.call_later(self._update_chat_view)
|
||||
self._update_agent_status_display()
|
||||
@@ -1710,17 +1607,10 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
if not content:
|
||||
return None
|
||||
|
||||
del metadata
|
||||
if role == "user":
|
||||
return UserMessageRenderer.render_simple(content)
|
||||
|
||||
if metadata.get("interrupted"):
|
||||
streaming_result = self._render_streaming_content(content)
|
||||
interrupted_text = Text()
|
||||
interrupted_text.append("\n")
|
||||
interrupted_text.append("⚠ ", style="yellow")
|
||||
interrupted_text.append("Interrupted by user", style="yellow dim")
|
||||
return self._merge_renderables([streaming_result, interrupted_text])
|
||||
|
||||
return AgentMessageRenderer.render_simple(content)
|
||||
|
||||
def _render_tool_content_simple(self, tool_data: dict[str, Any]) -> Any:
|
||||
@@ -1828,18 +1718,6 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
if not self.selected_agent_id:
|
||||
return
|
||||
|
||||
if self.tracer:
|
||||
streaming_content = self.tracer.get_streaming_content(self.selected_agent_id)
|
||||
if streaming_content and streaming_content.strip():
|
||||
self.tracer.clear_streaming_content(self.selected_agent_id)
|
||||
self.tracer.interrupted_content[self.selected_agent_id] = streaming_content
|
||||
self.tracer.log_chat_message(
|
||||
content=streaming_content,
|
||||
role="assistant",
|
||||
agent_id=self.selected_agent_id,
|
||||
metadata={"interrupted": True},
|
||||
)
|
||||
|
||||
if self.tracer:
|
||||
self.tracer.log_chat_message(
|
||||
content=message,
|
||||
|
||||
@@ -1,219 +0,0 @@
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
|
||||
from strix.config.config import Config, resolve_llm_config
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
MAX_TOTAL_TOKENS = 100_000
|
||||
MIN_RECENT_MESSAGES = 15
|
||||
|
||||
SUMMARY_PROMPT_TEMPLATE = """You are an agent performing context
|
||||
condensation for a security agent. Your job is to compress scan data while preserving
|
||||
ALL operationally critical information for continuing the security assessment.
|
||||
|
||||
CRITICAL ELEMENTS TO PRESERVE:
|
||||
- Discovered vulnerabilities and potential attack vectors
|
||||
- Scan results and tool outputs (compressed but maintaining key findings)
|
||||
- Access credentials, tokens, or authentication details found
|
||||
- System architecture insights and potential weak points
|
||||
- Progress made in the assessment
|
||||
- Failed attempts and dead ends (to avoid duplication)
|
||||
- Any decisions made about the testing approach
|
||||
|
||||
COMPRESSION GUIDELINES:
|
||||
- Preserve exact technical details (URLs, paths, parameters, payloads)
|
||||
- Summarize verbose tool outputs while keeping critical findings
|
||||
- Maintain version numbers, specific technologies identified
|
||||
- Keep exact error messages that might indicate vulnerabilities
|
||||
- Compress repetitive or similar findings into consolidated form
|
||||
|
||||
Remember: Another security agent will use this summary to continue the assessment.
|
||||
They must be able to pick up exactly where you left off without losing any
|
||||
operational advantage or context needed to find vulnerabilities.
|
||||
|
||||
CONVERSATION SEGMENT TO SUMMARIZE:
|
||||
{conversation}
|
||||
|
||||
Provide a technically precise summary that preserves all operational security context while
|
||||
keeping the summary concise and to the point."""
|
||||
|
||||
|
||||
def _count_tokens(text: str, model: str) -> int:
|
||||
try:
|
||||
count = litellm.token_counter(model=model, text=text)
|
||||
return int(count)
|
||||
except Exception:
|
||||
logger.exception("Failed to count tokens")
|
||||
return len(text) // 4 # Rough estimate
|
||||
|
||||
|
||||
def _get_message_tokens(msg: dict[str, Any], model: str) -> int:
|
||||
content = msg.get("content", "")
|
||||
if isinstance(content, str):
|
||||
return _count_tokens(content, model)
|
||||
if isinstance(content, list):
|
||||
return sum(
|
||||
_count_tokens(item.get("text", ""), model)
|
||||
for item in content
|
||||
if isinstance(item, dict) and item.get("type") == "text"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def _extract_message_text(msg: dict[str, Any]) -> str:
|
||||
content = msg.get("content", "")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
|
||||
if isinstance(content, list):
|
||||
parts = []
|
||||
for item in content:
|
||||
if isinstance(item, dict):
|
||||
if item.get("type") == "text":
|
||||
parts.append(item.get("text", ""))
|
||||
elif item.get("type") == "image_url":
|
||||
parts.append("[IMAGE]")
|
||||
return " ".join(parts)
|
||||
|
||||
return str(content)
|
||||
|
||||
|
||||
def _summarize_messages(
|
||||
messages: list[dict[str, Any]],
|
||||
model: str,
|
||||
timeout: int = 30,
|
||||
) -> dict[str, Any]:
|
||||
if not messages:
|
||||
empty_summary = "<context_summary message_count='0'>{text}</context_summary>"
|
||||
return {
|
||||
"role": "user",
|
||||
"content": empty_summary.format(text="No messages to summarize"),
|
||||
}
|
||||
|
||||
formatted = []
|
||||
for msg in messages:
|
||||
role = msg.get("role", "unknown")
|
||||
text = _extract_message_text(msg)
|
||||
formatted.append(f"{role}: {text}")
|
||||
|
||||
conversation = "\n".join(formatted)
|
||||
prompt = SUMMARY_PROMPT_TEMPLATE.format(conversation=conversation)
|
||||
|
||||
_, api_key, api_base = resolve_llm_config()
|
||||
|
||||
try:
|
||||
completion_args: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"timeout": timeout,
|
||||
}
|
||||
if api_key:
|
||||
completion_args["api_key"] = api_key
|
||||
if api_base:
|
||||
completion_args["api_base"] = api_base
|
||||
|
||||
response = litellm.completion(**completion_args)
|
||||
summary = response.choices[0].message.content or ""
|
||||
if not summary.strip():
|
||||
return messages[0]
|
||||
summary_msg = "<context_summary message_count='{count}'>{text}</context_summary>"
|
||||
return {
|
||||
"role": "user",
|
||||
"content": summary_msg.format(count=len(messages), text=summary),
|
||||
}
|
||||
except Exception:
|
||||
logger.exception("Failed to summarize messages")
|
||||
return messages[0]
|
||||
|
||||
|
||||
def _handle_images(messages: list[dict[str, Any]], max_images: int) -> None:
|
||||
image_count = 0
|
||||
for msg in reversed(messages):
|
||||
content = msg.get("content", [])
|
||||
if isinstance(content, list):
|
||||
for item in content:
|
||||
if isinstance(item, dict) and item.get("type") == "image_url":
|
||||
if image_count >= max_images:
|
||||
item.update(
|
||||
{
|
||||
"type": "text",
|
||||
"text": "[Previously attached image removed to preserve context]",
|
||||
}
|
||||
)
|
||||
else:
|
||||
image_count += 1
|
||||
|
||||
|
||||
class MemoryCompressor:
|
||||
def __init__(
|
||||
self,
|
||||
max_images: int = 3,
|
||||
model_name: str | None = None,
|
||||
timeout: int | None = None,
|
||||
):
|
||||
self.max_images = max_images
|
||||
self.model_name = model_name or Config.get("strix_llm")
|
||||
self.timeout = timeout or int(Config.get("strix_memory_compressor_timeout") or "120")
|
||||
|
||||
if not self.model_name:
|
||||
raise ValueError("STRIX_LLM environment variable must be set and not empty")
|
||||
|
||||
def compress_history(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Compress conversation history to stay within token limits.
|
||||
|
||||
Strategy:
|
||||
1. Handle image limits first
|
||||
2. Keep all system messages
|
||||
3. Keep minimum recent messages
|
||||
4. Summarize older messages when total tokens exceed limit
|
||||
|
||||
The compression preserves:
|
||||
- All system messages unchanged
|
||||
- Most recent messages intact
|
||||
- Critical security context in summaries
|
||||
- Recent images for visual context
|
||||
- Technical details and findings
|
||||
"""
|
||||
if not messages:
|
||||
return messages
|
||||
|
||||
_handle_images(messages, self.max_images)
|
||||
|
||||
system_msgs = []
|
||||
regular_msgs = []
|
||||
for msg in messages:
|
||||
if msg.get("role") == "system":
|
||||
system_msgs.append(msg)
|
||||
else:
|
||||
regular_msgs.append(msg)
|
||||
|
||||
recent_msgs = regular_msgs[-MIN_RECENT_MESSAGES:]
|
||||
old_msgs = regular_msgs[:-MIN_RECENT_MESSAGES]
|
||||
|
||||
# Type assertion since we ensure model_name is not None in __init__
|
||||
model_name: str = self.model_name # type: ignore[assignment]
|
||||
|
||||
total_tokens = sum(
|
||||
_get_message_tokens(msg, model_name) for msg in system_msgs + regular_msgs
|
||||
)
|
||||
|
||||
if total_tokens <= MAX_TOTAL_TOKENS * 0.9:
|
||||
return messages
|
||||
|
||||
compressed = []
|
||||
chunk_size = 10
|
||||
for i in range(0, len(old_msgs), chunk_size):
|
||||
chunk = old_msgs[i : i + chunk_size]
|
||||
summary = _summarize_messages(chunk, model_name, self.timeout)
|
||||
if summary:
|
||||
compressed.append(summary)
|
||||
|
||||
return system_msgs + compressed + recent_msgs
|
||||
@@ -1,99 +0,0 @@
|
||||
"""``StrixSession`` — Session wrapper that runs the MemoryCompressor.
|
||||
|
||||
Delegates storage to any underlying session implementation (in-memory,
|
||||
SQLite, Redis, …) and intercepts ``get_items`` so the
|
||||
``MemoryCompressor`` runs before the model sees the history.
|
||||
|
||||
Wrapping (rather than reimplementing) keeps the pentest-tuned
|
||||
summarization prompt and 90K-token budget intact. ``get_items`` is
|
||||
also the last hook before ``call_model_input_filter``, so compressing
|
||||
here means the filter sees a compressed history too.
|
||||
|
||||
If compression raises, we fall back to the uncompressed history and
|
||||
flip a flag so future attempts skip — a permanently broken compressor
|
||||
mustn't infinite-loop the agent into context starvation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from agents.memory.session import SessionABC
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from agents.items import TResponseInputItem
|
||||
|
||||
from strix.llm.memory_compressor import MemoryCompressor
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StrixSession(SessionABC):
|
||||
"""Wraps an underlying ``SessionABC`` with Strix's memory compressor.
|
||||
|
||||
The wrapped session owns persistence; ``StrixSession`` only intercepts
|
||||
``get_items`` to run compression. Writes (``add_items``, ``pop_item``,
|
||||
``clear_session``) pass through verbatim.
|
||||
|
||||
On compressor failure, the call returns the uncompressed history and
|
||||
a per-instance flag is set so subsequent ``get_items`` calls skip the
|
||||
compressor entirely. This avoids an infinite "compress → fail → grow"
|
||||
loop when the compressor LLM is itself unavailable.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
underlying: SessionABC,
|
||||
compressor: MemoryCompressor,
|
||||
) -> None:
|
||||
self._underlying = underlying
|
||||
self._compressor = compressor
|
||||
self._compression_disabled = False
|
||||
# ``SessionABC.session_id`` is a plain ``str`` field; pass through.
|
||||
self.session_id: str = getattr(underlying, "session_id", "strix-session")
|
||||
self.session_settings = getattr(underlying, "session_settings", None)
|
||||
|
||||
@property
|
||||
def compression_disabled(self) -> bool:
|
||||
"""True after the compressor has failed at least once on this session."""
|
||||
return self._compression_disabled
|
||||
|
||||
async def get_items(
|
||||
self,
|
||||
limit: int | None = None,
|
||||
) -> list[TResponseInputItem]:
|
||||
"""Read items from underlying storage and (optionally) compress.
|
||||
|
||||
On any compressor exception, log and return the uncompressed list.
|
||||
Set ``_compression_disabled`` so the next call short-circuits.
|
||||
"""
|
||||
items = await self._underlying.get_items(limit=limit)
|
||||
if self._compression_disabled or not items:
|
||||
return items
|
||||
try:
|
||||
# Compressor expects ``list[dict[str, Any]]``; SDK's
|
||||
# ``TResponseInputItem`` is a TypedDict union — structurally
|
||||
# compatible. Compressor mutates content but preserves shape.
|
||||
compressed = self._compressor.compress_history(
|
||||
cast("list[dict[str, Any]]", items),
|
||||
)
|
||||
return cast("list[TResponseInputItem]", compressed)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"MemoryCompressor failed; returning uncompressed history. "
|
||||
"Compression disabled for this session for the rest of the run.",
|
||||
)
|
||||
self._compression_disabled = True
|
||||
return items
|
||||
|
||||
async def add_items(self, items: list[TResponseInputItem]) -> None:
|
||||
await self._underlying.add_items(items)
|
||||
|
||||
async def pop_item(self) -> TResponseInputItem | None:
|
||||
return await self._underlying.pop_item()
|
||||
|
||||
async def clear_session(self) -> None:
|
||||
await self._underlying.clear_session()
|
||||
+15
-116
@@ -1,127 +1,26 @@
|
||||
"""Streaming + tool-format helpers used by the TUI's render pipeline.
|
||||
"""Helpers for the TUI message renderer.
|
||||
|
||||
The model can emit tool calls in a few XML shapes (``<function=…>``,
|
||||
``<invoke name="…">``, optionally wrapped in ``<function_calls>``); the
|
||||
streaming parser normalizes them into one canonical form so the
|
||||
renderer doesn't have to branch.
|
||||
|
||||
These helpers are pure string manipulation — no model client, no SDK
|
||||
dependency. They live here because the streaming parser and the
|
||||
agent-message renderer both consume them.
|
||||
The model occasionally echoes inter-agent XML envelopes in plain text
|
||||
despite the system prompt's "don't echo" rule, so :func:`clean_content`
|
||||
strips them defensively before display.
|
||||
"""
|
||||
|
||||
import html
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
|
||||
_INVOKE_OPEN = re.compile(r'<invoke\s+name=["\']([^"\']+)["\']>')
|
||||
_PARAM_NAME_ATTR = re.compile(r'<parameter\s+name=["\']([^"\']+)["\']>')
|
||||
_FUNCTION_CALLS_TAG = re.compile(r"</?function_calls>")
|
||||
_STRIP_TAG_QUOTES = re.compile(r"<(function|parameter)\s*=\s*([^>]*?)>")
|
||||
|
||||
|
||||
def normalize_tool_format(content: str) -> str:
|
||||
"""Convert alternative tool-call XML formats to the expected one.
|
||||
|
||||
Handles:
|
||||
<function_calls>...</function_calls> → stripped
|
||||
<invoke name="X"> → <function=X>
|
||||
<parameter name="X"> → <parameter=X>
|
||||
</invoke> → </function>
|
||||
<function="X"> → <function=X>
|
||||
<parameter="X"> → <parameter=X>
|
||||
"""
|
||||
if "<invoke" in content or "<function_calls" in content:
|
||||
content = _FUNCTION_CALLS_TAG.sub("", content)
|
||||
content = _INVOKE_OPEN.sub(r"<function=\1>", content)
|
||||
content = _PARAM_NAME_ATTR.sub(r"<parameter=\1>", content)
|
||||
content = content.replace("</invoke>", "</function>")
|
||||
|
||||
return _STRIP_TAG_QUOTES.sub(
|
||||
lambda m: f"<{m.group(1)}={m.group(2).strip().strip(chr(34) + chr(39))}>",
|
||||
content,
|
||||
)
|
||||
|
||||
|
||||
def parse_tool_invocations(content: str) -> list[dict[str, Any]] | None:
|
||||
content = normalize_tool_format(content)
|
||||
content = fix_incomplete_tool_call(content)
|
||||
|
||||
tool_invocations: list[dict[str, Any]] = []
|
||||
|
||||
fn_regex_pattern = r"<function=([^>]+)>\n?(.*?)</function>"
|
||||
fn_param_regex_pattern = r"<parameter=([^>]+)>(.*?)</parameter>"
|
||||
|
||||
fn_matches = re.finditer(fn_regex_pattern, content, re.DOTALL)
|
||||
|
||||
for fn_match in fn_matches:
|
||||
fn_name = fn_match.group(1)
|
||||
fn_body = fn_match.group(2)
|
||||
|
||||
param_matches = re.finditer(fn_param_regex_pattern, fn_body, re.DOTALL)
|
||||
|
||||
args = {}
|
||||
for param_match in param_matches:
|
||||
param_name = param_match.group(1)
|
||||
param_value = param_match.group(2).strip()
|
||||
|
||||
param_value = html.unescape(param_value)
|
||||
args[param_name] = param_value
|
||||
|
||||
tool_invocations.append({"toolName": fn_name, "args": args})
|
||||
|
||||
return tool_invocations if tool_invocations else None
|
||||
|
||||
|
||||
def fix_incomplete_tool_call(content: str) -> str:
|
||||
"""Fix incomplete tool calls by adding missing closing tag.
|
||||
|
||||
Handles both ``<function=…>`` and ``<invoke name="…">`` formats.
|
||||
"""
|
||||
has_open = "<function=" in content or "<invoke " in content
|
||||
count_open = content.count("<function=") + content.count("<invoke ")
|
||||
has_close = "</function>" in content or "</invoke>" in content
|
||||
if has_open and count_open == 1 and not has_close:
|
||||
content = content.rstrip()
|
||||
content = content + "function>" if content.endswith("</") else content + "\n</function>"
|
||||
return content
|
||||
|
||||
|
||||
def format_tool_call(tool_name: str, args: dict[str, Any]) -> str:
|
||||
xml_parts = [f"<function={tool_name}>"]
|
||||
|
||||
for key, value in args.items():
|
||||
xml_parts.append(f"<parameter={key}>{value}</parameter>")
|
||||
|
||||
xml_parts.append("</function>")
|
||||
|
||||
return "\n".join(xml_parts)
|
||||
_HIDDEN_XML_PATTERNS = [
|
||||
re.compile(r"<inter_agent_message>.*?</inter_agent_message>", re.DOTALL | re.IGNORECASE),
|
||||
re.compile(
|
||||
r"<agent_completion_report>.*?</agent_completion_report>",
|
||||
re.DOTALL | re.IGNORECASE,
|
||||
),
|
||||
]
|
||||
_BLANK_LINE_RUNS = re.compile(r"\n\s*\n")
|
||||
|
||||
|
||||
def clean_content(content: str) -> str:
|
||||
if not content:
|
||||
return ""
|
||||
|
||||
content = normalize_tool_format(content)
|
||||
content = fix_incomplete_tool_call(content)
|
||||
|
||||
tool_pattern = r"<function=[^>]+>.*?</function>"
|
||||
cleaned = re.sub(tool_pattern, "", content, flags=re.DOTALL)
|
||||
|
||||
incomplete_tool_pattern = r"<function=[^>]+>.*$"
|
||||
cleaned = re.sub(incomplete_tool_pattern, "", cleaned, flags=re.DOTALL)
|
||||
|
||||
partial_tag_pattern = r"<f(?:u(?:n(?:c(?:t(?:i(?:o(?:n(?:=(?:[^>]*)?)?)?)?)?)?)?)?)?$"
|
||||
cleaned = re.sub(partial_tag_pattern, "", cleaned)
|
||||
|
||||
hidden_xml_patterns = [
|
||||
r"<inter_agent_message>.*?</inter_agent_message>",
|
||||
r"<agent_completion_report>.*?</agent_completion_report>",
|
||||
]
|
||||
for pattern in hidden_xml_patterns:
|
||||
cleaned = re.sub(pattern, "", cleaned, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
cleaned = re.sub(r"\n\s*\n", "\n\n", cleaned)
|
||||
|
||||
return cleaned.strip()
|
||||
for pattern in _HIDDEN_XML_PATTERNS:
|
||||
content = pattern.sub("", content)
|
||||
return _BLANK_LINE_RUNS.sub("\n\n", content).strip()
|
||||
|
||||
@@ -1,266 +0,0 @@
|
||||
<nosql_injection_guide>
|
||||
<title>NoSQL INJECTION</title>
|
||||
|
||||
<critical>NoSQL injection exploits vulnerabilities in non-relational databases (MongoDB, CouchDB, Redis, Cassandra, etc.) where user input manipulates query logic, operators, or JavaScript execution contexts. Unlike SQL injection, NoSQL attacks often target JSON/BSON structures, query operators, and server-side JavaScript evaluation. Treat every user-controlled input destined for NoSQL queries as untrusted.</critical>
|
||||
|
||||
<scope>
|
||||
- Document stores: MongoDB, CouchDB, Couchbase, Amazon DocumentDB
|
||||
- Key-value stores: Redis, DynamoDB, Memcached
|
||||
- Wide-column stores: Cassandra, HBase, ScyllaDB
|
||||
- Graph databases: Neo4j, ArangoDB, Amazon Neptune
|
||||
- Integration paths: ODMs (Mongoose, Morphia), REST APIs, GraphQL resolvers, serverless functions
|
||||
</scope>
|
||||
|
||||
<methodology>
|
||||
1. Identify NoSQL database type from error messages, response patterns, headers, or technology fingerprinting.
|
||||
2. Determine input format: JSON body, query string, URL path, headers; note how input is parsed and merged into queries.
|
||||
3. Test operator injection: inject MongoDB operators ($ne, $gt, $regex, $where) or database-specific syntax to alter query logic.
|
||||
4. Establish extraction channel: boolean-based response diffs, timing via $where/JavaScript, regex-based character extraction, or error messages.
|
||||
5. Pivot to authentication bypass, data exfiltration, or JavaScript execution depending on database capabilities.
|
||||
</methodology>
|
||||
|
||||
<injection_surfaces>
|
||||
- JSON body parameters: direct object/operator injection via nested objects or arrays
|
||||
- Query string: array notation (?username[$ne]=) or JSON-encoded values
|
||||
- URL path segments: document IDs, collection names in RESTful APIs
|
||||
- Headers/cookies: session data parsed as JSON, JWT claims used in queries
|
||||
- GraphQL variables: unvalidated input passed directly to resolvers
|
||||
- Aggregation pipelines: $match, $lookup, $group stages with user-controlled fields
|
||||
</injection_surfaces>
|
||||
|
||||
<detection_channels>
|
||||
- Operator-based: inject query operators to modify predicate logic; test $gt/$gte/$lt/$lte/$ne/$eq/$in/$nin and $or/$and/$nor
|
||||
- Boolean-based: compare true/false predicates; diff status codes, body length, specific content; use $regex for extraction
|
||||
- Timing-based: use $where sleep payloads when server-side JavaScript is enabled; use heavy regex operations for ReDoS-style delays
|
||||
- Error-based: provoke type errors, invalid operator errors, or JavaScript runtime exceptions; inspect verbose errors
|
||||
</detection_channels>
|
||||
|
||||
<dbms_primitives>
|
||||
<mongodb>
|
||||
- Operators for injection: $ne, $gt, $lt, $gte, $lte, $in, $nin, $or, $and, $regex, $where, $exists, $type
|
||||
- JavaScript execution: $where clause accepts JavaScript; $function in aggregations (MongoDB 4.4+)
|
||||
- Version/info: db.version(), db.serverStatus(), db.hostInfo()
|
||||
- Authentication bypass: {% raw %}{"username": {"$ne": ""}, "password": {"$ne": ""}}{% endraw %}
|
||||
- Regex extraction: {% raw %}{"password": {"$regex": "^a.*"}}{% endraw %} iterate to extract full value
|
||||
- $where JavaScript: {% raw %}{"$where": "this.username == 'admin' && this.password.match(/^a/)"}{% endraw %}
|
||||
- Aggregation injection: $lookup to access other collections, $out to write results
|
||||
</mongodb>
|
||||
|
||||
<couchdb>
|
||||
- View injection via map/reduce functions (JavaScript execution)
|
||||
- Mango queries: operator injection similar to MongoDB ($eq, $ne, $gt, $regex, etc.)
|
||||
- _all_docs, _find endpoints with selector manipulation
|
||||
- Design document manipulation for persistent code execution
|
||||
</couchdb>
|
||||
|
||||
<redis>
|
||||
- Command injection via protocol manipulation in poorly sanitized inputs
|
||||
- Lua script injection: EVAL command with user-controlled scripts
|
||||
- Key enumeration: KEYS *, SCAN with patterns
|
||||
- Data exfiltration: GET, HGETALL, LRANGE, SMEMBERS
|
||||
- Config manipulation: CONFIG SET to modify runtime behavior
|
||||
- File write via RDB: CONFIG SET dir/dbfilename + SAVE (requires privileges)
|
||||
</redis>
|
||||
|
||||
<cassandra>
|
||||
- CQL injection: similar to SQL, string concatenation in WHERE clauses
|
||||
- ALLOW FILTERING abuse for unauthorized data access
|
||||
- UDF (User Defined Functions) if enabled: Java/JavaScript code execution
|
||||
</cassandra>
|
||||
|
||||
<neo4j>
|
||||
- Cypher injection: MATCH, WHERE, RETURN clause manipulation
|
||||
- APOC procedures: apoc.load.json, apoc.cypher.run for extended capabilities
|
||||
- Label/relationship injection to access unauthorized graph nodes
|
||||
</neo4j>
|
||||
</dbms_primitives>
|
||||
|
||||
<authentication_bypass>
|
||||
<mongodb_operators>
|
||||
- Basic bypass: {% raw %}{"username": "admin", "password": {"$ne": ""}}{% endraw %}
|
||||
- Always true: {% raw %}{"username": {"$gt": ""}, "password": {"$gt": ""}}{% endraw %}
|
||||
- Regex wildcard: {% raw %}{"username": "admin", "password": {"$regex": ".*"}}{% endraw %}
|
||||
- $or injection: {% raw %}{"$or": [{"username": "admin"}, {"admin": true}], "password": {"$ne": ""}}{% endraw %}
|
||||
- Type coercion: {% raw %}{"username": "admin", "password": {"$type": 2}}{% endraw %} (type 2 = string)
|
||||
</mongodb_operators>
|
||||
|
||||
<query_string_injection>
|
||||
- Array notation: ?username=admin&password[$ne]=wrongpass
|
||||
- Nested operators: ?user[username]=admin&user[password][$gt]=
|
||||
- URL-encoded JSON: ?filter=%7B%22username%22%3A%7B%22%24ne%22%3Anull%7D%7D
|
||||
</query_string_injection>
|
||||
</authentication_bypass>
|
||||
|
||||
<data_extraction>
|
||||
<regex_extraction>
|
||||
- Character-by-character: iterate {% raw %}{"field": {"$regex": "^<known>X"}}{% endraw %} for each position
|
||||
- Binary search: use character ranges [a-m] vs [n-z] to reduce requests
|
||||
- Case sensitivity: use $options: "i" for case-insensitive matching
|
||||
- Special chars: escape regex metacharacters (. * + ? ^ $ { } [ ] \ | ( ))
|
||||
</regex_extraction>
|
||||
|
||||
<boolean_extraction>
|
||||
- Use $regex or $where to create true/false conditions
|
||||
- Diff response length, status code, specific strings, or timing
|
||||
- Extract field names via $exists: {% raw %}{"unknownField": {"$exists": true}}{% endraw %}
|
||||
</boolean_extraction>
|
||||
|
||||
<javascript_extraction>
|
||||
- $where with conditional: {% raw %}{"$where": "if(this.password[0]=='a'){sleep(5000)}"}{% endraw %}
|
||||
- Access Object.keys(): {% raw %}{"$where": "Object.keys(this)[0][0]=='u'"}{% endraw %} to enumerate fields
|
||||
- String operations: substring, charAt for positional extraction
|
||||
</javascript_extraction>
|
||||
|
||||
<aggregation_based>
|
||||
- $lookup to join with other collections and leak data
|
||||
- $match with injected operators
|
||||
- $project to select fields, $group to aggregate sensitive data
|
||||
</aggregation_based>
|
||||
</data_extraction>
|
||||
|
||||
<javascript_injection>
|
||||
<where_clause>
|
||||
- MongoDB $where executes JavaScript on the server; requires server-side JavaScript enabled (often disabled via --noscripting/security.javascriptEnabled)
|
||||
- Basic: {% raw %}{"$where": "1==1"}{% endraw %} or {% raw %}{"$where": "true"}{% endraw %}
|
||||
- Sleep for timing: {% raw %}{"$where": "sleep(5000) || true"}{% endraw %}
|
||||
- Data access: {% raw %}{"$where": "this.password.length > 5"}{% endraw %}
|
||||
- External calls (if allowed): {% raw %}{"$where": "this.constructor.constructor('return fetch(...)')()"}{% endraw %}
|
||||
</where_clause>
|
||||
|
||||
<function_operator>
|
||||
- MongoDB 4.4+ $function in aggregation: {% raw %}{"$function": {"body": "function(){...}", "args": [], "lang": "js"}}{% endraw %}
|
||||
- Server-side JavaScript must be enabled (not disabled via --noscripting)
|
||||
</function_operator>
|
||||
|
||||
<mapreduce>
|
||||
- Inject into map/reduce functions if user input reaches these contexts
|
||||
- CouchDB views: JavaScript in map functions
|
||||
</mapreduce>
|
||||
</javascript_injection>
|
||||
|
||||
<odm_and_framework_issues>
|
||||
<mongoose>
|
||||
- Dangerous patterns: find(req.body), findOne(req.query) without sanitization
|
||||
- $where passthrough: user input reaching $where conditions
|
||||
- Population/reference injection: manipulating $lookup-like operations
|
||||
- Schema bypass: __proto__, constructor pollution via JSON parsing
|
||||
</mongoose>
|
||||
|
||||
<morphia_java>
|
||||
- String concatenation in filters instead of parameterized queries
|
||||
- Criteria API misuse with raw strings
|
||||
</morphia_java>
|
||||
|
||||
<pymongo>
|
||||
- eval() usage with user input (deprecated but still dangerous)
|
||||
- find() with unsanitized dictionaries from JSON input
|
||||
- Codec manipulation affecting serialization
|
||||
</pymongo>
|
||||
</odm_and_framework_issues>
|
||||
|
||||
<waf_and_filter_bypasses>
|
||||
<encoding_tricks>
|
||||
- URL encoding: %24ne instead of $ne
|
||||
- Double encoding: %2524ne
|
||||
- Unicode normalization: using different Unicode representations
|
||||
- JSON unicode escapes: \u0024ne for $ne
|
||||
</encoding_tricks>
|
||||
|
||||
<operator_alternatives>
|
||||
- $not instead of $ne: {% raw %}{"field": {"$not": {"$eq": "value"}}}{% endraw %}
|
||||
- $nin instead of $ne: {% raw %}{"field": {"$nin": ["wrong"]}}{% endraw %}
|
||||
- $expr with $eq/$ne in aggregation context
|
||||
</operator_alternatives>
|
||||
|
||||
<structure_manipulation>
|
||||
- Nested objects vs flat: {% raw %}{"a.b": "c"}{% endraw %} vs {% raw %}{"a": {"b": "c"}}{% endraw %}
|
||||
- Array injection: ["$or", ...] in systems parsing arrays as operators
|
||||
- Prototype pollution: __proto__, constructor.prototype in JSON
|
||||
</structure_manipulation>
|
||||
|
||||
<comment_injection>
|
||||
- MongoDB shell: // or /* */ in JavaScript contexts
|
||||
- Newline injection in string concatenation scenarios
|
||||
</comment_injection>
|
||||
</waf_and_filter_bypasses>
|
||||
|
||||
<blind_extraction>
|
||||
<binary_search>
|
||||
- Use $regex with character ranges: ^[a-m] vs ^[n-z]
|
||||
- Reduce character space: alphanumeric, then specific ranges
|
||||
- Position tracking: ^known_prefix[a-m] for next character
|
||||
</binary_search>
|
||||
|
||||
<timing_oracle>
|
||||
- $where with conditional sleep: {% raw %}if(condition){sleep(N)}{% endraw %}
|
||||
- ReDoS via pathological regex: ((a+)+)$ with long input
|
||||
- Heavy operations: sorting large datasets conditionally
|
||||
</timing_oracle>
|
||||
|
||||
<response_differential>
|
||||
- Track: status codes (200/401/403/500), body length, specific strings, JSON structure
|
||||
- Normalize responses (hash/length) to reduce noise
|
||||
- Account for caching and rate limiting affecting responses
|
||||
</response_differential>
|
||||
</blind_extraction>
|
||||
|
||||
<server_side_injection>
|
||||
<ssjs_mongodb>
|
||||
- Server-Side JavaScript (SSJS) when $where, $function, mapReduce are exposed
|
||||
- Potential for: DoS (infinite loops), data access, limited RCE depending on config
|
||||
- Check: {% raw %}db.adminCommand({getParameter: 1, javascriptEnabled: 1}){% endraw %}
|
||||
</ssjs_mongodb>
|
||||
|
||||
<dos_attacks>
|
||||
- ReDoS: {% raw %}{"field": {"$regex": "^(a+)+$"}}{% endraw %} against long strings
|
||||
- Resource exhaustion: large $in arrays, complex aggregations
|
||||
- Infinite loops in $where if SSJS is enabled without timeouts
|
||||
</dos_attacks>
|
||||
</server_side_injection>
|
||||
|
||||
<graphql_nosql>
|
||||
- Variables passed directly to MongoDB queries: {% raw %}query { user(filter: $input) }{% endraw %}
|
||||
- Operator injection via GraphQL variables: {% raw %}{"filter": {"password": {"$ne": ""}}}{% endraw %}
|
||||
- Batching attacks: multiple queries to enumerate data
|
||||
- Introspection combined with injection for schema-aware attacks
|
||||
</graphql_nosql>
|
||||
|
||||
<validation>
|
||||
1. Demonstrate operator injection alters query behavior (auth bypass, extra data returned).
|
||||
2. Show boolean/timing/error oracle confirms control over query predicates.
|
||||
3. Extract verifiable data: version info, field names, partial sensitive values.
|
||||
4. Provide minimal reproducible requests with clear injection points.
|
||||
5. Document database type and version; defenses vary significantly across NoSQL systems.
|
||||
</validation>
|
||||
|
||||
<false_positives>
|
||||
- Strong typing/schema validation rejecting operator objects
|
||||
- ODM sanitization stripping $ prefixes from keys
|
||||
- Parameterized queries where operators cannot be injected
|
||||
- WAF blocking all $ operators (verify with encoding bypasses first)
|
||||
- Application logic unrelated to database predicates causing response variations
|
||||
</false_positives>
|
||||
|
||||
<impact>
|
||||
- Authentication and authorization bypass via manipulated query predicates
|
||||
- Mass data exfiltration through regex extraction or aggregation manipulation
|
||||
- Server-side JavaScript execution leading to DoS or limited RCE
|
||||
- Privilege escalation by modifying user roles/permissions in database
|
||||
- Denial of service via ReDoS or resource-intensive queries
|
||||
</impact>
|
||||
|
||||
<pro_tips>
|
||||
1. Start with $ne and $gt operators—they're most commonly injectable and easy to detect.
|
||||
2. Use boolean oracles first; timing channels are noisier and slower.
|
||||
3. For MongoDB, always test both JSON body and query string injection vectors.
|
||||
4. $regex is powerful for extraction but escape special characters properly.
|
||||
5. Check if SSJS is enabled before investing time in $where payloads.
|
||||
6. Aggregation pipelines often have weaker validation than simple find() queries.
|
||||
7. GraphQL + MongoDB is a common vulnerable combination; test variable injection.
|
||||
8. Monitor for ReDoS potential—useful for both detection and responsible DoS impact assessment.
|
||||
9. ODMs don't guarantee safety; audit raw query patterns and merge operations.
|
||||
10. Different NoSQL databases have vastly different capabilities; tailor payloads to the target.
|
||||
</pro_tips>
|
||||
|
||||
<remember>NoSQL injection succeeds where applications trust user input structure, not just values. Validate that input types match expectations, strip or reject query operators from user data, and use ODM features that enforce schemas. The absence of SQL syntax does not mean the absence of injection risk.</remember>
|
||||
</nosql_injection_guide>
|
||||
@@ -57,8 +57,6 @@ class Tracer:
|
||||
self.agents: dict[str, dict[str, Any]] = {}
|
||||
self.tool_executions: dict[int, dict[str, Any]] = {}
|
||||
self.chat_messages: list[dict[str, Any]] = []
|
||||
self.streaming_content: dict[str, str] = {}
|
||||
self.interrupted_content: dict[str, str] = {}
|
||||
|
||||
self.vulnerability_reports: list[dict[str, Any]] = []
|
||||
self.final_scan_result: str | None = None
|
||||
@@ -447,33 +445,6 @@ class Tracer:
|
||||
self.save_run_data(mark_complete=True)
|
||||
posthog.end(self, exit_reason="finished_by_tool")
|
||||
|
||||
def log_agent_creation(
|
||||
self,
|
||||
agent_id: str,
|
||||
name: str,
|
||||
task: str,
|
||||
parent_id: str | None = None,
|
||||
) -> None:
|
||||
agent_data: dict[str, Any] = {
|
||||
"id": agent_id,
|
||||
"name": name,
|
||||
"task": task,
|
||||
"status": "running",
|
||||
"parent_id": parent_id,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"updated_at": datetime.now(UTC).isoformat(),
|
||||
"tool_executions": [],
|
||||
}
|
||||
|
||||
self.agents[agent_id] = agent_data
|
||||
self._emit_event(
|
||||
"agent.created",
|
||||
actor={"agent_id": agent_id, "agent_name": name},
|
||||
payload={"task": task, "parent_id": parent_id},
|
||||
status="running",
|
||||
source="strix.agents",
|
||||
)
|
||||
|
||||
def log_chat_message(
|
||||
self,
|
||||
content: str,
|
||||
@@ -503,110 +474,6 @@ class Tracer:
|
||||
)
|
||||
return message_id
|
||||
|
||||
def log_tool_execution_start(
|
||||
self,
|
||||
agent_id: str,
|
||||
tool_name: str,
|
||||
args: dict[str, Any],
|
||||
) -> int:
|
||||
execution_id = self._next_execution_id
|
||||
self._next_execution_id += 1
|
||||
|
||||
now = datetime.now(UTC).isoformat()
|
||||
execution_data = {
|
||||
"execution_id": execution_id,
|
||||
"agent_id": agent_id,
|
||||
"tool_name": tool_name,
|
||||
"args": args,
|
||||
"status": "running",
|
||||
"result": None,
|
||||
"timestamp": now,
|
||||
"started_at": now,
|
||||
"completed_at": None,
|
||||
}
|
||||
|
||||
self.tool_executions[execution_id] = execution_data
|
||||
|
||||
if agent_id in self.agents:
|
||||
self.agents[agent_id]["tool_executions"].append(execution_id)
|
||||
|
||||
self._emit_event(
|
||||
"tool.execution.started",
|
||||
actor={
|
||||
"agent_id": agent_id,
|
||||
"tool_name": tool_name,
|
||||
"execution_id": execution_id,
|
||||
},
|
||||
payload={"args": args},
|
||||
status="running",
|
||||
source="strix.tools",
|
||||
)
|
||||
|
||||
return execution_id
|
||||
|
||||
def update_tool_execution(
|
||||
self,
|
||||
execution_id: int,
|
||||
status: str,
|
||||
result: Any | None = None,
|
||||
) -> None:
|
||||
if execution_id not in self.tool_executions:
|
||||
return
|
||||
|
||||
tool_data = self.tool_executions[execution_id]
|
||||
tool_data["status"] = status
|
||||
tool_data["result"] = result
|
||||
tool_data["completed_at"] = datetime.now(UTC).isoformat()
|
||||
|
||||
tool_name = str(tool_data.get("tool_name", "unknown"))
|
||||
agent_id = str(tool_data.get("agent_id", "unknown"))
|
||||
error_payload = result if status in {"error", "failed"} else None
|
||||
|
||||
self._emit_event(
|
||||
"tool.execution.updated",
|
||||
actor={
|
||||
"agent_id": agent_id,
|
||||
"tool_name": tool_name,
|
||||
"execution_id": execution_id,
|
||||
},
|
||||
payload={"result": result},
|
||||
status=status,
|
||||
error=error_payload,
|
||||
source="strix.tools",
|
||||
)
|
||||
|
||||
if tool_name == "create_vulnerability_report":
|
||||
finding_status = "reviewed" if status == "completed" else "rejected"
|
||||
self._emit_event(
|
||||
"finding.reviewed",
|
||||
actor={"agent_id": agent_id, "tool_name": tool_name},
|
||||
payload={"execution_id": execution_id, "result": result},
|
||||
status=finding_status,
|
||||
error=error_payload,
|
||||
source="strix.findings",
|
||||
)
|
||||
|
||||
def update_agent_status(
|
||||
self,
|
||||
agent_id: str,
|
||||
status: str,
|
||||
error_message: str | None = None,
|
||||
) -> None:
|
||||
if agent_id in self.agents:
|
||||
self.agents[agent_id]["status"] = status
|
||||
self.agents[agent_id]["updated_at"] = datetime.now(UTC).isoformat()
|
||||
if error_message:
|
||||
self.agents[agent_id]["error_message"] = error_message
|
||||
|
||||
self._emit_event(
|
||||
"agent.status.updated",
|
||||
actor={"agent_id": agent_id},
|
||||
payload={"error_message": error_message},
|
||||
status=status,
|
||||
error=error_message,
|
||||
source="strix.agents",
|
||||
)
|
||||
|
||||
def set_scan_config(self, config: dict[str, Any]) -> None:
|
||||
self.scan_config = config
|
||||
self.run_metadata.update(
|
||||
@@ -804,13 +671,6 @@ class Tracer:
|
||||
pass
|
||||
return 0.0
|
||||
|
||||
def get_agent_tools(self, agent_id: str) -> list[dict[str, Any]]:
|
||||
return [
|
||||
exec_data
|
||||
for exec_data in list(self.tool_executions.values())
|
||||
if exec_data.get("agent_id") == agent_id
|
||||
]
|
||||
|
||||
def get_real_tool_count(self) -> int:
|
||||
return sum(
|
||||
1
|
||||
@@ -879,28 +739,5 @@ class Tracer:
|
||||
target["cost"] += cost
|
||||
target["requests"] += requests
|
||||
|
||||
def update_streaming_content(self, agent_id: str, content: str) -> None:
|
||||
self.streaming_content[agent_id] = content
|
||||
|
||||
def clear_streaming_content(self, agent_id: str) -> None:
|
||||
self.streaming_content.pop(agent_id, None)
|
||||
|
||||
def get_streaming_content(self, agent_id: str) -> str | None:
|
||||
return self.streaming_content.get(agent_id)
|
||||
|
||||
def finalize_streaming_as_interrupted(self, agent_id: str) -> str | None:
|
||||
content = self.streaming_content.pop(agent_id, None)
|
||||
if content and content.strip():
|
||||
self.interrupted_content[agent_id] = content
|
||||
self.log_chat_message(
|
||||
content=content,
|
||||
role="assistant",
|
||||
agent_id=agent_id,
|
||||
metadata={"interrupted": True},
|
||||
)
|
||||
return content
|
||||
|
||||
return self.interrupted_content.pop(agent_id, None)
|
||||
|
||||
def cleanup(self) -> None:
|
||||
self.save_run_data(mark_complete=True)
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
"""Phase 1 smoke tests for StrixSession (memory compression wrapper)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from agents.memory.session import SessionABC
|
||||
|
||||
from strix.llm.strix_session import StrixSession
|
||||
|
||||
|
||||
class _FakeUnderlying(SessionABC):
|
||||
"""In-memory SessionABC used to drive StrixSession in tests."""
|
||||
|
||||
def __init__(self, items: list[Any] | None = None) -> None:
|
||||
self.items: list[Any] = list(items or [])
|
||||
self.session_id = "fake-session"
|
||||
|
||||
async def get_items(self, limit: int | None = None) -> list[Any]:
|
||||
if limit is None:
|
||||
return list(self.items)
|
||||
return list(self.items[-limit:])
|
||||
|
||||
async def add_items(self, items: list[Any]) -> None:
|
||||
self.items.extend(items)
|
||||
|
||||
async def pop_item(self) -> Any | None:
|
||||
return self.items.pop() if self.items else None
|
||||
|
||||
async def clear_session(self) -> None:
|
||||
self.items.clear()
|
||||
|
||||
|
||||
class _CompressorOK:
|
||||
"""Compressor stand-in that compresses by keeping the last item."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
def compress_history(self, messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
self.calls += 1
|
||||
return messages[-1:] if len(messages) > 1 else messages
|
||||
|
||||
|
||||
class _CompressorBoom:
|
||||
"""Compressor stand-in that always raises."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
def compress_history(self, messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
self.calls += 1
|
||||
raise RuntimeError("compressor offline")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def items() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"role": "user", "content": "first"},
|
||||
{"role": "assistant", "content": "second"},
|
||||
{"role": "user", "content": "third"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_items_compresses_when_compressor_ok(items: list[dict[str, Any]]) -> None:
|
||||
underlying = _FakeUnderlying(items)
|
||||
session = StrixSession(underlying, compressor=_CompressorOK())
|
||||
|
||||
out = await session.get_items()
|
||||
assert len(out) == 1
|
||||
assert out[0]["content"] == "third"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_items_returns_empty_without_calling_compressor() -> None:
|
||||
"""If underlying has no items, don't even invoke the compressor."""
|
||||
underlying = _FakeUnderlying([])
|
||||
compressor = _CompressorOK()
|
||||
session = StrixSession(underlying, compressor=compressor)
|
||||
|
||||
out = await session.get_items()
|
||||
assert out == []
|
||||
assert compressor.calls == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_items_falls_back_to_uncompressed_on_exception(
|
||||
items: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""C10 (AUDIT_R2): compressor failure must not tear down the run."""
|
||||
underlying = _FakeUnderlying(items)
|
||||
compressor = _CompressorBoom()
|
||||
session = StrixSession(underlying, compressor=compressor)
|
||||
|
||||
out = await session.get_items()
|
||||
# Uncompressed history returned.
|
||||
assert out == items
|
||||
# Flag set so subsequent calls skip the compressor.
|
||||
assert session.compression_disabled is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compressor_disabled_after_first_failure(items: list[dict[str, Any]]) -> None:
|
||||
"""Round 3.4 §E2 / W5 — once the compressor fails, skip it forever."""
|
||||
underlying = _FakeUnderlying(items)
|
||||
compressor = _CompressorBoom()
|
||||
session = StrixSession(underlying, compressor=compressor)
|
||||
|
||||
# First call: compressor invoked, raises, flag set.
|
||||
await session.get_items()
|
||||
assert compressor.calls == 1
|
||||
assert session.compression_disabled is True
|
||||
|
||||
# Second + third call: compressor short-circuited.
|
||||
await session.get_items()
|
||||
await session.get_items()
|
||||
assert compressor.calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_writes_pass_through(items: list[dict[str, Any]]) -> None:
|
||||
underlying = _FakeUnderlying()
|
||||
session = StrixSession(underlying, compressor=_CompressorOK())
|
||||
|
||||
await session.add_items(items)
|
||||
assert underlying.items == items
|
||||
|
||||
popped = await session.pop_item()
|
||||
assert popped == items[-1]
|
||||
|
||||
await session.clear_session()
|
||||
assert underlying.items == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_id_passes_through() -> None:
|
||||
underlying = _FakeUnderlying()
|
||||
session = StrixSession(underlying, compressor=_CompressorOK())
|
||||
assert session.session_id == "fake-session"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_items_respects_limit(items: list[dict[str, Any]]) -> None:
|
||||
"""``limit`` is forwarded to the underlying session before compression."""
|
||||
underlying = _FakeUnderlying(items)
|
||||
session = StrixSession(underlying, compressor=_CompressorOK())
|
||||
|
||||
out = await session.get_items(limit=2)
|
||||
# Underlying returned last 2 items; compressor kept the last 1.
|
||||
assert len(out) == 1
|
||||
assert out[0]["content"] == "third"
|
||||
@@ -39,21 +39,15 @@ def test_tracer_local_mode_writes_jsonl_with_correlation(
|
||||
tracer = Tracer("local-observability")
|
||||
set_global_tracer(tracer)
|
||||
tracer.set_scan_config({"targets": ["https://example.com"], "user_instructions": "focus auth"})
|
||||
tracer.log_agent_creation("agent-1", "Root Agent", "scan auth")
|
||||
tracer.log_chat_message("starting scan", "user", "agent-1")
|
||||
execution_id = tracer.log_tool_execution_start(
|
||||
"agent-1",
|
||||
"send_request",
|
||||
{"url": "https://example.com/login"},
|
||||
)
|
||||
tracer.update_tool_execution(execution_id, "completed", {"status_code": 200, "body": "ok"})
|
||||
tracer.log_chat_message("scanning login form", "assistant", "agent-1")
|
||||
|
||||
events_path = tmp_path / "strix_runs" / "local-observability" / "events.jsonl"
|
||||
assert events_path.exists()
|
||||
|
||||
events = _load_events(events_path)
|
||||
assert any(event["event_type"] == "tool.execution.updated" for event in events)
|
||||
assert not any(event["event_type"] == "traffic.intercepted" for event in events)
|
||||
assert any(event["event_type"] == "chat.message" for event in events)
|
||||
assert any(event["event_type"] == "run.configured" for event in events)
|
||||
|
||||
for event in events:
|
||||
assert event["run_id"] == "local-observability"
|
||||
@@ -66,20 +60,15 @@ def test_tracer_redacts_sensitive_payloads(monkeypatch: pytest.MonkeyPatch, tmp_
|
||||
|
||||
tracer = Tracer("redaction-run")
|
||||
set_global_tracer(tracer)
|
||||
execution_id = tracer.log_tool_execution_start(
|
||||
tracer.log_chat_message(
|
||||
"request failed with token sk-secret-token-value",
|
||||
"assistant",
|
||||
"agent-1",
|
||||
"send_request",
|
||||
{
|
||||
"url": "https://example.com",
|
||||
metadata={
|
||||
"api_key": "sk-secret-token-value",
|
||||
"authorization": "Bearer super-secret-token",
|
||||
},
|
||||
)
|
||||
tracer.update_tool_execution(
|
||||
execution_id,
|
||||
"error",
|
||||
{"error": "request failed with token sk-secret-token-value"},
|
||||
)
|
||||
|
||||
events_path = tmp_path / "strix_runs" / "redaction-run" / "events.jsonl"
|
||||
events = _load_events(events_path)
|
||||
@@ -255,7 +244,10 @@ def test_events_with_agent_id_include_agent_name(
|
||||
|
||||
tracer = Tracer("agent-name-enrichment")
|
||||
set_global_tracer(tracer)
|
||||
tracer.log_agent_creation("agent-1", "Root Agent", "scan auth")
|
||||
# _enrich_actor pulls names from the tracer's agents dict; populate it
|
||||
# the same way the orchestration path will once wired (placeholder
|
||||
# until live wiring lands).
|
||||
tracer.agents["agent-1"] = {"name": "Root Agent"}
|
||||
tracer.log_chat_message("hello", "assistant", "agent-1")
|
||||
|
||||
events_path = tmp_path / "strix_runs" / "agent-name-enrichment" / "events.jsonl"
|
||||
|
||||
Reference in New Issue
Block a user