From 393548c6e8d7cea5e38e7575909c3bf690c7fdc1 Mon Sep 17 00:00:00 2001 From: 0xallam Date: Mon, 25 May 2026 23:22:01 -0700 Subject: [PATCH] Add Scarf telemetry alongside PostHog Both backends share session/version/first-run helpers in strix/telemetry/_common.py and fire from the same four call sites in strix/interface/main.py and strix/report/state.py. STRIX_TELEMETRY is the single toggle for both. Co-Authored-By: Claude Opus 4.7 --- docs/advanced/configuration.mdx | 10 +-- strix/config/settings.py | 10 +-- strix/interface/main.py | 20 +++-- strix/report/state.py | 4 +- strix/telemetry/README.md | 4 +- strix/telemetry/__init__.py | 3 +- strix/telemetry/_common.py | 50 ++++++++++++ strix/telemetry/posthog.py | 60 +++----------- strix/telemetry/scarf.py | 140 ++++++++++++++++++++++++++++++++ 9 files changed, 224 insertions(+), 77 deletions(-) create mode 100644 strix/telemetry/_common.py create mode 100644 strix/telemetry/scarf.py diff --git a/docs/advanced/configuration.mdx b/docs/advanced/configuration.mdx index a40b9fc..27fba22 100644 --- a/docs/advanced/configuration.mdx +++ b/docs/advanced/configuration.mdx @@ -42,15 +42,7 @@ Configure Strix using environment variables or a config file. - Global telemetry default toggle. Set to `0`, `false`, `no`, or `off` to disable both PostHog and OTEL unless overridden by per-channel flags below. - - - - Enable/disable OpenTelemetry run observability independently. When unset, falls back to `STRIX_TELEMETRY`. - - - - Enable/disable PostHog product telemetry independently. When unset, falls back to `STRIX_TELEMETRY`. + Telemetry toggle. Set to `0`, `false`, `no`, or `off` to disable telemetry (PostHog, Scarf, OTEL). diff --git a/strix/config/settings.py b/strix/config/settings.py index 6e696b1..bdd96c2 100644 --- a/strix/config/settings.py +++ b/strix/config/settings.py @@ -72,17 +72,11 @@ class RuntimeSettings(BaseSettings): class TelemetrySettings(BaseSettings): - """Telemetry toggles. ``posthog`` is None → inherit ``master``.""" + """Telemetry toggle.""" model_config = _BASE_CONFIG - master: bool = Field(default=True, alias="STRIX_TELEMETRY") - posthog: bool | None = Field(default=None, alias="STRIX_POSTHOG_TELEMETRY") - - @property - def posthog_enabled(self) -> bool: - """Effective PostHog toggle: explicit value if set, else ``master``.""" - return self.master if self.posthog is None else self.posthog + enabled: bool = Field(default=True, alias="STRIX_TELEMETRY") class IntegrationSettings(BaseSettings): diff --git a/strix/interface/main.py b/strix/interface/main.py index 6fafab1..b9c0145 100644 --- a/strix/interface/main.py +++ b/strix/interface/main.py @@ -44,7 +44,7 @@ from strix.interface.utils import ( ) from strix.report.state import get_global_report_state from strix.report.writer import read_run_record, write_run_record -from strix.telemetry import posthog +from strix.telemetry import posthog, scarf from strix.telemetry.logging import configure_dependency_logging @@ -735,13 +735,15 @@ def main() -> None: # re-supplying targets / instructions / scope. _persist_run_record(args) - posthog.start( - model=load_settings().llm.model, - scan_mode=args.scan_mode, - is_whitebox=is_whitebox_scan(args.targets_info), - interactive=not args.non_interactive, - has_instructions=bool(args.instruction), - ) + _telemetry_start_kwargs = { + "model": load_settings().llm.model, + "scan_mode": args.scan_mode, + "is_whitebox": is_whitebox_scan(args.targets_info), + "interactive": not args.non_interactive, + "has_instructions": bool(args.instruction), + } + posthog.start(**_telemetry_start_kwargs) + scarf.start(**_telemetry_start_kwargs) exit_reason = "user_exit" try: @@ -754,6 +756,7 @@ def main() -> None: except Exception as e: exit_reason = "error" posthog.error("unhandled_exception", str(e)) + scarf.error("unhandled_exception", str(e)) raise finally: report_state = get_global_report_state() @@ -764,6 +767,7 @@ def main() -> None: ) report_state.cleanup(status=status) posthog.end(report_state, exit_reason=exit_reason) + scarf.end(report_state, exit_reason=exit_reason) results_path = run_dir_for(args.run_name) display_completion_message(args, results_path) diff --git a/strix/report/state.py b/strix/report/state.py index c9541d0..a69ced7 100644 --- a/strix/report/state.py +++ b/strix/report/state.py @@ -16,7 +16,7 @@ from strix.report.writer import ( write_run_record, write_vulnerabilities, ) -from strix.telemetry import posthog +from strix.telemetry import posthog, scarf logger = logging.getLogger(__name__) @@ -202,6 +202,7 @@ class ReportState: self.vulnerability_reports.append(report) logger.info(f"Added vulnerability report: {report_id} - {title}") posthog.finding(severity) + scarf.finding(severity) if self.vulnerability_found_callback: self.vulnerability_found_callback(report) @@ -254,6 +255,7 @@ class ReportState: logger.info("Updated scan final fields") self.save_run_data(mark_complete=True) posthog.end(self, exit_reason="finished_by_tool") + scarf.end(self, exit_reason="finished_by_tool") def set_scan_config(self, config: dict[str, Any]) -> None: self.scan_config = config diff --git a/strix/telemetry/README.md b/strix/telemetry/README.md index bb90fe1..ba9fc1e 100644 --- a/strix/telemetry/README.md +++ b/strix/telemetry/README.md @@ -18,11 +18,9 @@ We collect only very **basic** usage data including: **Model Usage:** Which LLM model is being used (not prompts or responses)\ **Aggregate Metrics:** Vulnerability counts by severity -For complete transparency, you can inspect our [telemetry implementation](https://github.com/usestrix/strix/blob/main/strix/telemetry/posthog.py) to see the exact events we track. - ### What We **Never** Collect -- IP addresses, usernames, or any identifying information +- Usernames, or any identifying information - Scan targets, file paths, target URLs, or domains - Vulnerability details, descriptions, or code - LLM requests and responses diff --git a/strix/telemetry/__init__.py b/strix/telemetry/__init__.py index 8421eb7..b9ca597 100644 --- a/strix/telemetry/__init__.py +++ b/strix/telemetry/__init__.py @@ -1,6 +1,7 @@ -from . import posthog +from . import posthog, scarf __all__ = [ "posthog", + "scarf", ] diff --git a/strix/telemetry/_common.py b/strix/telemetry/_common.py new file mode 100644 index 0000000..ff53cee --- /dev/null +++ b/strix/telemetry/_common.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import logging +import platform +import sys +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any +from uuid import uuid4 + + +logger = logging.getLogger(__name__) + +SESSION_ID: str = uuid4().hex[:16] + +_FIRST_RUN_CACHED: bool | None = None + + +def get_version() -> str: + try: + return version("strix-agent") + except PackageNotFoundError: + logger.debug("strix-agent version lookup failed", exc_info=True) + return "unknown" + + +def is_first_run() -> bool: + global _FIRST_RUN_CACHED # noqa: PLW0603 + if _FIRST_RUN_CACHED is not None: + return _FIRST_RUN_CACHED + marker = Path.home() / ".strix" / ".seen" + if marker.exists(): + _FIRST_RUN_CACHED = False + return False + try: + marker.parent.mkdir(parents=True, exist_ok=True) + marker.touch() + except Exception: # noqa: BLE001, S110 + pass # nosec B110 + _FIRST_RUN_CACHED = True + return True + + +def base_props() -> dict[str, Any]: + return { + "os": platform.system().lower(), + "arch": platform.machine(), + "python": f"{sys.version_info.major}.{sys.version_info.minor}", + "strix_version": get_version(), + } diff --git a/strix/telemetry/posthog.py b/strix/telemetry/posthog.py index 3a2a669..3db7c0f 100644 --- a/strix/telemetry/posthog.py +++ b/strix/telemetry/posthog.py @@ -1,13 +1,15 @@ import json import logging -import platform -import sys import urllib.request -from pathlib import Path +from datetime import datetime from typing import TYPE_CHECKING, Any -from uuid import uuid4 from strix.config import load_settings +from strix.telemetry._common import ( + SESSION_ID, + base_props, + is_first_run, +) if TYPE_CHECKING: @@ -19,34 +21,9 @@ logger = logging.getLogger(__name__) _POSTHOG_PUBLIC_API_KEY = "phc_7rO3XRuNT5sgSKAl6HDIrWdSGh1COzxw0vxVIAR6vVZ" _POSTHOG_HOST = "https://us.i.posthog.com" -_SESSION_ID = uuid4().hex[:16] - def _is_enabled() -> bool: - """Master telemetry gate. ``STRIX_POSTHOG_TELEMETRY`` overrides ``STRIX_TELEMETRY``.""" - return load_settings().telemetry.posthog_enabled - - -def _is_first_run() -> bool: - marker = Path.home() / ".strix" / ".seen" - if marker.exists(): - return False - try: - marker.parent.mkdir(parents=True, exist_ok=True) - marker.touch() - except Exception: # noqa: BLE001, S110 - pass # nosec B110 - return True - - -def _get_version() -> str: - try: - from importlib.metadata import version - - return version("strix-agent") - except Exception: # noqa: BLE001 - logger.debug("strix-agent version lookup failed", exc_info=True) - return "unknown" + return load_settings().telemetry.enabled def _send(event: str, properties: dict[str, Any]) -> None: @@ -57,7 +34,7 @@ def _send(event: str, properties: dict[str, Any]) -> None: payload = { "api_key": _POSTHOG_PUBLIC_API_KEY, "event": event, - "distinct_id": _SESSION_ID, + "distinct_id": SESSION_ID, "properties": properties, } req = urllib.request.Request( # noqa: S310 @@ -74,15 +51,6 @@ def _send(event: str, properties: dict[str, Any]) -> None: logger.debug("posthog event sent: %s", event) -def _base_props() -> dict[str, Any]: - return { - "os": platform.system().lower(), - "arch": platform.machine(), - "python": f"{sys.version_info.major}.{sys.version_info.minor}", - "strix_version": _get_version(), - } - - def start( model: str | None, scan_mode: str | None, @@ -93,13 +61,13 @@ def start( _send( "scan_started", { - **_base_props(), + **base_props(), "model": model or "unknown", "scan_mode": scan_mode or "unknown", "scan_type": "whitebox" if is_whitebox else "blackbox", "interactive": interactive, "has_instructions": has_instructions, - "first_run": _is_first_run(), + "first_run": is_first_run(), }, ) @@ -108,7 +76,7 @@ def finding(severity: str) -> None: _send( "finding_reported", { - **_base_props(), + **base_props(), "severity": severity.lower(), }, ) @@ -123,8 +91,6 @@ def end(report_state: "ReportState", exit_reason: str = "completed") -> None: duration = 0.0 try: - from datetime import datetime - start = datetime.fromisoformat(report_state.start_time.replace("Z", "+00:00")) end_iso = report_state.end_time or datetime.now(start.tzinfo).isoformat() duration = (datetime.fromisoformat(end_iso.replace("Z", "+00:00")) - start).total_seconds() @@ -148,7 +114,7 @@ def end(report_state: "ReportState", exit_reason: str = "completed") -> None: _send( "scan_ended", { - **_base_props(), + **base_props(), "exit_reason": exit_reason, "duration_seconds": round(duration), "vulnerabilities_total": len(report_state.vulnerability_reports), @@ -159,7 +125,7 @@ def end(report_state: "ReportState", exit_reason: str = "completed") -> None: def error(error_type: str, error_msg: str | None = None) -> None: - props = {**_base_props(), "error_type": error_type} + props = {**base_props(), "error_type": error_type} if error_msg: props["error_msg"] = error_msg _send("error", props) diff --git a/strix/telemetry/scarf.py b/strix/telemetry/scarf.py new file mode 100644 index 0000000..25e38c4 --- /dev/null +++ b/strix/telemetry/scarf.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import logging +import urllib.parse +import urllib.request +from datetime import datetime +from typing import TYPE_CHECKING, Any + +from strix.config import load_settings +from strix.telemetry._common import ( + SESSION_ID, + base_props, + get_version, + is_first_run, +) + + +if TYPE_CHECKING: + from strix.report.state import ReportState + + +logger = logging.getLogger(__name__) + +_SCARF_ENDPOINT = "https://strix.gateway.scarf.sh" + + +def _is_enabled() -> bool: + return load_settings().telemetry.enabled + + +def _send(event: str, properties: dict[str, Any]) -> None: + if not _is_enabled(): + logger.debug("scarf disabled; skipping event %s", event) + return + try: + props = dict(properties) + version = str(props.pop("strix_version", get_version()) or "unknown") + path = f"/{urllib.parse.quote(event, safe='')}/{urllib.parse.quote(version, safe='')}" + query = urllib.parse.urlencode( + {k: ("" if v is None else str(v)) for k, v in props.items()}, + ) + url = f"{_SCARF_ENDPOINT}{path}" + if query: + url = f"{url}?{query}" + req = urllib.request.Request(url, method="POST") # noqa: S310 + with urllib.request.urlopen(req, timeout=10): # noqa: S310 # nosec B310 + pass + except Exception: # noqa: BLE001 + logger.debug("scarf send failed for event %s", event, exc_info=True) + else: + logger.debug("scarf event sent: %s", event) + + +def start( + model: str | None, + scan_mode: str | None, + is_whitebox: bool, + interactive: bool, + has_instructions: bool, +) -> None: + _send( + "scan_started", + { + **base_props(), + "session": SESSION_ID, + "model": model or "unknown", + "scan_mode": scan_mode or "unknown", + "scan_type": "whitebox" if is_whitebox else "blackbox", + "interactive": interactive, + "has_instructions": has_instructions, + "first_run": is_first_run(), + }, + ) + + +def finding(severity: str) -> None: + _send( + "finding_reported", + { + **base_props(), + "session": SESSION_ID, + "severity": severity.lower(), + }, + ) + + +def end(report_state: ReportState, exit_reason: str = "completed") -> None: + vulnerabilities_counts = {"critical": 0, "high": 0, "medium": 0, "low": 0, "info": 0} + for v in report_state.vulnerability_reports: + sev = v.get("severity", "info").lower() + if sev in vulnerabilities_counts: + vulnerabilities_counts[sev] += 1 + + duration = 0.0 + try: + scan_start = datetime.fromisoformat(report_state.start_time.replace("Z", "+00:00")) + end_iso = report_state.end_time or datetime.now(scan_start.tzinfo).isoformat() + duration = ( + datetime.fromisoformat(end_iso.replace("Z", "+00:00")) - scan_start + ).total_seconds() + except (ValueError, TypeError, AttributeError): + pass + + llm_props: dict[str, int | float] = {} + try: + usage = report_state.get_total_llm_usage() + if isinstance(usage, dict): + llm_props = { + "llm_requests": int(usage.get("requests") or 0), + "llm_input_tokens": int(usage.get("input_tokens") or 0), + "llm_output_tokens": int(usage.get("output_tokens") or 0), + "llm_tokens": int(usage.get("total_tokens") or 0), + "llm_cost": float(usage.get("cost") or 0.0), + } + except (TypeError, ValueError, AttributeError): + pass + + _send( + "scan_ended", + { + **base_props(), + "session": SESSION_ID, + "exit_reason": exit_reason, + "duration_seconds": round(duration), + "vulnerabilities_total": len(report_state.vulnerability_reports), + **{f"vulnerabilities_{k}": v for k, v in vulnerabilities_counts.items()}, + **llm_props, + }, + ) + + +def error(error_type: str, error_msg: str | None = None) -> None: + props: dict[str, Any] = { + **base_props(), + "session": SESSION_ID, + "error_type": error_type, + } + if error_msg: + props["error_msg"] = error_msg + _send("error", props)