feat: Add transport-level compression, remove application-level is_compressed

.github/workflows/ci.yml

@@ -0,0 +1,100 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [cli-v2, master]
+  push:
+    branches: [cli-v2, master]
+
+jobs:
+  unit-tests:
+    name: "Tests - Python ${{ matrix.python-version }} / ${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package
+        run: |
+          python -m pip install --upgrade pip
+          pip install ".[dev]"
+
+      - name: Verify zstandard is installed
+        run: python scripts/ci/verify_zstd_installed.py
+
+      - name: Verify transport compression imports
+        run: python scripts/ci/verify_transport_compression.py
+
+      - name: Run unit tests
+        run: pytest -v --durations=10 tests/unit/
+
+  wheel-sanity:
+    name: "Wheel install - Python ${{ matrix.python-version }} / ${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Build and install wheel
+        run: |
+          python -m pip install --upgrade pip setuptools wheel build
+          python -m build
+          python -m pip install --find-links dist limacharlie
+
+      - name: Verify CLI works
+        run: limacharlie --version
+
+      - name: Verify zstandard available after wheel install
+        run: python scripts/ci/verify_zstd_installed.py
+
+      - name: Verify zstd decompression works end-to-end
+        run: python scripts/ci/verify_zstd_decompression.py
+
+  no-zstd-fallback:
+    name: "Fallback without zstandard - Python ${{ matrix.python-version }} / ${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package then remove zstandard
+        run: |
+          python -m pip install --upgrade pip
+          pip install ".[dev]"
+          pip uninstall -y zstandard
+
+      - name: Verify zstandard is NOT importable
+        run: python scripts/ci/verify_zstd_not_importable.py
+
+      - name: Verify fallback Accept-Encoding (no zstd)
+        run: python scripts/ci/verify_fallback_encoding.py
+
+      - name: Run unit tests without zstandard
+        run: pytest -v --durations=10 tests/unit/

limacharlie/client.py

@@ -28,6 +28,7 @@
     RateLimitError,
     error_from_status_code,
 )
+from .transport_compression import ACCEPT_ENCODING, decompress_response
 from .user_agent_utils import build_user_agent
 
 __version__ = "5.0.0"
@@ -153,10 +154,13 @@ def _debug(self, msg: str) -> None:
     def unwrap(data: str, is_raw: bool = False) -> Any:
         """Decompress gzip+base64 encoded data from the API.
 
-        Used when is_compressed=true is set on requests. The API returns
-        data as base64-encoded gzip-compressed JSON.
+        .. deprecated::
+            Application-level compression (is_compressed=true) has been
+            replaced by transport-level compression (Accept-Encoding).
+            This method is kept for backward compatibility with external
+            callers but is no longer used internally by the SDK.
 
-        Args:
+        Parameters:
             data: Base64-encoded gzip-compressed string.
             is_raw: If True, return raw bytes instead of parsed JSON.
 
@@ -310,6 +314,10 @@ def _rest_call(self, url: str, verb: str, params: dict[str, Any] | None = None,
         request = URLRequest(full_url, body, headers=headers)
         request.get_method = lambda: verb
         request.add_header("User-Agent", self._user_agent)
+        # Request compressed responses at the transport level.
+        # urllib doesn't auto-decompress like requests does, so we
+        # handle decompression ourselves in the response path.
+        request.add_header("Accept-Encoding", ACCEPT_ENCODING)
         if content_type is not None:
             request.add_header("Content-Type", content_type)
 
@@ -323,6 +331,12 @@ def _rest_call(self, url: str, verb: str, params: dict[str, Any] | None = None,
 
             try:
                 data = u.read()
+                # Decompress transport-level encoding (gzip, zstd, etc.)
+                # before JSON parsing. The server may compress the entire
+                # response body when we send Accept-Encoding.
+                content_enc = u.headers.get("Content-Encoding")
+                if data and content_enc:
+                    data = decompress_response(data, content_enc)
                 resp = json.loads(data.decode()) if data else {}
             except ValueError:
                 resp = {}
@@ -347,6 +361,10 @@ def _rest_call(self, url: str, verb: str, params: dict[str, Any] | None = None,
 
         except HTTPError as e:
             error_body = e.read()
+            # Error responses can also be transport-compressed.
+            error_enc = e.headers.get("Content-Encoding") if hasattr(e, "headers") else None
+            if error_body and error_enc:
+                error_body = decompress_response(error_body, error_enc)
             try:
                 resp = json.loads(error_body.decode())
             except Exception:

limacharlie/config.py

@@ -92,7 +92,10 @@ def save_config(config: dict[str, Any]) -> None:
 
     fd, tmp_path = tempfile.mkstemp()
     try:
-        os.chown(tmp_path, os.getuid(), os.getgid())
+        # os.chown/os.getuid are Unix-only; skip on Windows where file
+        # ownership is managed by the OS via ACLs.
+        if hasattr(os, "chown"):
+            os.chown(tmp_path, os.getuid(), os.getgid())
         os.chmod(tmp_path, stat.S_IWUSR | stat.S_IRUSR)  # 0o600
         try:
             os.write(fd, content)

limacharlie/sdk/organization.py

@@ -868,15 +868,15 @@ def get_detections(self, start: int, end: int, limit: int | None = None, categor
         cursor = "-"
         n_returned = 0
         while cursor:
-            qp = {"start": str(int(start)), "end": str(int(end)), "cursor": cursor, "is_compressed": "true"}
+            qp = {"start": str(int(start)), "end": str(int(end)), "cursor": cursor}
             if limit is not None:
                 qp["limit"] = str(limit)
             if category:
                 qp["cat"] = category
 
             resp = self._client.request("GET", f"insight/{self.oid}/detections", query_params=qp)
             cursor = resp.get("next_cursor")
-            for d in self._client.unwrap(resp.get("detects", "")):
+            for d in resp.get("detects", []):
                 yield d
                 n_returned += 1
                 if limit is not None and n_returned >= limit:
@@ -913,7 +913,7 @@ def get_audit_logs(self, start: int, end: int, limit: int | None = None, event_t
         cursor = "-"
         n_returned = 0
         while cursor:
-            qp = {"start": str(int(start)), "end": str(int(end)), "cursor": cursor, "is_compressed": "true"}
+            qp = {"start": str(int(start)), "end": str(int(end)), "cursor": cursor}
             if limit is not None:
                 qp["limit"] = str(limit)
             if event_type:
@@ -923,7 +923,7 @@ def get_audit_logs(self, start: int, end: int, limit: int | None = None, event_t
 
             resp = self._client.request("GET", f"insight/{self.oid}/audit", query_params=qp)
             cursor = resp.get("next_cursor")
-            for entry in self._client.unwrap(resp.get("events", "")):
+            for entry in resp.get("events", []):
                 yield entry
                 n_returned += 1
                 if limit is not None and n_returned >= limit:
@@ -946,7 +946,7 @@ def get_jobs(self, start_time: int | None = None, end_time: int | None = None, l
             list: Job dicts.
         """
         import time as _time
-        qp = {"is_compressed": "true", "with_data": "false"}
+        qp = {"with_data": "false"}
         if start_time is None:
             start_time = int(_time.time()) - 86400
         if end_time is None:
@@ -958,8 +958,7 @@ def get_jobs(self, start_time: int | None = None, end_time: int | None = None, l
         if sid is not None:
             qp["sid"] = str(sid)
         resp = self._client.request("GET", f"job/{self.oid}", query_params=qp)
-        raw_jobs = resp.get("jobs", "")
+        raw_jobs = resp.get("jobs", {})
         if not raw_jobs:
             return []
-        jobs = self._client.unwrap(raw_jobs)
-        return [job for job_id, job in jobs.items()]
+        return [job for job_id, job in raw_jobs.items()]

limacharlie/sdk/sensor.py

@@ -253,7 +253,6 @@ def get_events(self, start: int, end: int, limit: int | None = None, event_type:
             qp = {
                 "start": str(int(start)),
                 "end": str(int(end)),
-                "is_compressed": "true",
                 "is_forward": "true" if is_forward else "false",
                 "cursor": cursor,
             }
@@ -264,7 +263,7 @@ def get_events(self, start: int, end: int, limit: int | None = None, event_type:
 
             resp = self.client.request("GET", f"insight/{self._org.oid}/{self.sid}", query_params=qp)
             cursor = resp.get("next_cursor")
-            for evt in self.client.unwrap(resp.get("events", "")):
+            for evt in resp.get("events", []):
                 yield evt
                 n_returned += 1
                 if limit is not None and n_returned >= limit:
@@ -306,9 +305,8 @@ def get_children_events(self, atom: str) -> list[dict[str, Any]]:
         Returns:
             list: Child events.
         """
-        data = self.client.request("GET", f"insight/{self._org.oid}/{self.sid}/{atom}/children",
-                                   query_params={"is_compressed": "true"})
-        return self.client.unwrap(data.get("events", ""))
+        data = self.client.request("GET", f"insight/{self._org.oid}/{self.sid}/{atom}/children")
+        return data.get("events", [])
 
     def get_event_retention(self, start: int, end: int, is_detailed: bool = False) -> dict[str, Any]:
         """Get event retention statistics.

limacharlie/transport_compression.py

@@ -0,0 +1,73 @@
+"""Transport-level HTTP compression support.
+
+Handles Accept-Encoding negotiation and Content-Encoding decompression
+for HTTP responses. Supports zstd, gzip, and deflate.
+
+zstd is preferred because it offers better compression ratios and faster
+decompression than gzip. The zstandard package is a hard dependency but
+the runtime gracefully falls back to gzip/deflate if it's unavailable
+(e.g., exotic platform where the wheel couldn't be installed).
+"""
+
+from __future__ import annotations
+
+import zlib
+
+# zstandard is a hard dependency (listed in pyproject.toml) with pre-built
+# wheels for all major platforms. However, we guard the import so the SDK
+# still works if someone is on an exotic platform where the wheel isn't
+# available and there's no C compiler to build from source.
+try:
+    import zstandard as _zstd
+
+    _HAS_ZSTD = True
+except ImportError:
+    _zstd = None  # type: ignore[assignment]
+    _HAS_ZSTD = False
+
+# Header value sent on every request. Prefer zstd when available.
+ACCEPT_ENCODING: str = "zstd, gzip, deflate" if _HAS_ZSTD else "gzip, deflate"
+
+
+def decompress_response(data: bytes, content_encoding: str | None) -> bytes:
+    """Decompress an HTTP response body based on Content-Encoding.
+
+    If the encoding is unrecognized or absent, the data is returned as-is
+    (passthrough). This matches standard HTTP client behavior - servers may
+    return uncompressed responses even when Accept-Encoding was sent.
+
+    Parameters:
+        data: Raw response body bytes.
+        content_encoding: Value of the Content-Encoding response header,
+            or None if the header was absent.
+
+    Returns:
+        Decompressed bytes, or the original bytes if no decompression needed.
+    """
+    if not content_encoding:
+        return data
+
+    encoding = content_encoding.strip().lower()
+
+    if encoding == "zstd":
+        if not _HAS_ZSTD:
+            # Server sent zstd but we can't decompress - return as-is.
+            # JSON parsing will fail with a clear error downstream.
+            return data
+        return _zstd.ZstdDecompressor().decompress(data)
+
+    if encoding in ("gzip", "x-gzip"):
+        # 16 + MAX_WBITS tells zlib to auto-detect gzip vs raw deflate
+        return zlib.decompress(data, 16 + zlib.MAX_WBITS)
+
+    if encoding == "deflate":
+        # Try raw deflate first, fall back to zlib-wrapped deflate
+        try:
+            return zlib.decompress(data, -zlib.MAX_WBITS)
+        except zlib.error:
+            return zlib.decompress(data)
+
+    # Unknown encoding - return data as-is rather than crashing.
+    # The caller will attempt JSON parsing which will fail with a clear
+    # error if the data is actually compressed.
+    return data

pyproject.toml

@@ -21,6 +21,7 @@ dependencies = [
     "cryptography==46.0.3",
     "click==8.1.8",
     "jmespath==1.1.0",
+    "zstandard>=0.22.0",
 ]
 
 [project.optional-dependencies]

scripts/ci/verify_fallback_encoding.py

@@ -0,0 +1,8 @@
+"""Verify Accept-Encoding falls back to gzip/deflate when zstd is unavailable."""
+
+from limacharlie.transport_compression import ACCEPT_ENCODING, _HAS_ZSTD
+
+assert not _HAS_ZSTD, "_HAS_ZSTD should be False"
+assert "zstd" not in ACCEPT_ENCODING, f"zstd should not be in: {ACCEPT_ENCODING}"
+assert "gzip" in ACCEPT_ENCODING, f"gzip missing from: {ACCEPT_ENCODING}"
+print(f"Accept-Encoding (fallback): {ACCEPT_ENCODING}")

scripts/ci/verify_transport_compression.py

@@ -0,0 +1,6 @@
+"""Verify transport compression module loads with zstd support."""
+
+from limacharlie.transport_compression import ACCEPT_ENCODING
+
+assert "zstd" in ACCEPT_ENCODING, f"zstd missing from: {ACCEPT_ENCODING}"
+print(f"Accept-Encoding: {ACCEPT_ENCODING}")

scripts/ci/verify_zstd_decompression.py

@@ -0,0 +1,13 @@
+"""Verify zstd decompression works end-to-end (compress then decompress)."""
+
+import json
+
+import zstandard
+
+from limacharlie.transport_compression import decompress_response
+
+original = json.dumps({"events": [{"type": "test"}]}).encode()
+compressed = zstandard.ZstdCompressor().compress(original)
+result = decompress_response(compressed, "zstd")
+assert result == original, "zstd round-trip failed"
+print("zstd round-trip OK")

scripts/ci/verify_zstd_installed.py

@@ -0,0 +1,5 @@
+"""Verify zstandard is installed and importable."""
+
+import zstandard
+
+print(f"zstandard {zstandard.__version__} OK")

scripts/ci/verify_zstd_not_importable.py

@@ -0,0 +1,8 @@
+"""Verify zstandard is NOT importable (used after pip uninstall)."""
+
+try:
+    import zstandard
+
+    raise AssertionError("zstandard should not be importable")
+except ImportError:
+    print("zstandard correctly unavailable")