endpoint security

Deploying Osquery for Endpoint Monitoring

Deploys and configures osquery for real-time endpoint monitoring using SQL-based queries to inspect running processes, open ports, installed software, and system configuration. Use when building visibility into endpoint state, threat hunting across fleet, or implementing compliance monitoring. Activates for requests involving osquery deployment, endpoint visibility, fleet management, or SQL-based endpoint querying.

endpointendpoint-monitoringfleet-managementosquerythreat-hunting
Install this skill
npx skills add mukul975/Anthropic-Cybersecurity-Skills
Framework mappings

When to Use

Use this skill when:

  • Deploying osquery across Windows, macOS, and Linux endpoints for fleet-wide visibility
  • Building threat hunting queries using osquery's SQL interface
  • Monitoring endpoint compliance (installed software, open ports, running services)
  • Integrating osquery data with SIEM or Kolide/Fleet for centralized management

Do not use for real-time alerting (osquery is periodic/on-demand; use EDR for real-time).

Prerequisites

  • Osquery package for target OS (https://osquery.io/downloads)
  • Fleet management server (Kolide Fleet or FleetDM) for enterprise deployment
  • TLS certificates for secure agent-to-server communication
  • Log aggregation pipeline (Filebeat, Fluentd) for osquery result logs

Workflow

Step 1: Install Osquery

# Ubuntu/Debian
export OSQUERY_KEY=1484120AC4E9F8A1A577AEEE97A80C63C9D8B80B
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $OSQUERY_KEY
add-apt-repository 'deb [arch=amd64] https://pkg.osquery.io/deb deb main'
apt-get update && apt-get install osquery -y
 
# Windows (MSI)
# Download from https://osquery.io/downloads/official
msiexec /i osquery-5.12.1.msi /quiet
 
# macOS
brew install osquery

Step 2: Configure Osquery

// /etc/osquery/osquery.conf (Linux/macOS) or C:\ProgramData\osquery\osquery.conf
{
  "options": {
    "config_plugin": "filesystem",
    "logger_plugin": "filesystem",
    "logger_path": "/var/log/osquery",
    "disable_logging": "false",
    "schedule_splay_percent": "10",
    "events_expiry": "3600",
    "verbose": "false",
    "worker_threads": "2",
    "enable_monitor": "true",
    "disable_events": "false",
    "disable_audit": "false",
    "audit_allow_config": "true",
    "host_identifier": "hostname",
    "enable_syslog": "true"
  },
  "schedule": {
    "process_monitor": {
      "query": "SELECT pid, name, path, cmdline, uid, parent FROM processes WHERE on_disk = 0;",
      "interval": 300,
      "description": "Detect processes running without on-disk binary (fileless)"
    },
    "listening_ports": {
      "query": "SELECT DISTINCT p.name, p.path, lp.port, lp.protocol, lp.address FROM listening_ports lp JOIN processes p ON lp.pid = p.pid WHERE lp.port != 0;",
      "interval": 600,
      "description": "Monitor listening network ports"
    },
    "persistence_check": {
      "query": "SELECT name, path, source FROM startup_items;",
      "interval": 3600,
      "description": "Monitor persistence mechanisms"
    },
    "installed_packages": {
      "query": "SELECT name, version, source FROM deb_packages;",
      "interval": 86400,
      "description": "Daily software inventory"
    },
    "users_and_groups": {
      "query": "SELECT u.username, u.uid, u.gid, u.shell, u.directory FROM users u WHERE u.uid >= 1000;",
      "interval": 3600
    },
    "crontab_monitor": {
      "query": "SELECT * FROM crontab;",
      "interval": 3600,
      "description": "Monitor scheduled tasks"
    },
    "suid_binaries": {
      "query": "SELECT path, username, permissions FROM suid_bin;",
      "interval": 86400,
      "description": "Detect SUID binaries"
    }
  },
  "packs": {
    "incident-response": "/usr/share/osquery/packs/incident-response.conf",
    "ossec-rootkit": "/usr/share/osquery/packs/ossec-rootkit.conf",
    "vuln-management": "/usr/share/osquery/packs/vuln-management.conf"
  }
}

Step 3: Threat Hunting Queries

-- Detect processes with no on-disk binary (potential fileless malware)
SELECT pid, name, path, cmdline FROM processes WHERE on_disk = 0;
 
-- Find listening ports not associated with known services
SELECT lp.port, lp.protocol, p.name, p.path
FROM listening_ports lp JOIN processes p ON lp.pid = p.pid
WHERE lp.port NOT IN (22, 80, 443, 3306, 5432);
 
-- Detect unauthorized SSH keys
SELECT * FROM authorized_keys WHERE NOT key LIKE '%admin-team%';
 
-- Find recently modified system binaries
SELECT path, mtime, size FROM file
WHERE path LIKE '/usr/bin/%' AND mtime > (strftime('%s', 'now') - 86400);
 
-- Detect processes connecting to external IPs
SELECT DISTINCT p.name, p.path, pn.remote_address, pn.remote_port
FROM process_open_sockets pn JOIN processes p ON pn.pid = p.pid
WHERE pn.remote_address NOT LIKE '10.%'
  AND pn.remote_address NOT LIKE '172.16.%'
  AND pn.remote_address NOT LIKE '192.168.%'
  AND pn.remote_address != '127.0.0.1'
  AND pn.remote_address != '0.0.0.0';
 
-- Windows: Detect unsigned running executables
SELECT p.name, p.path, a.result AS signature_status
FROM processes p JOIN authenticode a ON p.path = a.path
WHERE a.result != 'trusted';

Step 4: Deploy FleetDM for Centralized Management

# FleetDM provides centralized osquery management
# Deploy FleetDM server, configure agents to report to it
# Agents use TLS enrollment and config from Fleet
 
# Agent configuration for Fleet:
# --tls_hostname=fleet.corp.com
# --tls_server_certs=/etc/osquery/fleet.pem
# --enroll_secret_path=/etc/osquery/enroll_secret

Key Concepts

Term Definition
Osquery Open-source endpoint agent that exposes OS state as SQL tables for querying
Schedule Periodic queries that run at defined intervals and log results
Pack Collection of related queries grouped for specific use cases (IR, compliance)
FleetDM Open-source osquery fleet management platform
Differential Results Osquery logs only changes between query executions, reducing data volume

Tools & Systems

  • Osquery: https://osquery.io/ - endpoint visibility agent
  • FleetDM: https://fleetdm.com/ - centralized fleet management
  • Kolide: Cloud-based osquery management with Slack integration
  • osquery-go: Go client library for osquery extensions

Common Pitfalls

  • Query performance: Complex queries with large table scans impact endpoint performance. Use WHERE clauses and test query cost with EXPLAIN.
  • Schedule intervals too aggressive: Running heavy queries every 60 seconds causes CPU spikes. Use 300-3600 second intervals for most queries.
  • Not using differential mode: Without differential logging, osquery logs all results every interval. Differential mode logs only changes.
  • Missing event tables: Some osquery tables require events framework enabled (process_events, socket_events). Enable with --disable_events=false.
Source materials

References and resources

Everything below is rendered for inspection. Script files are read-only and never run.

References 3

api-reference.md1.4 KB

osquery Endpoint Monitoring — API Reference

Installation

Platform Command
macOS brew install osquery
Ubuntu apt install osquery
Windows MSI installer from osquery.io

Key osquery Tables

Table Description
processes Running processes with pid, name, cmdline, uid
listening_ports Open network ports with bound process
suid_bin SUID/SGID binaries on the system
crontab Scheduled cron jobs
authorized_keys SSH authorized keys per user
kernel_modules Loaded kernel modules
docker_containers Docker container status
startup_items Boot/login startup items
file File metadata, hashes, timestamps

Fleet API Endpoints

Method Endpoint Description
GET /api/v1/fleet/hosts List enrolled hosts
GET /api/v1/fleet/hosts/{id} Host details
POST /api/v1/fleet/queries Create scheduled query
GET /api/v1/fleet/queries List queries

osquery CLI

osqueryi --json "SELECT * FROM processes LIMIT 5"
osqueryctl start   # Start osquery daemon
osqueryctl config-check  # Validate configuration

External References

standards.md0.4 KB

Standards & References

workflows.md0.6 KB

Workflows

Workflow 1: Osquery Fleet Deployment

[Install FleetDM server] → [Generate enrollment secret]
  → [Package osquery with fleet config] → [Deploy to pilot group]
  → [Verify enrollment and scheduled queries] → [Deploy to production]
  → [Create dashboards from query results] → [Ongoing monitoring]

Workflow 2: Threat Hunt with Osquery

[Define hypothesis] → [Write SQL query targeting hypothesis]
  → [Execute via FleetDM live query across fleet]
  → [Analyze results] → [Investigate anomalies]
  → [Document findings] → [Create scheduled detection if recurrent]

Scripts 2

agent.py4.3 KB
Display-only source. This catalog never executes bundled scripts.
#!/usr/bin/env python3
"""osquery endpoint monitoring agent for security auditing."""

import json
import argparse
import subprocess
from datetime import datetime


SECURITY_QUERIES = {
    "listening_ports": "SELECT p.pid, p.name, l.port, l.protocol, l.address FROM listening_ports l JOIN processes p ON l.pid = p.pid WHERE l.port != 0",
    "suid_binaries": "SELECT path, username, permissions FROM suid_bin",
    "crontab_entries": "SELECT command, path, event FROM crontab",
    "authorized_keys": "SELECT uid, username, key_file FROM authorized_keys",
    "logged_in_users": "SELECT user, host, type, time FROM logged_in_users",
    "kernel_modules": "SELECT name, size, used_by, status FROM kernel_modules WHERE status = 'Live'",
    "processes_high_cpu": "SELECT pid, name, uid, resident_size, percent_processor_time FROM processes WHERE percent_processor_time > 50",
    "docker_containers": "SELECT id, name, image, status, started_at FROM docker_containers",
    "browser_extensions": "SELECT name, identifier, version, path, browser_type FROM chrome_extensions UNION ALL SELECT name, identifier, version, path, browser_type FROM firefox_addons",
    "startup_items": "SELECT name, path, source FROM startup_items",
}


def run_osquery(query, output_format="json"):
    """Execute osquery and return results."""
    cmd = ["osqueryi", f"--{output_format}", query]
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
        if output_format == "json" and result.stdout.strip():
            return json.loads(result.stdout)
        return [{"raw": result.stdout[:1000]}]
    except FileNotFoundError:
        return [{"error": "osquery not installed. Install from https://osquery.io/downloads/"}]
    except (json.JSONDecodeError, subprocess.TimeoutExpired) as e:
        return [{"error": str(e)}]


def check_fleet_status(fleet_url, api_token):
    """Check Fleet server host enrollment status."""
    import requests
    headers = {"Authorization": f"Bearer {api_token}"}
    try:
        resp = requests.get(f"{fleet_url}/api/v1/fleet/hosts", headers=headers, timeout=10)
        resp.raise_for_status()
        hosts = resp.json().get("hosts", [])
        return [{
            "hostname": h.get("hostname", ""),
            "platform": h.get("platform", ""),
            "osquery_version": h.get("osquery_version", ""),
            "status": h.get("status", ""),
            "last_seen": h.get("seen_time", ""),
        } for h in hosts]
    except Exception as e:
        return [{"error": str(e)}]


def run_audit(queries=None, fleet_url=None, api_token=None):
    """Execute osquery security audit."""
    print(f"\n{'='*60}")
    print(f"  OSQUERY ENDPOINT MONITORING AUDIT")
    print(f"  Generated: {datetime.utcnow().isoformat()} UTC")
    print(f"{'='*60}\n")

    selected = queries or list(SECURITY_QUERIES.keys())
    results = {}
    for name in selected:
        if name in SECURITY_QUERIES:
            data = run_osquery(SECURITY_QUERIES[name])
            results[name] = data
            count = len(data) if isinstance(data, list) else 0
            print(f"--- {name.upper()} ({count} results) ---")
            for row in (data[:5] if isinstance(data, list) else []):
                if "error" not in row:
                    print(f"  {json.dumps(row)[:100]}")

    if fleet_url and api_token:
        fleet = check_fleet_status(fleet_url, api_token)
        results["fleet_hosts"] = fleet
        print(f"\n--- FLEET HOSTS ({len(fleet)}) ---")
        for h in fleet[:10]:
            if "error" not in h:
                print(f"  {h['hostname']}: {h['platform']} ({h['status']})")

    return results


def main():
    parser = argparse.ArgumentParser(description="osquery Monitoring Agent")
    parser.add_argument("--queries", nargs="+", choices=list(SECURITY_QUERIES.keys()),
                        help="Specific queries to run")
    parser.add_argument("--fleet-url", help="Fleet server URL")
    parser.add_argument("--api-token", help="Fleet API token")
    parser.add_argument("--output", help="Save report to JSON file")
    args = parser.parse_args()

    report = run_audit(args.queries, args.fleet_url, args.api_token)
    if args.output:
        with open(args.output, "w") as f:
            json.dump(report, f, indent=2, default=str)
        print(f"\n[+] Report saved to {args.output}")


if __name__ == "__main__":
    main()
process.py2.7 KB
Display-only source. This catalog never executes bundled scripts.
#!/usr/bin/env python3
"""Osquery Results Analyzer - Parses osquery JSON results for anomaly detection."""

import json
import sys
import os
from collections import Counter, defaultdict
from datetime import datetime


def parse_osquery_results(json_path: str) -> list:
    """Parse osquery result log (JSON lines format)."""
    results = []
    with open(json_path, "r") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                entry = json.loads(line)
                results.append(entry)
            except json.JSONDecodeError:
                continue
    return results


def analyze_results(results: list) -> dict:
    """Analyze osquery results for security anomalies."""
    analysis = {
        "total_entries": len(results),
        "queries": Counter(),
        "hosts": Counter(),
        "added_items": [],
        "removed_items": [],
    }

    for entry in results:
        name = entry.get("name", "unknown")
        analysis["queries"][name] += 1
        analysis["hosts"][entry.get("hostIdentifier", "unknown")] += 1

        action = entry.get("action", "")
        columns = entry.get("columns", {})

        if action == "added":
            analysis["added_items"].append({
                "query": name,
                "host": entry.get("hostIdentifier", ""),
                "timestamp": entry.get("unixTime", ""),
                "data": columns,
            })
        elif action == "removed":
            analysis["removed_items"].append({
                "query": name,
                "host": entry.get("hostIdentifier", ""),
                "data": columns,
            })

    return analysis


def generate_report(analysis: dict, output_path: str) -> None:
    """Generate osquery analysis report."""
    report = {
        "report_generated": datetime.utcnow().isoformat() + "Z",
        "total_entries": analysis["total_entries"],
        "queries_executed": dict(analysis["queries"]),
        "hosts_reporting": dict(analysis["hosts"].most_common(50)),
        "new_items_detected": len(analysis["added_items"]),
        "items_removed": len(analysis["removed_items"]),
        "recent_additions": analysis["added_items"][:50],
    }
    with open(output_path, "w") as f:
        json.dump(report, f, indent=2)


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python process.py <osqueryd.results.log>")
        sys.exit(1)
    results = parse_osquery_results(sys.argv[1])
    analysis = analyze_results(results)
    out = os.path.join(os.path.dirname(sys.argv[1]) or ".", "osquery_analysis.json")
    generate_report(analysis, out)
    print(f"Entries: {analysis['total_entries']} | New items: {len(analysis['added_items'])}")

Assets 1

template.mdtext/markdown · 0.6 KB
Keep exploring