digital forensics

Performing Linux Log Forensics Investigation

Perform forensic investigation of Linux system logs including syslog, auth.log, systemd journal, kern.log, and application logs to reconstruct user activity, detect unauthorized access, and establish event timelines on compromised Linux systems.

audit-logauth-logcronjournalctllinux-forensicslinux-logslog-analysisssh-forensics
Install this skill
npx skills add mukul975/Anthropic-Cybersecurity-Skills
Framework mappings

Overview

Linux systems maintain extensive logs that serve as primary evidence sources in forensic investigations. Unlike Windows Event Logs, Linux logs are typically plain-text files stored in /var/log/ and binary journal files managed by systemd-journald. Key forensic logs include auth.log (authentication events, sudo usage, SSH sessions), syslog (system-wide messages), kern.log (kernel events), and application-specific logs. The Linux Audit framework (auditd) provides detailed security event logging comparable to Windows Security Event Logs. Forensic analysis of these logs enables investigators to reconstruct user sessions, identify unauthorized access, detect privilege escalation, trace lateral movement, and establish comprehensive event timelines.

When to Use

  • When conducting security assessments that involve performing linux log forensics investigation
  • When following incident response procedures for related security events
  • When performing scheduled security testing or auditing activities
  • When validating security controls through hands-on testing

Prerequisites

  • Familiarity with digital forensics concepts and tools
  • Access to a test or lab environment for safe execution
  • Python 3.8+ with required dependencies installed
  • Appropriate authorization for any testing activities

Key Log Files and Locations

Log File Path Contents
auth.log / secure /var/log/auth.log (Debian) or /var/log/secure (RHEL) Authentication, sudo, SSH, PAM
syslog / messages /var/log/syslog (Debian) or /var/log/messages (RHEL) General system messages
kern.log /var/log/kern.log Kernel messages, USB events, driver loads
lastlog /var/log/lastlog Last login per user (binary)
wtmp /var/log/wtmp Login/logout records (binary, read with last)
btmp /var/log/btmp Failed login attempts (binary, read with lastb)
faillog /var/log/faillog Failed login counter (binary)
cron.log /var/log/cron or /var/log/syslog Scheduled task execution
audit.log /var/log/audit/audit.log Linux Audit Framework events
journal /var/log/journal/ or /run/log/journal/ systemd binary journal
dpkg.log /var/log/dpkg.log Package installation/removal (Debian)
yum.log /var/log/yum.log Package installation/removal (RHEL)

Analysis Techniques

Authentication Log Analysis

# Find all successful SSH logins
grep "Accepted" /var/log/auth.log
 
# Find failed SSH login attempts
grep "Failed password" /var/log/auth.log
 
# Extract unique source IPs from failed logins
grep "Failed password" /var/log/auth.log | grep -oP '\d+\.\d+\.\d+\.\d+' | sort -u
 
# Find sudo command execution
grep "sudo:" /var/log/auth.log | grep "COMMAND"
 
# Detect brute force patterns (>10 failures from same IP)
grep "Failed password" /var/log/auth.log | awk '{print $(NF-3)}' | sort | uniq -c | sort -rn | head -20
 
# Find account creation events
grep "useradd\|adduser" /var/log/auth.log
 
# Detect SSH key authentication
grep "Accepted publickey" /var/log/auth.log

Systemd Journal Analysis

# Export journal in JSON format for forensic processing
journalctl --output=json --no-pager > journal_export.json
 
# Filter by time range
journalctl --since "2025-02-01" --until "2025-02-15" --output=json > timerange.json
 
# Filter by unit/service
journalctl -u sshd --output=json > sshd_journal.json
 
# Show kernel messages (USB events, module loads)
journalctl -k --output=json > kernel_journal.json
 
# Filter by priority (0=emerg to 7=debug)
journalctl -p err --output=json > errors.json
 
# Boot-specific logs
journalctl -b 0 --output=json > current_boot.json
journalctl --list-boots  # List all recorded boot sessions

Linux Audit Framework Analysis

# Search audit log for specific event types
ausearch -m USER_AUTH --start today
 
# Search for file access events
ausearch -f /etc/shadow
 
# Search for process execution
ausearch -m EXECVE --start "02/01/2025" --end "02/28/2025"
 
# Generate report of login events
aureport --login --start "02/01/2025"
 
# Generate summary of failed authentications
aureport --auth --failed
 
# Search for specific user activity
ausearch -ua 1001  # By UID
ausearch -ua username  # By username

Cron Job Investigation

# Check system-wide crontab
cat /etc/crontab
 
# Check user crontabs
ls -la /var/spool/cron/crontabs/
 
# Review cron execution logs
grep "CRON" /var/log/syslog
 
# Check for at/batch jobs
ls -la /var/spool/at/
atq

Python Forensic Log Parser

import re
import json
import sys
import os
from datetime import datetime
from collections import defaultdict
 
 
class LinuxLogForensicAnalyzer:
    """Analyze Linux system logs for forensic investigation."""
 
    def __init__(self, log_dir: str, output_dir: str):
        self.log_dir = log_dir
        self.output_dir = output_dir
        os.makedirs(output_dir, exist_ok=True)
 
    def parse_auth_log(self, auth_log_path: str) -> dict:
        """Parse auth.log for authentication events."""
        events = {
            "successful_logins": [],
            "failed_logins": [],
            "sudo_commands": [],
            "account_changes": [],
            "ssh_sessions": []
        }
 
        ssh_accepted = re.compile(
            r'(\w+\s+\d+\s+[\d:]+)\s+(\S+)\s+sshd\[\d+\]:\s+Accepted\s+(\S+)\s+for\s+(\S+)\s+from\s+([\d.]+)'
        )
        ssh_failed = re.compile(
            r'(\w+\s+\d+\s+[\d:]+)\s+(\S+)\s+sshd\[\d+\]:\s+Failed\s+password\s+for\s+(\S*)\s+from\s+([\d.]+)'
        )
        sudo_cmd = re.compile(
            r'(\w+\s+\d+\s+[\d:]+)\s+(\S+)\s+sudo:\s+(\S+)\s+:.*COMMAND=(.*)'
        )
        useradd = re.compile(
            r'(\w+\s+\d+\s+[\d:]+)\s+(\S+)\s+useradd\[\d+\]:\s+new user: name=(\S+)'
        )
 
        with open(auth_log_path, "r", errors="replace") as f:
            for line in f:
                m = ssh_accepted.search(line)
                if m:
                    events["successful_logins"].append({
                        "timestamp": m.group(1), "host": m.group(2),
                        "method": m.group(3), "user": m.group(4), "source_ip": m.group(5)
                    })
                    continue
 
                m = ssh_failed.search(line)
                if m:
                    events["failed_logins"].append({
                        "timestamp": m.group(1), "host": m.group(2),
                        "user": m.group(3), "source_ip": m.group(4)
                    })
                    continue
 
                m = sudo_cmd.search(line)
                if m:
                    events["sudo_commands"].append({
                        "timestamp": m.group(1), "host": m.group(2),
                        "user": m.group(3), "command": m.group(4).strip()
                    })
                    continue
 
                m = useradd.search(line)
                if m:
                    events["account_changes"].append({
                        "timestamp": m.group(1), "host": m.group(2),
                        "new_user": m.group(3)
                    })
 
        return events
 
    def detect_brute_force(self, auth_events: dict, threshold: int = 10) -> list:
        """Detect brute force attempts from auth log data."""
        ip_failures = defaultdict(int)
        for event in auth_events.get("failed_logins", []):
            ip_failures[event["source_ip"]] += 1
 
        brute_force = []
        for ip, count in ip_failures.items():
            if count >= threshold:
                brute_force.append({"source_ip": ip, "failed_attempts": count})
 
        return sorted(brute_force, key=lambda x: x["failed_attempts"], reverse=True)
 
    def generate_report(self, auth_log_path: str) -> str:
        """Generate comprehensive forensic analysis report."""
        auth_events = self.parse_auth_log(auth_log_path)
        brute_force = self.detect_brute_force(auth_events)
 
        report = {
            "analysis_timestamp": datetime.now().isoformat(),
            "log_source": auth_log_path,
            "summary": {
                "successful_logins": len(auth_events["successful_logins"]),
                "failed_logins": len(auth_events["failed_logins"]),
                "sudo_commands": len(auth_events["sudo_commands"]),
                "account_changes": len(auth_events["account_changes"]),
                "brute_force_sources": len(brute_force)
            },
            "brute_force_detected": brute_force,
            "auth_events": auth_events
        }
 
        report_path = os.path.join(self.output_dir, "linux_log_forensics.json")
        with open(report_path, "w") as f:
            json.dump(report, f, indent=2)
 
        print(f"[*] Successful logins: {report['summary']['successful_logins']}")
        print(f"[*] Failed logins: {report['summary']['failed_logins']}")
        print(f"[*] Sudo commands: {report['summary']['sudo_commands']}")
        print(f"[*] Brute force sources: {report['summary']['brute_force_sources']}")
        return report_path
 
 
def main():
    if len(sys.argv) < 3:
        print("Usage: python process.py <auth_log_path> <output_dir>")
        sys.exit(1)
    analyzer = LinuxLogForensicAnalyzer(os.path.dirname(sys.argv[1]), sys.argv[2])
    analyzer.generate_report(sys.argv[1])
 
 
if __name__ == "__main__":
    main()

References

Source materials

References and resources

Everything below is rendered for inspection. Script files are read-only and never run.

References 3

api-reference.md1.7 KB

API Reference — Performing Linux Log Forensics Investigation

Libraries Used

  • re: Pattern matching for log entries (IPs, users, timestamps, suspicious commands)
  • gzip: Read compressed log files (.gz)
  • pathlib: File system operations
  • collections.Counter: Aggregate brute force IPs, command tags

CLI Interface

python agent.py auth --file /var/log/auth.log
python agent.py syslog --file /var/log/syslog
python agent.py history --file /home/user/.bash_history
python agent.py timeline --files /var/log/auth.log /var/log/syslog /var/log/kern.log

Core Functions

analyze_auth_log(log_file) — Authentication log analysis

Detects: failed logins, successful logins, sudo commands, SSH events. Identifies brute force suspects (>=5 failed attempts from same IP).

analyze_syslog(log_file) — System log anomaly detection

Flags: errors/critical messages, kernel anomalies (segfault, OOM, panic), cron jobs.

analyze_command_history(history_file) — Suspicious command detection

12 patterns: remote code execution (curl|sh), reverse shells, base64 decode, crontab modification, firewall flush, history clearing, destructive commands.

timeline_analysis(log_files) — Multi-source timeline reconstruction

Merges events from multiple log files sorted by timestamp. Supports syslog format (Mon DD HH:MM:SS) and ISO 8601.

Suspicious Command Tags

Tag Pattern
REMOTE_CODE_EXECUTION curl/wget piped to sh/bash
BASH_REVERSE_SHELL /dev/tcp/ usage
NETCAT_LISTENER nc -e/-l/-p
HISTORY_CLEAR history -c
DESTRUCTIVE_COMMAND rm -rf /

Dependencies

No external packages — Python standard library only.

standards.md0.6 KB

Standards - Linux Log Forensics

Standards

  • NIST SP 800-92: Guide to Computer Security Log Management
  • NIST SP 800-86: Guide to Integrating Forensic Techniques
  • RFC 5424: The Syslog Protocol

Key Log Locations

  • /var/log/auth.log (Debian) or /var/log/secure (RHEL): Authentication events
  • /var/log/syslog (Debian) or /var/log/messages (RHEL): System messages
  • /var/log/kern.log: Kernel messages
  • /var/log/audit/audit.log: Linux Audit Framework
  • /var/log/wtmp, /var/log/btmp, /var/log/lastlog: Login records (binary)

Tools

  • journalctl: Systemd journal query tool
  • ausearch/aureport: Audit log analysis
  • logwatch, lnav: Log analysis utilities
workflows.md0.6 KB

Workflows - Linux Log Forensics

Workflow 1: Authentication Investigation

Collect /var/log/auth.log and rotated copies
    |
Parse for successful and failed SSH logins
    |
Identify brute force sources (>10 failures per IP)
    |
Trace sudo command execution by user
    |
Detect account creation/modification events
    |
Correlate with wtmp/btmp login records

Workflow 2: Full System Timeline

Collect all logs from /var/log/
    |
Export systemd journal (journalctl --output=json)
    |
Parse audit.log for security events
    |
Merge into unified timeline
    |
Identify unauthorized access and persistence

Scripts 2

agent.py6.7 KB
Display-only source. This catalog never executes bundled scripts.
#!/usr/bin/env python3
"""Agent for performing Linux log forensics investigation."""

import json
import argparse
import re
import gzip
from pathlib import Path
from collections import Counter


def analyze_auth_log(log_file):
    """Analyze /var/log/auth.log for suspicious authentication events."""
    content = _read_log(log_file)
    failed_logins = []
    successful_logins = []
    sudo_events = []
    ssh_events = []
    for line in content.splitlines():
        if "Failed password" in line or "authentication failure" in line:
            ip_match = re.search(r"from (\d+\.\d+\.\d+\.\d+)", line)
            user_match = re.search(r"for (?:invalid user )?(\S+)", line)
            failed_logins.append({
                "user": user_match.group(1) if user_match else "",
                "source_ip": ip_match.group(1) if ip_match else "",
                "line": line.strip()[:200],
            })
        elif "Accepted" in line:
            ip_match = re.search(r"from (\d+\.\d+\.\d+\.\d+)", line)
            user_match = re.search(r"for (\S+)", line)
            successful_logins.append({
                "user": user_match.group(1) if user_match else "",
                "source_ip": ip_match.group(1) if ip_match else "",
            })
        elif "sudo:" in line and "COMMAND=" in line:
            cmd_match = re.search(r"COMMAND=(.*)", line)
            user_match = re.search(r"sudo:\s+(\S+)", line)
            sudo_events.append({
                "user": user_match.group(1) if user_match else "",
                "command": cmd_match.group(1)[:200] if cmd_match else "",
            })
        elif "sshd" in line:
            ssh_events.append(line.strip()[:200])
    brute_force_ips = Counter(f.get("source_ip") for f in failed_logins if f.get("source_ip"))
    bf_suspects = [{"ip": ip, "attempts": count} for ip, count in brute_force_ips.most_common(10) if count >= 5]
    return {
        "log_file": log_file,
        "failed_logins": len(failed_logins),
        "successful_logins": len(successful_logins),
        "sudo_commands": len(sudo_events),
        "brute_force_suspects": bf_suspects,
        "failed_users": dict(Counter(f.get("user") for f in failed_logins).most_common(10)),
        "sudo_events": sudo_events[:20],
        "successful_logins_detail": successful_logins[:20],
    }


def analyze_syslog(log_file):
    """Analyze /var/log/syslog for anomalies."""
    content = _read_log(log_file)
    errors = []
    kernel_events = []
    cron_events = []
    for line in content.splitlines():
        lower = line.lower()
        if "error" in lower or "fail" in lower or "critical" in lower:
            errors.append(line.strip()[:200])
        if "kernel:" in lower:
            if any(kw in lower for kw in ["segfault", "oom", "killed", "panic", "bug", "usb"]):
                kernel_events.append(line.strip()[:200])
        if "cron" in lower:
            cron_events.append(line.strip()[:200])
    return {
        "log_file": log_file,
        "total_errors": len(errors),
        "kernel_anomalies": len(kernel_events),
        "cron_jobs": len(cron_events),
        "top_errors": errors[:20],
        "kernel_events": kernel_events[:15],
        "cron_events": cron_events[:15],
    }


def analyze_command_history(history_file):
    """Analyze bash history for suspicious commands."""
    content = Path(history_file).read_text(encoding="utf-8", errors="replace")
    suspicious_patterns = [
        (r"curl.*\|.*sh", "REMOTE_CODE_EXECUTION"),
        (r"wget.*\|.*bash", "REMOTE_CODE_EXECUTION"),
        (r"chmod\s+[47]77", "WORLD_WRITABLE_PERMISSION"),
        (r"nc\s+-[elp]", "NETCAT_LISTENER"),
        (r"/dev/tcp/", "BASH_REVERSE_SHELL"),
        (r"base64\s+-d", "BASE64_DECODE"),
        (r"python.*-c.*import.*socket", "PYTHON_REVERSE_SHELL"),
        (r"crontab\s+-[er]", "CRONTAB_MODIFICATION"),
        (r"iptables\s+-F", "FIREWALL_FLUSH"),
        (r"history\s+-c", "HISTORY_CLEAR"),
        (r"rm\s+-rf\s+/", "DESTRUCTIVE_COMMAND"),
        (r"dd\s+if=.*of=/dev/", "DISK_OVERWRITE"),
    ]
    findings = []
    for line in content.splitlines():
        cmd = line.strip()
        for pattern, tag in suspicious_patterns:
            if re.search(pattern, cmd, re.I):
                findings.append({"command": cmd[:200], "tag": tag, "pattern": pattern})
                break
    return {
        "history_file": history_file,
        "total_commands": len(content.splitlines()),
        "suspicious_commands": len(findings),
        "findings": findings[:30],
        "tags": dict(Counter(f["tag"] for f in findings)),
    }


def timeline_analysis(log_files, start_time=None, end_time=None):
    """Create timeline from multiple log sources."""
    events = []
    ts_patterns = [
        re.compile(r"(\w{3}\s+\d+\s+\d+:\d+:\d+)"),
        re.compile(r"(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2})"),
    ]
    for log_file in log_files:
        content = _read_log(log_file)
        for line in content.splitlines():
            ts = None
            for tp in ts_patterns:
                m = tp.search(line)
                if m:
                    ts = m.group(1)
                    break
            if ts:
                events.append({"timestamp": ts, "source": Path(log_file).name, "event": line.strip()[:200]})
    events.sort(key=lambda x: x["timestamp"])
    return {
        "sources": log_files,
        "total_events": len(events),
        "timeline": events[:100],
    }


def _read_log(filepath):
    """Read log file, supporting gzip."""
    p = Path(filepath)
    if p.suffix == ".gz":
        with gzip.open(filepath, "rt", encoding="utf-8", errors="replace") as f:
            return f.read()
    return p.read_text(encoding="utf-8", errors="replace")


def main():
    parser = argparse.ArgumentParser(description="Linux Log Forensics Investigation Agent")
    sub = parser.add_subparsers(dest="command")
    a = sub.add_parser("auth", help="Analyze auth.log")
    a.add_argument("--file", required=True)
    s = sub.add_parser("syslog", help="Analyze syslog")
    s.add_argument("--file", required=True)
    h = sub.add_parser("history", help="Analyze bash history")
    h.add_argument("--file", required=True)
    t = sub.add_parser("timeline", help="Create event timeline")
    t.add_argument("--files", nargs="+", required=True)
    args = parser.parse_args()
    if args.command == "auth":
        result = analyze_auth_log(args.file)
    elif args.command == "syslog":
        result = analyze_syslog(args.file)
    elif args.command == "history":
        result = analyze_command_history(args.file)
    elif args.command == "timeline":
        result = timeline_analysis(args.files)
    else:
        parser.print_help()
        return
    print(json.dumps(result, indent=2, default=str))


if __name__ == "__main__":
    main()
process.py2.1 KB
Display-only source. This catalog never executes bundled scripts.
#!/usr/bin/env python3
"""Linux Log Forensic Analyzer - Parses auth.log for forensic investigation."""
import re, json, os, sys
from datetime import datetime
from collections import defaultdict

def parse_auth_log(path: str, output_dir: str) -> str:
    os.makedirs(output_dir, exist_ok=True)
    successful, failed, sudo_cmds = [], [], []
    ssh_ok = re.compile(r'(\w+\s+\d+\s+[\d:]+)\s+\S+\s+sshd\[\d+\]:\s+Accepted\s+(\S+)\s+for\s+(\S+)\s+from\s+([\d.]+)')
    ssh_fail = re.compile(r'(\w+\s+\d+\s+[\d:]+)\s+\S+\s+sshd\[\d+\]:\s+Failed\s+password\s+for\s+(\S*)\s+from\s+([\d.]+)')
    sudo_re = re.compile(r'(\w+\s+\d+\s+[\d:]+)\s+\S+\s+sudo:\s+(\S+)\s+:.*COMMAND=(.*)')
    with open(path, "r", errors="replace") as f:
        for line in f:
            m = ssh_ok.search(line)
            if m:
                successful.append({"time": m.group(1), "method": m.group(2), "user": m.group(3), "ip": m.group(4)})
                continue
            m = ssh_fail.search(line)
            if m:
                failed.append({"time": m.group(1), "user": m.group(2), "ip": m.group(3)})
                continue
            m = sudo_re.search(line)
            if m:
                sudo_cmds.append({"time": m.group(1), "user": m.group(2), "command": m.group(3).strip()})
    # Brute force detection
    ip_counts = defaultdict(int)
    for e in failed:
        ip_counts[e["ip"]] += 1
    brute_force = [{"ip": k, "attempts": v} for k, v in ip_counts.items() if v >= 10]
    report = {"successful_logins": len(successful), "failed_logins": len(failed),
              "sudo_commands": len(sudo_cmds), "brute_force_ips": brute_force,
              "top_successful": successful[:50], "top_failed": failed[:50], "top_sudo": sudo_cmds[:50]}
    out = os.path.join(output_dir, "linux_auth_forensics.json")
    with open(out, "w") as f:
        json.dump(report, f, indent=2)
    print(f"[*] OK:{len(successful)} FAIL:{len(failed)} SUDO:{len(sudo_cmds)} BruteForce:{len(brute_force)}")
    return out

if __name__ == "__main__":
    if len(sys.argv) < 3: print("Usage: process.py <auth.log> <output_dir>"); sys.exit(1)
    parse_auth_log(sys.argv[1], sys.argv[2])

Assets 1

template.mdtext/markdown · 0.4 KB
Keep exploring