Before diving into tools, check these red flags:

# Check unusual processes
ps auxf | grep -E '(ncat|socat|netcat|\./)|(\.\/)' | grep -v grep

# Verify critical binaries
ls -la /usr/bin/passwd /bin/ls /usr/bin/ssh

# Look for hidden directories
find / -type d -name ".*" -exec ls -ld {} \; | grep -v "/\.ssh"

Install these tools through isolated package manager:

# For Debian/Ubuntu
sudo apt-get install -y rkhunter chkrootkit lynis unhide aide

# For RHEL/CentOS
sudo yum install -y rkhunter chkrootkit lynis unhide aide

Run these commands sequentially:

# Rkhunter system check
sudo rkhunter --update
sudo rkhunter --propupd
sudo rkhunter --check --sk

# Chkrootkit scan
sudo chkrootkit -x | grep -E 'INFECTED|Warning'

# Lynis audit
sudo lynis audit system --quick

# AIDE database initialization (first run)
sudo aideinit -y -f

Detect C2 connections and suspicious ports:

# Network connections analysis
ss -tulnp | grep -E '(127.0.0.1|::1)' -v

# Hidden processes detection
unhide-tcp

# Packet capture analysis (run for 30s)
timeout 30 tcpdump -i any -w capture.pcap
tcpdump -n -r capture.pcap | awk '{print $3,$5}' | sort | uniq -c | sort -n

Compare against known-good states:

# Find recently modified binaries
find /usr/bin /usr/sbin /bin /sbin -type f -mtime -7 -exec ls -la {} \;

# Check for unauthorized SUID binaries
find / -perm -4000 -type f -exec ls -ld {} \; 2>/dev/null

# Verify package integrity
dpkg -V || rpm -Va

For quick mitigation (run as root):

#!/bin/bash
# Revoke suspicious crons
crontab -l | grep -E '(wget|curl|bash -c)' | cut -f 1 -d ' ' | xargs -I {} crontab -l | grep -v {} | crontab -

# Reset critical permissions
chmod 750 /root
chmod 700 /root/.ssh
chmod 600 /root/.ssh/authorized_keys

# Kill suspicious processes
ps aux | awk '{if($3>50.0) print $2}' | xargs -I {} kill -9 {}

After remediation steps:

# Verify SSH integrity
ssh-keygen -lf /etc/ssh/ssh_host_rsa_key.pub

# Check kernel module tampering
lsmod | grep -Ev '(ahci|ext4|usbcore)'

# Verify DNS resolution
dig +short myip.opendns.com @resolver1.opendns.com

Implement ongoing protection:

# Install OSSEC HIDS
wget https://github.com/ossec/ossec-hids/archive/3.6.0.tar.gz
tar -xvzf 3.6.0.tar.gz
cd ossec-hids-3.6.0
./install.sh

# Configure AIDE for daily checks
echo "0 3 * * * root /usr/bin/aide --check" > /etc/cron.d/aide-check

---

When dealing with a potentially compromised Linux server, start with these fundamental checks:

# Check for unusual processes
ps aux | grep -E '(\.tmp|\.var|\.cache)' 

# Verify critical binaries
ls -la /bin/ /sbin/ /usr/bin/ /usr/sbin/ | grep -E '(1970|1980|1990)'

# Examine cron jobs
ls -la /etc/cron* /var/spool/cron/

Compare system files against known good baselines:

# RPM-based systems:
rpm -Va | grep '^..5'

# Debian-based systems:
debsums -ae

# Check for modified shared libraries
ldd /bin/ls | grep -v 'linux-vdso.so'

Identify suspicious network activity:

# Established connections
netstat -tulnp | grep -v '127.0.0.1'

# Hidden connections
ss -tulnp | grep -v 'Local Address'

# Deep packet inspection
tcpdump -i eth0 -nnvvXSs 1514 'port not 22' -c 100

Run specialized detection utilities:

# Install and run rkhunter
wget https://sourceforge.net/projects/rkhunter/files/latest/download
tar xvf rkhunter-*.tar.gz
cd rkhunter-*
./installer.sh --install
rkhunter --check --sk

# Run chkrootkit
sudo apt install chkrootkit -y
chkrootkit -q

# Lynis system audit
sudo apt install lynis -y
lynis audit system

Check for malicious kernel modules:

# List loaded modules
lsmod | grep -E '(hp|hid|usb)'

# Verify module signatures
for module in $(ls /lib/modules/$(uname -r)/kernel); do
    modinfo $module | grep -E 'signer|sig_key'
done

Analyze running processes in memory:

# Install and use Volatility
git clone https://github.com/volatilityfoundation/volatility.git
cd volatility
python vol.py -f /proc/kcore linux_pslist

# Check for process hollowing
python vol.py -f /proc/kcore linux_check_afinfo

Audit system accounts thoroughly:

# Check for unexpected UID 0 accounts
awk -F: '($3 == 0) {print}' /etc/passwd

# Verify last logins
lastlog | grep -v "Never logged in"

# Check sudoers file
cat /etc/sudoers | grep -v '^#'

After cleaning, implement this basic hardening:

#!/bin/bash
# Disable unused services
systemctl disable smbd nmbd firebird-super

# Reset all passwords
for user in $(cut -d: -f1 /etc/passwd); do
    passwd -l $user
done

# Install and configure fail2ban
apt install fail2ban -y
cat > /etc/fail2ban/jail.local << EOF
[sshd]
enabled = true
maxretry = 3
bantime = 1h
EOF

Implement ongoing security monitoring:

# Install OSSEC
wget https://github.com/ossec/ossec-hids/archive/3.6.0.tar.gz
tar -xzf 3.6.0.tar.gz
cd ossec-hids-3.6.0
./install.sh

# Configure auditd
apt install auditd -y
auditctl -e 1
auditctl -a always,exit -F arch=b64 -S execve -k process_exec