©2019 by Liquid Binary. Free to use with attribution.

    BLACK HAT FINDER

    A sprawling custom bash script.

    I have a spam filter, PF firewall with some automatic banning, and logs for SSH, SMTP, IMAP, and web services.  I correlate these and analyze events over the past ~7 days to block offensive traffic from getting any further.  The biggest threat is not a single IP hammering away at your machine, it's the systematic mapping of all vulnerabilities accumulated by botnets to be sold as weapons for cyber warfare.

    bhFinder.jpeg

    CODE

    script content for "blackhatfinder.bash"

    #!/usr/local/bin/bash

    #!/usr/bin/env bash

     

    # find offensive IP addresses and netblocks. by searching:

    # firewall logs, ssh, smtp-submit, dovecot, spamdb, web, pf tables.

    # find properly connecting IPs and netblocks (ssh, web, smtp-submit, dovecot)

    # suggest bans that block attacks but protect use.

    # -x actually implement in pf tables ban-full

    # an IP can get likes and dislikes.

    # so can class C and class B nets. (class A?)

    # if an IP has any likes, it stays.

    # otherwise, if it has strong dislike, it is banned.

    # if a class C has multiple disliked IPs and no liked, if sum dislike is strong, ban C

    # if a class B has multiple disliked class Cs, and no liked, if sum dislike is strong, ban B

    # it's fine to ban an IP in a banned class C in a banned class B

    ## new code:

    # create friendly list and enemy list of IPs, each offense gets a listing with notes.

    # then it's easy to find aggregate offenses across services

     

    # ToDo : 

    # accept a real list of args

    # print help

    # maybe take a config file

    # verify that whitelisted IPs do not count toward subnet shards for blacklisting.

    # keep-cache option.  use-cache option.  Useful for private server and rapid IP investigating.

     

     

    # if provided with an IP, should show offending logs for that IP.

    #   and if that IP is in ban-half or ban-full

    # if run with -x (only) will add ips and nets to ban-full via pfctl

    # otherwise set variables inside if desired and get plaintext advice.

     

    verbosity=1 ;

    DEBUG=0 ;

    daysBack=6 ; # number of days to reach back for logs. 1 = today + yesterday.

    min=10 ; # this offense score gets you banned

    minmul=5 ; # this offense score over multiple offense types gets you banned

    minC=50 ; # this many offense from multiple IPs in a class C /24 bans the whole /24

    minB=150 ; # this many offense from multiple class Cs in a class B /16 bans the whole /16

    minshards=3 ; # this is how many diff. IPs justify banning a /24, or /24s to ban /16

    max=3 ; # the max number of IPs per category to list ; #defunct

     

     

    #tmpfile=$(mktemp) ;

    tmpdir=$(mktemp -d -t blackHatFinder.XXXXXX) ;

    # mkdir $tmpdir ; # file exists already ;

    tmpfile=$tmpdir/tmp ;

    tmpfile2=$tmpdir/tmp2 ;

    dayslog=$tmpdir/multiday.log

    dayfile=$tmpdir/logDuJour ;

    listA=$tmpdir/ipList.goodguy ;

    listB=$tmpdir/ipList.badguy ;

    listC=$tmpdir/ipList.parole ;

    listF=$tmpdir/ipList.block ;

     

    # check for tty attached, real people get verbose mode.

    TTY="true" ;

    if [ $(tty | grep -c -w "not") -ge 1 ] ; then 

            unset TTY ; verbosity=0 ; fi ;

     

    cleanup()

    {

            local tmp tmplist ;

            tmplist="$listA $listB $listC $listF $tmpfile $tmpfile2 $dayfile $dayslog" ;

            for tmp in $tmplist ; do

                    if [ -f $tmp ] ; then rm $tmp ; fi ;

            done ;

            if [ -d $tmpdir ] ; then rmdir $tmpdir ; fi ;

    }

     

     

    logg()

    {

            if [ $TTY ] ; then

                    echo "$*" ; fi ;

    }

     

    logL()

    {

            if [ $# -lt 1 ] ; then return ; fi ;

            if [ $verbosity -ge $1 ] ; then

                    shift ; logg $@ ; fi ;

    }

     

    #debug()

    #{

    ##      if [ $DEBUG -ge 1 ] ; then

    ##      logg "$*" ; fi ;

    #       if [ $verbosity -ge 2 ] ; then

    #               echo "$*" ;

    #       fi ;

    #}

     

    # uses $dayfile, $dayslog, $searchString, arg$1 as logfilename

    # given arg of fileNameStub, fills $dayfile with lines including $searchstring

    # found in (/var/log/)$1*. $searchString is assumed '^Dec  9' or similar

    fillDay()

    {

            logL 2 fillDay with $searchString found in $1 ;

            local files mfile datefoundever datefoundlast;

    #       echo -n '' > $dayfile ;

            fileName="$1" ;

            datefoundlast=0 ;

            if [ ${fileName:0:1} == / ] ; then

                    logL 2 absolute path for log file $fileName ;

                    files=$( ls -t1 ${fileName}* ) ;

            else    files=$( ls -t1 /var/log/${fileName}* ) ;

            fi ;

            for mfile in $files ; do

                if [ $datefoundever ] && [ $datefoundlast -lt 1 ] ; then

                    logL 2 not even searching $mfile or older ; 

                    break ;

                fi ;

            # cat file through gz if zipped

                filetype=$(file "$mfile") ;

                if [ $(echo $filetype | grep -c gzip) -ge 1 ] ; then

                    cat=gzcat ; 

                    else cat=cat ;

                fi ;

            # cat file through tcpdump if pflog

                filetype=$($cat "$mfile" | file - ) ;

                if [ $(echo $filetype | grep -c ASCII) -ge 1 ] ; then

                    $cat $mfile | grep -i "$searchString" > $dayfile ;

                elif [ $(echo $filetype | grep -c tcpdump) -ge 1 ] ; then

                        $cat $mfile | tcpdump -n -e -ttt -s 500 -r - | grep -i "$searchString" > $dayfile ;

                else logg no valid log format found for $mfile ;

                fi ;

                    datefoundlast=$(< $dayfile grep -c .) ;

                    logL 4 $datefoundlast log items match $searchstring in $mfile ;

                if [ $datefoundlast -ge 1 ] ; then 

                    datefoundever=yes ; 

                    cat $dayfile >> $dayslog ;

                fi ;

            done ;  

    }

     

     

    todaysec=$(date +%s) ;

    priorsec=$[ $todaysec - ( 86400 * $daysBack ) ] ;

    echo -n '' > $dayfile ;

     

    # go back through logs of whatever type to find events with the target date;

    # Keep searching until you have found at least one event,

    # and until the current file has no more of target date's events.

     

    logg "# Gathering ${daysBack}+ days worth of logs..." ;

     

    ## style "Dec 05" # pflog tcpdump

    for (( datesec=$priorsec; $datesec < $todaysec ; datesec += 86400 )) ; do

            datestring=$(date -r $datesec '+%b %d') ;

            searchString="^$datestring" ;

            fillDay "pflog" ;

    done ;

     

    ## style "Dec  5" # smtpd dovecot ssh

    for (( datesec=$priorsec; $datesec < $todaysec ; datesec += 86400 )) ; do

            mo=$(date -r $datesec +%b) ;

            da=$(date -r $datesec +%d) ;

            if [ $da -le 9 ] ; then

                    datestring="$mo  ${da#0}" ;

            else    datestring="$mo $da" ;

            fi ;

            searchString="^$datestring" ;

            fillDay "authlog" ;

            fillDay "maillog" ;

    done ;

     

    ## style "05/Dec/2018" # httpd

    for (( datesec=$priorsec; $datesec < $todaysec ; datesec += 86400 )) ; do

            datestring=$(date -r $datesec '+%d/%b/%Y') ;

            searchString="${datestring}:" ;

            fillDay "/var/www/logs/access.log" ;

    done ;

     

    logg "# Analyzing logs for offenses..." ;

     

     

    # it would be good here to abstract things out by label

    # associating label with rule#, and searching accordingly.

     

    ## PFLOG offenders ##

    # rule 6 is randtcp, connecting to not-services

    cat $dayslog | grep "rule 6/" | cut -d \  -f 10 | cut -d . -f 1-4 > $tmpfile ;

    while read line ; do 

            echo $line tcp >> $listB ;

    done < $tmpfile ;

    logL 3 $(< $tmpfile grep -c .) lines for PFLOG rule 6 randtcp

     

    # rule 4 is randudp, connecting to not-services

    cat $dayslog | grep "rule 4/" | cut -d \  -f 10 | cut -d . -f 1-4 > $tmpfile ;

    while read line ; do 

            echo $line udp >> $listB ;

    done < $tmpfile ;

    logL 3 $(< $tmpfile grep -c .) lines for PFLOG rule 4 randudp

     

    # rule 5 is randicmp

    cat $dayslog | grep "rule 5/" | cut -d \  -f 10 | cut -d . -f 1-4 > $tmpfile ;

    while read line ; do 

            echo $line icmp >> $listB ;

    done < $tmpfile ;

    logL 3 $(< $tmpfile grep -c .) lines for PFLOG rule 5 randicmp

     

     

    ## SSHLOG offenders ##

    cat $dayslog | grep ' sshd\[' | grep 'Disconnected' | grep '\[preauth\]' | grep -o -E -w '([0-9]{1,3}[.]{1}){3}[0-9]{1,3}' > $tmpfile ;

    while read line ; do

            echo $line SSH-auth-fail >> $listB ;

    done < $tmpfile ;

    logL 3 $(< $tmpfile grep -c .) lines SSH-auth-fail

     

    # SSH success publickey

    cat $dayslog | grep ' sshd\[' | grep ' Accepted publickey for '| grep -o -E -w '([0-9]{1,3}[.]{1}){3}[0-9]{1,3}' > $tmpfile ;

    while read line ; do

            echo $line SSH-auth-publickey >> $listA ;

    done < $tmpfile ;

    logL 3 $(< $tmpfile grep -c .) lines SSH-auth-publickey

     

     

    ## DOVECOT imap/pop offenders ##

    cat $dayslog | grep ' dovecot: ' | grep 'login:' | grep 'fail' | grep -o -E 'rip=([0-9]{1,3}[.]{1}){3}[0-9]{1,3}' | cut -d = -f 2 > $tmpfile ;

    while read line ; do

            echo $line dovecot-auth-fail >> $listB ;

    done < $tmpfile ;

    logL 3 $(< $tmpfile grep -c .) lines dovecot-auth-fail                 

     

    # DOVECOT success auth

    cat $dayslog | grep ' dovecot: ' | grep 'login:' | grep -v 'fail' | grep -v -i Disconnected | \

            grep -v Aborted | grep -o -E 'rip=([0-9]{1,3}[.]{1}){3}[0-9]{1,3}' | cut -d = -f 2 > $tmpfile ;

    while read line ; do

            echo $line dovecot-auth-success >> $listA ;

    done < $tmpfile ;

    logL 3 $(< $tmpfile grep -c .) lines dovecot-auth-success

     

     

    ## SMTPD offenders ## 

    echo -n '' > $tmpfile ;

    cat $dayslog | grep ' smtpd\[' > $tmpfile2 ;

    msglist=$(< $tmpfile2 grep ' Authentication failed' | grep -o -E '[0-9a-fA-F]{16} smtp ' | cut -d \  -f 1 | sort -u) ;

    for msg in $msglist ; do

            cat $tmpfile2 | grep $msg | grep -o -E ' address=([0-9]{1,3}[.]{1}){3}[0-9]{1,3}' | cut -d = -f 2 >> $tmpfile ;

    done ;

    while read line ; do

            echo $line smtp-auth-fail >> $listB ;

    done < $tmpfile ;

    logL 3 $(< $tmpfile grep -c .) lines smtp-auth-fail

     

    # smtpd submit success

    echo -n '' > $tmpfile ;

    cat $dayslog | grep ' smtpd\[' > $tmpfile2 ;

    msglist=$(< $tmpfile2 grep ' authentication user' | grep result=ok | grep -o -E '[0-9a-fA-F]{16} smtp ' | cut -d \  -f 1 | sort -u) ;

    for msg in $msglist ; do

            cat $tmpfile2 | grep $msg | grep -o -E ' address=([0-9]{1,3}[.]{1}){3}[0-9]{1,3}' | cut -d = -f 2 >> $tmpfile ;

    done ;

    while read line ; do

            echo $line smtp-submit >> $listA ;

    done < $tmpfile ;                                                                                                                              

    logL 3 $(< $tmpfile grep -c .) lines smtp-submit

     

     

    # web from OpenBSD httpd, with web site set to only respond properly to name:

    # requests for ^[www.]liquidbinary.com, ^[www.]liqbin.com not 404 may be friendly

    # requests for ^default are questionable, with 404 they are malicious

     

    ## HTTPD offenders ##

    cat $dayslog | grep '^default' | grep ' 404 ' | grep -o -E -w '([0-9]{1,3}[.]{1}){3}[0-9]{1,3}' > $tmpfile ;

    while read line ; do 

            echo $line web-nameless-404 >> $listB ;

    done < $tmpfile ;

    logL 3 $(< $tmpfile grep -c .) lines web-nameless-404

     

    ## auto ban in firewall ## prior offense, but not a type of offense

    pfctl -t ban-half -T show | grep -o -E -w '([0-9]{1,3}[.]{1}){3}[0-9]{1,3}' > $tmpfile ;

    while read line ; do

            echo $line pf-ban-half >> $listC ;  

    done < $tmpfile ;

    pfctl -t ban-full -T show | grep -o -E -w '([0-9]{1,3}[.]{1}){3}[0-9]{1,3}' > $tmpfile ;

    while read line ; do

            echo $line pf-ban-full >> $listC ;

    done < $tmpfile ;

    logL 3 $(< $tmpfile grep -c .) lines PF auto-ban unique IPs;

     

     

    ## spamdb banned ##

    spamdb | grep ^TRAPPED | cut -d \| -f 2 | sort -n -t . -k1,1 -k2,2 -k3,3 -k4,4 > $tmpfile ;

    while read line ; do

            echo $line spamdb-blocked >> $listB ;

    done < $tmpfile ;

    logL 3 $(< $tmpfile grep -c .) lines spamDB Blocked unique IPs;

     

    # spamdb WHITE

    spamdb | grep ^WHITE | cut -d \| -f 2 | sort -n -t . -k1,1 -k2,2 -k3,3 -k4,4 > $tmpfile ;

    while read line ; do

            echo $line spamdb-white >> $listA ;

    done < $tmpfile ;

    logL 3 $(< $tmpfile grep -c .) lines spamDB whitelisted IPs;

     

     

     

     

    ## MAIN PROGRAM LOGIC STARTS HERE ## after loading up stuff and setting files.

     

    ## grab args

    if [ $# -ge 1 ] ; then 

            if [ "$1" == "-x" ] ; then 

                    logL 2 forRealzies ; 

                    forRealzies=yes ;

            else echo searching for $1 ;

                    cat $listA $listB | grep $1 | sort | uniq -c ;

                    < $dayslog ggrep --color $1 ;

                    for tableName in ban-half ban-full ; do

                            if [ $(pfctl -t $tableName -T show | grep -c -w $1) -ge 1 ] ; then 

                                    echo pf table $tableName ;

                            fi ;

                    done

                    exit ;

            fi ;

    fi ;

     

    # worst first IP list ;

    lastscore=$min ; if [ $minmul -lt $min ] ; then lastscore=$minmul ; fi ;

    iplist=$(cat $listB | cut -d \  -f 1 | sort | uniq -c | sort -nr | awk '{print $2}') ;

    for ip in $iplist ; do 

            offscore=$(cat $listB $listC | grep -c -w ^$ip) ;

            if [ $offscore -lt $lastscore ] ; then break ; fi ;

            if [ $(< $listA grep -c ^$ip) -ge 1 ] ; then 

                    echo skipping $ip, whitelisted with errors ; # always mention dual-list conflicts

                    if [ $verbosity -ge 3 ] ; then

                            cat $listA $listB | grep $ip | sort | uniq -c ;

                    fi ;

                    continue ; 

            fi ;

            if [ $(< $listC grep ^$ip | grep -c 'ban-full') -ge 1 ] ; then 

                    logL 1 " - skipping $ip, already banned " ;

                    continue ; 

            fi ;

            offenses=$(< $listB grep -w ^$ip | cut -d \  -f 2- | sort -u ) ;

            if [ $offscore -ge $min ] ; then

                    echo $ip reason=score score:$offscore $offenses >> $listF ;

            elif [ $offscore -ge $minmul ] && [ $(echo $offenses | wc -w) -ge 2 ] ; then

                    echo $ip reason=multiple score:$offscore $offenses >> $listF ; 

            fi ;

    done ;

     

     

    # find classC attacks

    logL 2 " # Analyzing class C networks: " ;

    cat $listB | cut -d \  -f 1 | cut -d . -f 1-3 | sort | uniq -c | sort -nr > $tmpfile ;

    while read count badnet ; do 

            if [ $count -lt $minC ] ; then break ; fi ;

            savior=$( < $listA grep "^$badnet" | cut -d \  -f 1 | sort -u) ;

            if [ ${#savior} -ge 10 ] ; then

                    logg ":$badnet saved by white IP $(echo $savior)" ;

                    continue ;

            fi ;

            if [ $(pfctl -t ban-full -T show | grep -c ${badnet}.0/24) -ge 1 ] ; then

                    logL 1 " - class C $badnet already banned" ;

                    continue ;

            fi ;

            count=$(( $count + $( < $listC grep -c ^$badnet ) )) ;

            offenders=$(cat $listB $listC | grep ^$badnet | cut -d \  -f 1 | sort -u ) ;

            offendercount=$(echo "$offenders" | wc -w) ; # extra space included in offendercount ;

            if [ $offendercount -ge $minshards ] ; then

    #               logg $count offenses from ${badnet}.0/24 by $offenders ;

                    echo ${badnet}.0/24 reason=classC-score score:$count shards:$offendercount ${offenders} >> $listF ;

            fi ;

    done < $tmpfile ;

     

    # find classB attacks

    ## you must be guilty of something this round to be banned, 

    ## but your score is affected by your past behavior. (ban-half)

    logL 2 " # Analyzing class B networks: " ;

    cat $listB | cut -d \  -f 1 | cut -d . -f 1-2 | sort | uniq -c | sort -nr > $tmpfile ;

    while read count badnet ; do 

            if [ $count -lt $minB ] ; then break ; fi ;

            savior=$( < $listA grep "^$badnet" | cut -d \  -f 1 | sort -u) ;

            if [ ${#savior} -ge 10 ] ; then

                    logg ":$badnet saved by white IP $(echo $savior)" ;

                    continue ;

            fi ;

            if [ $(pfctl -t ban-full -T show | grep -c ${badnet}.0.0/16) -ge 1 ] ; then

                    logL 1 " - class B $badnet already banned" ;

                    continue ;

            fi ;

            count=$(( $count + $( < $listC grep -c ^$badnet ) )) ;

            offenders=$(cat $listB $listC | grep ^$badnet | cut -d \  -f 1 | \

                    cut -d . -f 1-3 | sort -u ) ;

            offendercount=$(echo "$offenders" | wc -w) ;

            if [ $offendercount -ge $minshards ] ; then

    #               logL 1 $count offenses from ${badnet}.0.0/16 by \

    #                       $( for off in ${offenders} ; do echo ${off}/24 ; done ) ;

                    echo ${badnet}.0.0/16 reason=classB-score score:$count shards:$offendercount ${offenders} >> $listF ;

            fi ;

    done < $tmpfile ;

     

    if [ -f $listF ] ; then

            logg " # Suggest banning the following IPs and networks:" ;

            if [ $TTY ] ; then cat $listF ; fi ;

    else

            logg "# No new suggestions." ;

    fi ;

     

    if [ $forRealzies ] ; then

            touch $listF ;

            while read ip stuff ; do

                    pfctl -t ban-full -T add "$ip" > /dev/null 2>&1 ;

            done < "$listF" ;

    # ban-full expires after 7 days and entries are moved to ban-half.

            pfctl -t ban-full -vT expire 604800 > $tmpfile 2>/dev/null ;

            while read letter ip ; do

                    if [ $letter == D ] ; then

                            pfctl -t ban-half -T add "$ip" > /dev/null 2>&1 ;

                            logg moving $ip from ban-full to ban-half

                    fi ;

            done < "$tmpfile" ;

    # ban-half expires after 30 days.

            pfctl -t ban-half -vT expire 2592000 > /dev/null 2>&1 ;

    fi ;

     

    if [ $DEBUG -lt 1 ] ; then

            cleanup ; fi ;

    Dependencies and Notes

    This script is implemented on OpenBSD using OpenSMTPD, Dovecot, OpenSSH, native httpd, and PF for firewall.  It needs to run as root because it accesses sensitive logs and sends commands to the firewall.  There are 2 tables in PF used here, ban-half which has a 50% drop rate, and ban full which has a 98% drop rate, and a 2% random rdr-to connection to a benign tcp service.  

    Running running the script without arguments analyzes logs and makes suggestions on what to ban and why.  Adding "-x" will manage pf tables named ban-half and ban-full.  If run with no tty, only warnings get output, making for a clean cron script with e-mail notifications of strangeness.