split string in multiple lines

split csv string in multiple lines

echo 'a,b,c' | sed 's/,/\n/g'

split | (pipe) regex string in multiple lines

echo 'a|b|c' | sed 's/|/\n/g'
Advertisements

analyzes apache log and blocking url access 3 times in X minutes

block multiple URLs

#!/bin/bash
# analyzes apache log and blocking ip than access url 3 times in 2 minutes
# block only access to url not to all urls

# you can use | to separate multiple strings to block
URL_STRING_TO_BLOCK="insertPartOfUrl1Here|insertPartOfUrl2Here"

RANGE_TIME=$( date '+%H:%M' )

# make array with 2 past minutes using | separator. used as a separator field in the egrep regex 
for i in {1..2};do
  RANGE_TIME=${RANGE_TIME}\|$(date -d "-$i min" '+%H:%M:')
done

# list of ips with hits to the same url
IPs_LIST=$( egrep "$(date '+%d\/%b\/%Y:')($RANGE_TIME).*($URL_STRING_TO_BLOCK)" /var/log/httpd/access.log )

# group ips by number of hits to the same url
GROUP_BY_IPs=$( echo "$IPs_LIST" | cut -d' ' -f1 | sort | uniq -c )

# create iptables rule to block each ip that access url 3 times in N minutes than not starts with 192.168.0 range
for i in $( echo "$GROUP_BY_IPs" | awk '$1 >= 3 && $2 !~ "^192.168.0" {print $2}' );do
  /sbin/iptables -nvL | awk '{print $8}' | grep "^$i$" > /dev/null
  if [ $? -ne 0 ]; then
    for j in $( echo $URL_STRING_TO_BLOCK | sed 's/|/\n/g' ); do
      echo $( date '+%d/%m/%Y %R' ) "blocking ip: $i than access part of URL: $j" | tee -a /var/log/$( basename $0 ).log
      /sbin/iptables -A INPUT -s $i -p tcp -m tcp --dport 80 -m string --string "$j" --algo bm --to 65535 -m comment --comment "block IP than access this URL many times" -j DROP
    done
  fi
done

one liner code to blocking url access 3 times in 1 minute. (block only 1 url)

URL='insertUrlHere'; RANGE_TIME=$(date '+%H:%M'); for i in {1..1};do RANGE_TIME=${RANGE_TIME}\|$(date -d "-$i min" '+%H:%M:'); done; egrep "$(date '+%d\/%b\/%Y:')($RANGE_TIME).*$URL" /var/log/httpd/access.log | cut -d' ' -f1 | sort | uniq -c | awk '$1 >= 3 && $2 !~ "^192.168" {print $2}' | xargs -I% iptables -A INPUT -s % -p tcp -m tcp --dport 80 -m string --string "$URL" --algo bm --to 65535 -j DROP

filter specific day in apache log

grep of current day in apache

grep "$(date +%d/%b/%Y)" /var/log/httpd/access.log

you can insert part of url to filter too

grep "$(date +%d/%b/%Y).*urlToFilterHere" /var/log/httpd/access.log

grep of 1 day before the current date in apache

grep "$(date -d '-1 day' +%d/%b/%Y)" /var/log/httpd/access.log

to add, group and sort IPs

grep "$(date +%d/%b/%Y)" /var/log/httpd/access.log | cut -d' ' -f1 | sort | uniq -c | sort -g

grep of the last 10 minutes in apache

LAST10MIN=$(date '+%H:%M:'); for i in {1..10};do LAST10MIN=${LAST10MIN}\|$(date -d "-$i min" '+%H:%M:'); done; egrep "$(date '+%d\/%b\/%Y:')($LAST10MIN)" /var/log/httpd/access.log

grep of the last 10 minutes with utl to filter in apache

LAST10MIN=$(date '+%H:%M:'); for i in {1..10};do LAST10MIN=${LAST10MIN}\|$(date -d "-$i min" '+%H:%M:'); done; egrep "$(date '+%d\/%b\/%Y:')($LAST10MIN).*urlToFilterHere" /var/log/httpd/access.log

filter IPs with more access in the apache log of the last 10 minutes with specific url

LAST10MIN=$(date '+%H:%M'); for i in {1..10};do LAST10MIN=${LAST10MIN}\|$(date -d "-$i min" '+%H:%M:'); done; egrep "$(date '+%d\/%b\/%Y:')($LAST10MIN).*urlToFilterHere" /var/log/httpd/access.log | cut -d' ' -f1 | sort | uniq -c | sort -g 

filter IPs with more access in the apache log of the last 10 minutes with specific url ( previous day )

LAST10MIN=$(date '+%H:%M'); for i in {1..10};do LAST10MIN=${LAST10MIN}\|$(date -d "-$i min" '+%H:%M:'); done; egrep "$(date -d '-1 day' '+%d\/%b\/%Y:')($LAST10MIN).*urlToFilterHere" /var/log/httpd/access.log | cut -d' ' -f1 | sort | uniq -c | sort -g 

regex and pattern matching with awk

Filter rows that start with ‘r’ or ‘b’

awk '/^[rb]/' /etc/passwd

Filter rows that contain ‘G’ in the third column

df -h | awk '$3 ~ /G/'

Sum rows that contain ‘G’ in the third column

df -h | awk '$3 ~ /G/ { n+=$3 } END { print n }'

Filter rows containing ‘G’ in the third column and the first column start with ‘/dev/sd’ followed by the range of ‘a-z’

df -h | awk '$3 ~ /G/ && $1 ~ "/dev/sd[a-z]"'

Filter rows that have ‘a’ value in the first column and that contain ‘1’ in the second column

echo -e "a 1x\nb 2y" | awk '$1 == "a" && $2 ~ /1/'

Filter rows that have ‘httpd’ value in the second column and that contain value greater than 0 in columns 3 and 4

ps -A --sort -rss -o pid,comm,pcpu,pmem | awk '$2 ~ /httpd/ && ( $3>0 || $4>0 )'

prints only next line containing string

awk '/root/ {getline; print}' /etc/passwd

search for apache error in the last x minutes

one liner to show erros in last 10 minutes

LAST10MIN=$(date -d '-20 min' '+%M'); grep "$(date '+%d/%b/%Y:%H:[$LAST10MIN-%M]').*erro" /var/log/apache2/error.log
#!/bin/bash
# Search for apache error in the last X minutes

MINUTES=30
DATA=$( date '+%d/%m/%Y %R' )
LAST_ERROR_LOG=$( grep 'erro' /var/log/apache2/error.log | tail -n 1 )
CUT_TIME_ERRO_LOG=$( echo $LAST_ERROR_LOG | egrep -o '.*([0-9]){2}:[0-9]{2}|.*([0-9]){2}:[0-9]{2} [0-9]{4}' | tr -d [ )
UNIX_TIMESTAMP_CUT_TIME_ERROR_LOG=$( date -d "$CUT_TIME_ERRO_LOG" +%s )

if [ ${#LAST_ERROR_LOG} -gt 0 ] && [ $(( `date +%s` - $UNIX_TIMESTAMP_CUT_TIME_ERROR_LOG )) -lt $(( $MINUTES*60 )) ]; then

 echo -e "[$DATA] \e[31;1m[ ERROR FOUND ]\e[m $LAST_ERROR_LOG" | tee -a /var/log/$( basename $0 ).log
fi

cut time in kernel or apache log

cut last time error in kernel log

awk '/erro/ {print $1,$2,$3}' /var/log/messages | tail -1

cut last time error in apache log

awk '/erro/ {print $1,$2,$3,$4,$5}' /var/log/httpd/error_log | tail -1 | tr -d []
#!/bin/bash
# cut last time error in both kernel and apache log

for i in /var/log/messages /var/log/httpd/error_log; do
  grep 'erro' $i | tail -1 | egrep -o '.*([0-9]){2}:[0-9]{2}|.*([0-9]){2}:[0-9]{2} [0-9]{4}' | tr -d [
done