scan for errors in all system disks
echo 'device : errors'; for i in $( lsblk | awk '{ print $1 }' | grep ^sd | sort -h );do smartctl -ai /dev/$i | grep -B4 '^Non-medium error count' | sed '$d' | awk '{s+=$NF} END{ print s }' | awk -v dev=$i '$1 { print dev,":",$1 }'; done
convert -density 50x50 -quality 25 -resize 80% -compress jpeg input.pdf output.pdf
#!/bin/bash
test ! -e /tmp/curl.format && cat > /tmp/curl.format <<EOF
time_namelookup: %{time_namelookup}s\n
time_connect: %{time_connect}s\n
time_appconnect: %{time_appconnect}s\n
time_pretransfer: %{time_pretransfer}s\n
time_redirect: %{time_redirect}s\n
time_starttransfer: %{time_starttransfer}s\n
----------\n
time_total: %{time_total}s\n
EOF
curl -L -s -w "@/tmp/curl.format" $url_here
delete the current file if it exist
tar --delete -f file.tar etc/passwd
append/update the new file
tar -uvf file.tar /etc/passwd
list tar file content
tar -tvf file.tar
show the file content inside of tar archive
tar xfO file.tar etc/passwd
script to update/add a file to a tar archive
#!/bin/bash
# usage: update_tar_file.sh file.tar file_to_update
test $# -ne 2 && sed -n '/^# usage/ { s/^# //p }' $0 && exit
tar -tvf $1 | grep -q $2 && tar --delete -f $1 $2 # delete file it it already exist in tar archive
tar -uvf $1 $2 # add/update the file in the tar archive
tar -tvf $1 | grep $2 # list updated/new file in the tar archive
tar xfO $1 $2 # show content of updated/new file in the tar archive
only redirect to output file if grep return some data
grep -q root123 /etc/passwd && grep root123 /etc/passwd | tee -a out
awk
awk 'END { print NR ? $0 : "EMPTY" }' <(cat file)
bash
cat file | tee /tmp/stdout | wc -l | xargs -i test {} -eq 0 && echo empty || cat /tmp/stdout
#!/bin/bash
# check service status
# usage: check_service_status.sh service
test $# -eq 0 && echo a service need to be passed as param! && sed -n '/^# usage/ { s/^# //p }' $0 && exit # print usage and quit
systemctl status $1 | # check service status
grep -q 'active (running)' && # if service is running execute next command
systemctl show $1 --property MainPID | # get service PID
grep -q =0$ && # if PID is equal 0 execute next command
echo 'em_result=PIDFAILED|' || # print to stdout and quit if PID is equal 0 OR execute next command
systemctl status $1 | # check service status and print current status
awk '/Active:/ { $2=="active" && $3=="(running)" ? $0=$3 : $0=$2"_"$3; print }'
option 1 – line without space
printf '%.s─' $(seq 1 $(tput cols))
option 2 – line with space
printf %"$(tput cols)"s | tr ' ' '-'
option 3 – line with space
seq -s- $COLUMNS| tr -d '[:digit:]'
option 4 – line with yellow color
tput setaf 3 && printf '%.s─' $(seq 1 $(tput cols)) && tput sgr0
option 5 – line with random color
tput setaf $(shuf -i 1-4 -n 1) && printf '%.s─' $(seq 1 $(tput cols)) && tput sgr0
get seconds since the file creation
echo $(( $( date +%s )-$( date +%s -r /etc/passwd ) ))
test/compare if the file has more than 10 minutes
test $(( $( date +%s )-$( date +%s -r /etc/passwd ) )) -gt 600 && echo more than 10min
join two lines using bash
cat file | xargs -n2 -d'\n'
join two lines using sed
sed 'N;s/\n/ /' fiile
join two lines using awk
awk 'NR%2{printf "%s ",$0;next;}1' file
read -p 'inform the hostname: ' host && sed -i.bak "/^127.0.0.1/ { s/[[:blank:]]*[[:alnum:]-]*$/ $host/ }" /etc/hosts
date -r /etc/passwd '+%F %X'
if [[ $( awk '$1 > $2' <<<"77.999 77.998" ) ]]; then
echo first number is bigger than second
fi
test $# -eq 0 && echo usage: $0 file && exit
tmp_file=/tmp/tmp.$( basename $1 )
cp $1 $tmp_file
sed 1d $tmp_file |
sort -k1 |
cat <( head -1 $tmp_file ) - > $1
rm -f $tmp_file
options=(
"SHOW USERNAME"
"LIST FILES"
"Quit"
)
select option in "${options[@]}"; do
case $option in
${options[0]}) whoami ;;
${options[1]}) ls ;;
${options[2]}) exit ;;
*) echo invalid option ;;
esac
done
echo $(( $( date +%s )-$( date -r /etc/passwd +%s ) ))
pass an array as argument to a function
#!/bin/bash
array=("one 1" "two 2" "three 3")
function myFunc() {
for i in "$@"; do # print all elements
echo "$i"
done
echo $1 # print a single element
}
myFunc "${array[@]}" # call function
pass a hash as argument to a function
#!/bin/bash
declare -A age_location=( ['Man']='21 Lisbon' ['Woman']='22 Rio' )
function myFunc() {
hashmap=("$@")
for i in "${hashmap[@]}"; do # print all values
echo "value: $i"
done
echo "$1" # get a single value from an element
}
myFunc "${age_location[@]}" # call function
renew wifi IP
ip link set wlan0 down; ip link set wlan0 up
if internet is down renew wifi IP
ping -c5 8.8.8.8 || ( ip link set wlan0 down; ip link set wlan0 up )
echo 1 | tee /dev/stderr | sed 's/1/2/'
using iw command
iw dev | sed -n '/Interface/,/addr/p' | sed -n '1p; $p' | awk '{print $NF}' | xargs
using ip command
ip link show | grep ether -B 1 | awk 'NR % 2 {print $2; getline; print $2}' | awk 'ORS=NR%2?" ":"\n"'
execute commands in a single line
ssh user@192.168.0.7 'hostname; ifconfig; ls /tmp'
execute commands using here document
ssh user@192.168.0.7 <<EOF hostname ifconfig ls /tmp EOF
execute commands in multiple hosts
for i in user@lisbon user@rio user@california;do ssh $i <<EOF hostname ifconfig ls /tmp EOF done
join/merge output of 2 commands using bash builtin { }
{ echo a b; echo c d; }
join output of 2 commands and getting second column
{ echo a b; echo c d; } | awk '{ print $2 }'
join output of 3 commands and getting last column of them
{ hostname -I; uname -a; cat /etc/hosts; } | awk '{print $NF}'
using ip command
ip route | awk '/^default/ {print $5}' | xargs ip address show | sed -n '/ether /p; /inet /p' | awk 'ORS=" " { print $2 }' | awk 'split($2,arr,"/") {print arr[1], $1}'
using route and ifconfig commands
route -n | awk '/^0.0.0.0/ {print $NF}' | xargs ifconfig | sed -n '/inet /p; /ether /p' | awk 'ORS=" " {print $2}'
ping -c 10 8.8.8.8 | sed -n 'x; $p' | awk '{print $6,"loss"}'
use here document to create a file using cat
create a file maintaining interpolation
cat > file1 <<EOF $HOME abc $(hostname) EOF
suppress interpolation creating a file
cat > file2 <<'EOF' $HOME abc $(hostname) EOF
append content to a file
cat >> file2 <<FIN new content FIN
if the file was created with success execute a command printing a message
cat <<FIN> file3 && echo file created def FIN
execute command after append content to a file
cat <<EOF>> file3; wc -l file3 ghi EOF
create a formatted file inside an if statement(use of TAB is mandatory)
if true; then
cat > file4 <<-EOF
srv07:
host: 192.168.0.7
user: root
EOF
fi
* if you copy this block of code, delete heredoc lines begin and use TAB
awk
get line with maximum value in column 2 using awk
echo -e 'a 3\nb 5\na 7\nb 9' | awk '{print $2, $1}' | sort -k2 -k1nr | awk '$2 != x { print } { x = $2 }'
get line with minimum value in column 2 using awk
echo -e 'a 3\nb 5\na 7\nb 9' | awk '{print $2, $1}' | sort -k2 -k1n | awk '$2 != x { print } { x = $2 }'
datamash
get line with maximum value in column 2 using datamash
echo -e 'a 3\nb 5\na 7\nb 9' | datamash -s -W -g 1 max 2
get line with minimum value in column 2 using datamash
echo -e 'a 3\nb 5\na 7\nb 9' | datamash -s -W -g 1 min 2
put quotation marks ‘ ‘ between first EOF
cat > /tmp/test <<'EOF' $HOME EOF
while read line; do echo $line; done <<FIN line1 line2 FIN
See commands history with line number
cat -n ~/.bash_history
Delete a single history entry number 77
sed -i '77d' ~/.bash_history
Delete a range of history entry from 12 to 25
sed -i '12,25d' ~/.bash_history
Delete a range of history entry from 90 until last line
sed -i '90,$d' ~/.bash_history
save modifications in place like sed in awk
awk '{print $1,$3}' file | sponge file
cat <<EOF line$((++i)) line$((++i)) line$((++i)) EOF
nohup bash -c "while true;do date +%c; sleep 1; done" &
other way to execute a command in background immune to hangups is using screen command. Run command inside screen session and detach screen Ctrl-a-d to quit
bash
echo -e "xy\nxyz" | grep '\bxy\b'
awk
echo -e "xy\nxyz" | awk '$1 ~ /\<xy\>/'
sed
echo -e "xy\nxyz" | sed -n '/\bxy\b/p'
use here document to create a file using cat
create a file in memory
cat <<EOF $HOME abc $(hostname) EOF
create a file in memory and process it. heredoc can by piped
cat <<EOF | wc -l $HOME abc $(hostname) EOF
heredoc inside a statement or loop needs <<- redirection operator and lines starting with TAB
if true; then
cat <<-EOF
$HOME
abc
$(hostname)
EOF
fi
* if you copy this block of code, delete heredoc lines begin and use TAB
using heredoc with ssh
ssh -T user@192.168.0.7 <<EOF echo "command on local machine: $(hostname)" echo "command on remote machine: \$(hostname)" EOF
find /etc -path "*/gr*/*" -type f
param -y put output side-by-side
diff -y <( printf "%s\n" {1..5} ) <( printf "%s\n" {1..5..2} )
param -suppress-common-lines not print lines equal
diff -y --suppress-common-lines <( printf "%s\n" {1..5} ) <( printf "%s\n" {1..5..2} )
convert pdf maintain the columns
pdftotext -layout file.pdf
convert pdf maintain the columns. fixed param changes output format
pdftotext -fixed 5 file.pdf
split a single page. page 7
pdftk file.pdf cat 7 output splited_output.pdf
split range of pages from 3 to 9
pdftk file.pdf cat 3-9 output splited_output.pdf
split pages 1,3,7
pdftk file.pdf cat 1 3 7 output splited_output.pdf
split pages 1,7 and range from 9 to 15
pdftk file.pdf cat 1 7 9-15 output splited_output.pdf
#!/usr/bin/expect -f #Usage: script.sh user pass command set user [lindex $argv 0]; set pass [lindex $argv 1]; set cmd [lindex $argv 2]; spawn su -c "$cmd" - $user expect "Password: " send "$pass\r" expect "$ "
try to see the content of a file that only root has permission
./script.sh root passwdHere 'cat /etc/shadow'
how to generate an expect script automatically
1 – Use autoexpect command to generate a expect script automatically
autoexpect ssh -o StrictHostKeyChecking=no userHere@192.168.0.1
2 – Run script.exp file generated by autoexpect command
./script.exp
test if number of columns in a file is equal
awk -F: '{ print NF }' /etc/passwd | sort | uniq | wc -l | xargs -i test {} -eq 1 && echo is equal
test if number of columns in a file is different
awk -F: '{ print NF }' /etc/passwd | sort | uniq | wc -l | xargs -i test {} -ne 1 && echo is diff
test if number of columns in a file is different, print in stderr and exit
awk -F: '{ print NF }' /etc/passwd | sort | uniq | wc -l | xargs -i test {} -ne 1 && echo is diff > /dev/stderr && exit 127
join/merge output of 2 commands using bash builtin { } and getting second column
{ echo a b; echo c d; } | awk '{ print $2 }'
using tee command – way 1
seq 3 | tee >( sort -nr )
using tee command – way 2
seq 3 | tee >( sort -nr ) >( sort -n ) > /dev/null
using tee command to print to output and test content
seq 7 | tee >( cat | grep -q 5 && echo has number 5 )
using pee command
seq 3 | pee 'sort -n' 'sort -nr'
print the date of a remote file and test if your date is equal today
sshpass -p "myPasswd" ssh 192.168.0.1 'ls -l /etc/hosts' | tee >(cat | awk 'END {print $7}' | test $(cat -) -eq $(date +%d) && echo day equal || echo day diff)
search for hostname word in first column on all files in /etc directory
find /etc -exec awk '$1=="hostname" { print FILENAME":",$0 }' {} \; 2>&-
show files that has linux word on second column on /etc directory
find /etc -exec awk '$2=="linux" { print FILENAME":",$0 }' {} \; 2>&-
show files that contains unknown word in part of string on first column on /etc and /sys directories
find /{etc,sys} -exec awk '$1 ~ "unknown" { print FILENAME":",$0 }' {} \; 2>&-
To search on any column
using awk
find /etc -exec awk '$0 ~ "linux" { print FILENAME":",$0 }' {} \; 2>&-
using grep
find /etc -exec grep linux {} \; 2>&-
awk
echo -e "a\n\nb" | awk NF
sed
echo -e "a\n\nb" | sed '/^$/d'
bash
echo -e "a\n\nb" | grep -v '^$'
bash
print lines with duplicated values in column 2
tr ',' "\n" <<<'a,b,c,b,a' | nl | sort -k2 | uniq -f1 -d | cut -f2
awk + bash
creates aditional column when the value of column 1 change and show only duplicated values
tr ',' "\n" <<<'a,b,c,b,a' | sort -k1 | awk 'BEGIN{ vc=$1; n=0 } { if ( $1 != vc ) { n++ } vc=$1; print n, $0 }' | uniq -d
creates aditional column when the value of column 2 change and show only duplicated values
tr ',' "\n" <<<'a,b,c,b,a' | nl | sort -k2 | awk 'BEGIN{ vc=$2; n=0 } { if ( $2 != vc ) { n++ } vc=$2; print n, $0 }' | uniq -f2 -d
creates aditional column when the value of columns 1 and 2 is equal and the value of column 3 change
echo -e 'Rio BR 10\nRio BR 10\nLisboa PT 10\nLisboa PT 20' | awk 'BEGIN{ c1=$1; c2=$2; c3=$3; n=0 } { if ( $1 == c1 && $2 == c2 && $3 != c3 ) { n++ } c1=$1; c2=$2; c3=$3; print n, $0 }'
creates aditional column when the value of columns 1 and 2 is equal and the value of column 3 change printing only differences when the value of column 3 was different
echo -e 'Rio BR 10\nRio BR 10\nLisboa PT 10\nLisboa PT 20' | awk 'BEGIN{ c1=$1; c2=$2; c3=$3; n=0 } { if ( $1 == c1 && $2 == c2 && $3 != c3 ) { n++ } c1=$1; c2=$2; c3=$3; print n, $0 }' | awk 'BEGIN{ c1=0 } { if ( $1 != c1 ) { print p; print } p=$0; c1=$1 }'
using awk
print line match and next 5
awk '/mail/ { for(i=0; i<=5; i++) {print; getline} } /etc/passwd
print line match and previous 5
awk '/mail/{for(i=1;i<=x;)print a[i++];print}{for(i=1;i<x;i++)a[i]=a[i+1];a[x]=$0;}' x=5 /etc/passwd
using sed
print line match and next 5
sed -n '/mail/,+5p' /etc/passwd
using bash
print line match and next 5 with grep
grep mail -A 5 /etc/passwd
print line match and previous 5 with grep
grep mail -B 5 /etc/passwd
search for root word in all files on /etc dir
find /etc -type f -exec grep root {} +
trim string using bash
echo ' abc' | xargs
trim string using awk
awk 'gsub(" ","",$0)' <<<"' abc123 '"
trim string using sed
echo ' abc' | sed -r 's/\s{1,}//'
put this entry on cron to see the value of PATH variable
crontab -e * * * * * /usr/bin/env > /tmp/env.output
add the full path on the first line of your crontab
crontab -e PATH=/bin:/sbin:/usr/bin/usr/sbin:/usr/local/bin:/usr/local/sbin
egrep -n "^.{$(wc -L < /etc/passwd)}$" /etc/passwd
Creates 2 files and print to standard output
echo test1 | tee {/tmp/file1,/tmp/file2}
Concatenates content to 2 files and print to standard output
echo test2 | tee -a {/tmp/file1,/tmp/file2}
logic in a single file
#!/bin/bash
#
# ERROR HANDLER BASH
#set -o xtrace # -x display value of variables on debbug
set -o errexit # -e exit if a command or pipe fail
set -o nounset # -u exit if try to use unset variable
#---------------------------------------------------
# initial variables and error handler
#---------------------------------------------------
dirLog=/tmp
outputLog=$dirLog/$(basename $0).OUTPUT.$(date +%Y%m%d).log
debugLog=$dirLog/$(basename $0).DEBUG.$(date +%Y%m%d).log
errorLog=$dirLog/$(basename $0).ERROR.$(date +%Y%m%d).log
# redirect output and error with debug to log
exec &> >( tee $debugLog )
# redirect only error to errorLog and outputLog
exec 2> >( grep -v '^+' | sed "s/^/$(tput setaf 1)ERRO$(tput sgr0): [ $(date '+%F_%R') ] /" | tee -a {$errorLog,$outputLog} )
# redirect only output to log
exec 1> >( grep -v '^+' | tee -a $outputLog )
echo -e "START time: $(date '+%F %T')\n"
#---------------------------------------------------
# put the code of the script here
#---------------------------------------------------
echo 'the code of the script begins here!'
# try to use a command that doesn't exist
kasjhdkashjd
echo -e "\nEND time: $(date '+%F %T')"
logic in two files doing an include of error handler in a script
file: err_h.sh
#!/bin/bash -xeu
#
# ERROR HANDLER BASH
test $# -eq 0 && echo -e "\e[31;1mERRO:\e[m" you need to pass a directory as a param to error handler && exit
test ! -d $1 && echo -e "\e[31;1mERRO:\e[m" param passed to error handler is not a directory && exit
#---------------------------------------------------
# initial variables and error handler
#---------------------------------------------------
dirLog=$1
outputLog=$dirLog/$(basename $0).OUTPUT.$(date +%Y%m%d).log
debugLog=$dirLog/$(basename $0).DEBUG.$(date +%Y%m%d).log
errorLog=$dirLog/$(basename $0).ERROR.$(date +%Y%m%d).log
# redirect output and error with debug to log
exec &> >( tee $debugLog )
# redirect only error to errorLog and outputLog
exec 2> >( grep -v '^+' | sed "s/^/$(tput setaf 1)ERRO$(tput sgr0): [ $(date '+%F_%R') ] /" | tee -a {$errorLog,$outputLog} )
# redirect only output to log
exec 1> >( grep -v '^+' | tee -a $outputLog )
file: code.sh
#!/bin/bash # load content of err_h.sh using /tmp as log directory source err_h.sh /tmp echo -e "START time: $(date '+%F %T')\n" #--------------------------------------------------- # put the code of the script here #--------------------------------------------------- echo 'the code of the script begins here!' # try to use a command that doesn't exist kasjhdkashjd echo -e "\nEND time: $(date '+%F %T')"
write on errorLog and quit if a test fail in a sub script
test 0 -ne 1 && echo diff > /dev/stderr && exit 127
compress all files in a directory
find /path/to/dir -type f -exec gzip {} +
compress all files with 2017 in the name on a directory
find /path/to/dir -type f -name '*2017*' -exec gzip {} +
uncompress all files in a directory
find /path/to/dir -type f -exec gunzip {} +
using only trap command
trap command will be executed removing the temporary file and exit if an error occurs or if script finish
!/bin/bash tmp=/tmp/tmp_$$ touch $tmp trap "rm -f $tmp; exit" ERR EXIT ls -l $tmp try_to_execute_an_inexistent_command echo 'only will print this line if none err occurs'
using -e bash option + trap command
-e bash param will catch the error and will exit and the trap the trap command will be executed removing the temporary file
#!/bin/bash -e tmp=/tmp/tmp_$$ touch $tmp trap "rm -f $tmp" EXIT ls -l $tmp try_to_execute_an_inexistent_command echo 'only will print this line if none err occurs'
convert ods
convert all sheets using ssconvert (belongs to gnumeric package)
ssconvert -O 'separator=, quoting-mode=never' -S file.ods /tmp/sheet%s.txt
convert xlsx
convert xlsx to csv
xlsx2csv file.xlsx > file.csv
convert xlsx to tsv (TAB)
xlsx2csv -d '\t' file.xlsx > file.tsv
convert only sheet 3 to tsv
xlsx2csv -s 3 -d '\t' file.xlsx > file.tsv
convert multiple sheets to tsv
for i in {1..7};do xlsx2csv -s $i -d '\t' excel.xlsx > sheet-$i.csv & done
check if number of columns in excel is not equal
awk -F, '{ print NF }' file.csv | sort | uniq | wc -l | xargs -i test {} -ne 1 && echo number of columns is not equal
check if number of columns in excel is not equal, write message on stdout and quit
awk -F, '{ print NF }' file.csv | sort | uniq | wc -l | xargs -i test {} -ne 1 && echo number of columns is not equal > /dev/stderr && exit 127
tips and tricks
formulas to clean/replace ENTER inside excel cells (see ascii codes)
=CLEAN(A1) =SUBSTITUTE(SUBSTITUTE(A1,CHAR(13)," "),CHAR(10)," ") =TRIM(SUBSTITUTE(SUBSTITUTE(A1,CHAR(13)," "),CHAR(10)," "))
debug error using long options
#!/bin/bash set -o xtrace # -x display value of variables on debbug set -o errexit # -e exit if a command or pipe fail set -o nounset # -u exit if try to use unset variable # print only error to a log with current date exec 2> >( grep -v '^+' | xargs echo $(date '+%F %R') | tee -a /var/log/$( basename $0 ).log )
debug error using abbreviated options
#!/bin/bash -xeu # print only error to a log with current date exec 2> >( grep -v '^+' | xargs echo $(date '+%F %R') | tee -a /var/log/$( basename $0 ).log )
debug output and error and separated files
#!/bin/bash -xeu # redirect output and error to file exec &>> /var/log/$(basename $0).$(date +%Y%m%d).log # redirect only error to a file exec 2> >( grep -v '^+' | xargs echo ERROR [ $(date '+%F_%R') ] | tee -a /var/log/$(basename $0).ERROR.$(date +%Y%m%d).log )
#!/bin/bash -xeu exec 2> >( grep -v '^+' | xargs echo $(date '+%F %R') | tee -a /var/log/$( basename $0 ).log )
copy last file created on /usr/bin to /tmp
ls -trd /usr/bin/* | tail -1 | xargs -I% cp -P % /tmp
| Exit Code | Meaning | Example | Comments |
| 1 | Catchall for general errors | let “var1 = 1/0” | Miscellaneous errors, such as “divide by zero” and other impermissible operations |
| 2 | Misuse of shell builtins (according to Bash documentation) | empty_function() {} | Missing keyword or command, or permission problem (and diff return code on a failed binary file comparison) |
| 126 | Command invoked cannot execute | /dev/null | Permission problem or command is not an executable |
| 127 | “command not found” | illegal_command | Possible problem with $PATH or a typo |
| 128 | Invalid argument to exit | exit 3.14159 | exit takes only integer args in the range 0 – 255 (see first footnote) |
| 128+n | Fatal error signal “n” | kill -9 $PPID of script | $? returns 137 (128 + 9) |
| 130 | Script terminated by Control-C | Ctl-C | Control-C is fatal error signal 2, (130 = 128 + 2, see above) |
| 255* | Exit status out of range | exit -1 | exit takes only integer args in the range 0 – 255 |
using sed
function error {
[ $( sed 's/\s/+/g' <<<${PIPESTATUS[*]} | bc ) -eq 0 ] && return || echo error && exit 1
}
using awk
function error {
[ $( awk 'BEGIN{ OFS="+"} $1=$1; { print }' <<<${PIPESTATUS[*]} | bc ) -eq 0 ] && return || echo error && exit 1
}
apeend a new line to stdin using cat
printf %s\\n ID-{1..3}' '{1..9..4} | cat <(echo c1 c2) -
join a text to stdin using bash
echo 1 2 3 | echo nums: $(</dev/stdin)
join a text to stdin using awk
awk '{ print "nums:", $0 }' <<< '1 2 3'
join a header text to stdin using awk
echo -e "a\nb\nc" | awk 'BEGIN{print "num"} {print $0}'
filter content on file maintaining the first line “header”
grep mail /etc/passwd | cat <(head -1 /etc/passwd) -
Generate a random char
head /dev/urandom | tr -cd '[:alpha:]' | cut -c -1
Generate a random string with length 3
head /dev/urandom | tr -cd '[:alpha:]' | cut -c -3
Generate a random number with length 2
head /dev/urandom | tr -cd '[:digit:]' | cut -c -2
Generate a random alpha numeric string with length 5
head /dev/urandom | tr -cd '[:alnum:]' | cut -c -5
1 – Create 10 files with name “tmp file N”
touch tmp\ file\ {1..10}
2 – Rename all files replacing all spaces by underscores
rename 'y/ /\_/' tmp\ *
3 – Rename all files removing tmp_ from beginning
rename 's/tmp_//' tmp*
– Print names of files to be renamed, without renaming them
rename -n 's/tmp_//' tmp*
– Convert filenames to uppercase
rename 'y/a-z/A-Z/' tmp*
1 – Create a named pipe file
mkfifo /tmp/npipe
2 – Join the data
echo -e '1 Casa\n2 Carro' > /tmp/npipe & echo -e '1 Branca\n2 Preto' | join /tmp/npipe -
reading multiple values from plain text
read v1 v2 v3 <<<'car house sun'
reading multiple values from a command
read countLines fileName <<<$( wc -l /etc/passwd )
reading multiple values from multiple commands
read VAR1 VAR2 <<< "$(date +%Y%m%d) $(whoami)"
parsing a YAML file creating multiple variables. Input file:
# some comment here NAME: Mickey AGE: 33 LOCATION:PARIS
parse yaml file creating multiple variables with bash
eval $( grep ^[A-Z] file | tr -s ': ' '=' )
parse yaml file creating multiple variables with awk
eval $( awk -F':|: ' '/^[A-Z]/ { print $1"="$2 }' file )
parse yaml file creating multiple variables with sed
eval $( sed -nr 's/^(\w*): *(.*)/\1=\2/p' file )
To see the content of variables
echo $NAME $AGE $LOCATION
changing file permissions
find /path/to/dir -type f -exec chmod 600 {} +
changing directory permissions
find /path/to/dir -type d -exec chmod 700 {} +
delete all mp3 files on directory
find /path/to/dir -name '*.mp3' -delete
delete only gz files most old than 7 days on directory
find /path/to/dir -name '*.gz' -mtime +7 -delete
deletes only tar files on /tmp most old than 12 hours, not search on subdirectories
find /tmp -maxdepth 1 -type f -name '*.tar' -cmin +720 -delete
deletes only tar files of current user on /tmp, not search on subdirectories
find /tmp -maxdepth 1 -type f -name '*.tar' -user $(whoami) -delete
ssh user@host test -e /tmp && echo true || echo false
Simple way to convert
man -t ls | ps2pdf - ls.pdf
Another way to convert
zcat /usr/share/man/man1/ls.1.gz | groff -mandoc | ps2pdf - ls.pdf
using sed command very fast uses little cpu and memory
echo '<root><age>7</age></root>' | # xml in one line sed 's;><;>\n<;g' | # put new line between >< sed 's;></;>\n</;g' # put new line between ></
xmllint command is very fast and consumes too much memory and little cpu
echo '<root><age>7</age></root>' | xmllint --format -
xml_pp command is slow and consumes little memory and too much cpu
echo '<root><age>7</age></root>' | xml_pp
To improve speed use sed without pipe
sed 's;><;>\n<;g; s;></;>\n</;g' file
get only 1 random line from /etc/passwd – way 1
shuf -n 1 /etc/passwd
get only 1 random line from /etc/passwd – way 2
sed -n $[RANDOM%$(wc -l /etc/passwd | cut -d' ' -f1)+1]p /etc/passwd
get 7 random lines from /etc/passwd – way 1
shuf -n 7 /etc/passwd
get 7 random lines from /etc/passwd -way 2
for i in {1..7}; do sed -n $[RANDOM%$(wc -l /etc/passwd | cut -d' ' -f1)+1]p /etc/passwd; done
concatenate var to string
var=abc; echo ${var}def
concatenate two strings- way 1
echo abc | xargs -I% echo %def
concatenate two strings- way 2
echo abc$(echo def)
get/split substring using bash
get year of a string date
cut -c1-4 <<<'2018/05/30'
get month of a string date
cut -c6-7 <<<'2018/05/30'
get day of a string date
cut -c9-10 <<<'2018/05/30'
get/split substring using awk
using substr function
get year of a string date
awk '{ print substr($0,1,4) }' <<<'2018/05/30'
get month of a string date
awk '{ print substr($0,6,2) }' <<<'2018/05/30'
get day of a string date
awk '{ print substr($0,9) }' <<<'2018/05/30'
using split function
get year of a string date
awk 'split($0,arr,"/") { print arr[1] }' <<<'2018/05/30'
get month of a string date
awk 'split($0,arr,"/") { print arr[2] }' <<<'2018/05/30'
get day of a string date
awk 'split($0,arr,"/") { print arr[3] }' <<<'2018/05/30'
Create multiple files in parallel
for i in {1..3}; do
{
echo iteration: $i
dd if=/dev/urandom of=/tmp/file-$i bs=1MB count=100
du -hs /tmp/file-*
} &
done
Wait all anonymous block finish before end the script
{
dd if=/dev/urandom of=/tmp/file1 bs=200MB count=100
echo finish block: 1
} &
done
{
dd if=/dev/urandom of=/tmp/file2 bs=200MB count=100
echo finish block: 2
} &
wait
echo finish
echo 1 2 10 | tr ' ' '\n' | sort -V
wait 2 second while running 8 processes in parallel
for i in {1..16}; do
echo running process: $i $( hostname ) &
if [ $(($i % 8)) -eq 0 ]; then
sleep 2
fi
done
wait 2 second while processing 8 files in parallel
for i in /tmp/file*; do ((p++)) echo running file $i process: $p $( hostname ) & test $(($p % 8)) -eq 0 && sleep 2 done
excludes executable files with rc.d name in path
find /etc -type f -executable -not -path "*rc.d*"
while read line; do echo $line; done < <( paste /etc/hosts /etc/passwd )
generates a list of all users with command to create them in another machine
cut -d: --output-delimiter=' ' -f1,3,4 /etc/passwd | sort -k2n | xargs -n3 sh -c 'echo useradd -u $1 -g $2 $0'
generates a list of all groups with command to create them in another machine
cut -d: --output-delimiter=' ' -f1,3 /etc/group | sort -k2n | xargs -n2 sh -c 'echo groupadd -g $1 $0'
using ping
while sleep 3;do ping -a -c 1 8.8.8.8; done
using beep
while sleep 3;do ping -c 1 8.8.8.8 && beep; done
using mplayer
while sleep 3;do ping -c 1 8.8.8.8 && mplayer file.mp3; done
create 1 column
seq 10
create 2 columns
printf %s\\n ID-{1..3}' '{10..25..5}
create 3 columns
printf %s\\n ID-{1..3}' '{a,b,c}' '{10..20..5}
create 4 or more columns
seq 12 | pr -4t | column -t
using echo and read
echo -e Press any key to "\e[32;1mconfirm\e[m" or Ctrl-C to Canncel
read
echo some key was pressed!
using only read
read -p 'Press any key to confirm or Ctrl-C to Cancel'
echo some key was pressed!
confirm action with yes or no using only read
read -p 'Choose one: Y/n ' y_n && test "$y_n" == "" -o "$y_n" == "Y" -o "$y_n" == "y" && echo typed: y || echo n or other key was pressed!
read input from the user and only continue if user press y
read -p 'Do you want to continue? N/y ' y_n && test "$( tr '[:upper:]' '[:lower:]' <<<$y_n )" != "y" && exit
if current time is lower than 15 get user input to confirm if him want to continue
test $( date +%M ) -le 15 && read -p 'Current time is: 0-15. Do you want to continue? N/y ' y_n && test "$( tr '[:upper:]' '[:lower:]' <<<$y_n )" != "y" && exit
rename file to filenew
mv file{,new}
make a copy of file as file.backup
cp file{,.backup}
bash remove first char
MYVAR=abcd; echo ${MYVAR:1}
bash remove last char
MYVAR=abcd; echo ${MYVAR::-1}
sed remove first char
echo abcd | sed 's/^.//'
sed remove last char
echo abcd | sed 's/.$//'
awk remove first char
echo abcd | awk '{ print substr( $0, 2 ) }'
awk remove last char
echo abcd | awk '{ print substr( $0, 1, length($0)-1 ) }'
Remove first directory from path filename
D=/var/spool/cron; echo ${D#/*/}
format output as a table
echo -e name age"\n"nickollas 30 | column -t
format first column with 2 places and third column with comma and 7 places
ls -ld /etc/h* | awk '{ printf( "%2s %s %7d %s\n", $(NF-2),$(NF-3),$(NF-4),$NF ) }'
grep column 3 of file
grep -P '^\S+(\s+\K\S+){2}' /etc/hosts
grep column 3 of file keeping doesn’t match content
grep --color -zP '^\S+(\s+\K\S+){2}' /etc/hosts
if you want to filter column that have a regex/word
awk '$NF ~ /bash/' /etc/passwd
read -p 'Enter your name: ' name
echo you typed: $name
execute multiple commands if a regex test fail
grep newroot /etc/passwd || { echo cmd1; echo cmd2; }
negation regex inside test condition
[[ ! $1 =~ [0-9] ]] && echo Param isn't a number && exit
output file content without line starting with root string
grep ^[^root] /etc/passwd
Test if $1 has a value – way 1
[ -n "$1" ] && echo $1 || echo param required
Test if $1 has a value – way 2
: ${1?'param required'}
echo $1
Test if any param was passed to script
[ $# -ne 0 ] && echo total of params: $# || echo param is empty
Concatenates two commands without space between them
echo $(hostname -s)$(whoami)
Concatenates two commands with space between them
echo $(TZ=America/New_York date +%H) $(date +%H)
Get result of calculation math of two commands – get timezone difference
echo $[$(TZ=America/New_York date +%k)-$(date +%k)]
Concatenates one command and one string without space between them – way 1
hostname | cat - <(echo xyz) | sed ':l N; s/\n//g; t l'
Concatenates one command and one string without space between them – way 2
hostname | cat - <(echo xyz) | tr -s "\n" ' ' | sed 's/ //'
Concatenates one command and one string without space between them – way 3
hostname | cat - <(echo xyz) | tr -d "\n"
bash way
(head -1; tail -1) < /etc/passwd
sed way
sed -n '1p; $p' /etc/passwd
awk way
awk 'NR==1; END { print }' /etc/passwd
tar -rvf existing_tar_file.tar new_file_to_add
* This process dosen’t work with .tar.bz2 extension
list of tar content
tar -tf files.tar.bz2 usr/bin/ls usr/bin/cat
decompress only files removing full path
tar -xjf files.tar.bz2 --strip-components 2 -C /tmp
decompress tar files removing 1 level directory
tar -xjf files.tar.bz2 --strip-components 1 -C /tmp
using bash
MYVAR=abcd; echo ${MYVAR:1:-1}
using sed
echo abcd | sed s'/^.\(.*\).$/\1/'
using awk
echo abcd | awk '{ print substr( $0, 2, length($0)-2 ) }'
while read i;do egrep ^$i$ file2 && echo $i >> exist.out || echo $i >> non_exist.out; done < file1
It will jumps the 2, printing only 1 3 4 5
for i in {1..5};do
[ $i -eq 2 ] && continue
echo $i
done
Add comma for thousand number, printf version
printf "%'d\n" 1234567
Add comma for thousand number, numfmt version*
numfmt --group 1234567 LANG=pt_BR.utf8 numfmt --group 1234567 LANG=en_AG numfmt --group 1234567
*To list locales: localectl list-locales
Add comma for thousand number, awk version
awk '{ printf( "%'"'"'d\n",$0 ) }' <<< 1234567
Add dot for 2 last numbers with sed
sed 's/\B[0-9]\{2\}\>/.&/' <<< 1234567
Add dot for 2 last numbers only on 2 column with sed
sed 's/\B[0-9]\{2\}\>/.&/2' <<< '123456 123456'
Replace comma by dot and dot by comma with sed
sed -e 's/,/./g' -e 's/\./,/3' <<<'1,234,567.89'
delete first column
cut -d: -f1 --complement /etc/passwd
delete last column
rev /etc/passwd | cut -d: -f1 --complement | rev
delete columns one and four
cut -d: -f1,4 --complement /etc/passwd
delete range of columns 2 to 4
cut -d: -f2-4 --complement /etc/passwd
print all params passed to bash script
echo $*
get last param passed to bash script
echo ${@: -1}
remove first param passed to bash script
echo ${*:1-$#}
remove two first params passed to bash script
echo ${*:2-$#}
remove last param passed to bash script
echo ${*:0:$#}
remove two last params passed to bash script
echo ${*:0:$#-1}
#!/bin/bash ( echo 'HELO' sleep 1 echo 'mail from: insertSourceMailHere@gmail.com' sleep 1 echo 'rcpt to: insertDestinationMailHere@gmail.com' echo 'rcpt to: insertDestinationMail2Here@gmail.com' sleep 1 echo 'data' sleep 1 echo "Subject: message from server: $(hostname) Content of file: /etc/passwd $(cat /etc/passwd) " echo '.' sleep 1 echo 'quit' ) | telnet localhost smtp
#!/bin/bash for i in $@; do echo $i done
get return values of all pipes
ls /etc | grep passwd | j3RJ; echo ${PIPESTATUS[*]}
get return value of first pipe
ls /etc | grep passwd | j3RJ; echo ${PIPESTATUS[0]}
get return value of last pipe
ls /etc | grep passwd | j3RJ; echo ${PIPESTATUS[-1]}
find files between two specified dates
find / -type f -newermt 2017-07-01 ! -newermt 2017-08-30 -ls
find files created between begin of current month and the current day
find / -type f -newermt $( date +%Y-%m-01 ) ! -newermt $( date +%Y-%m-%d ) -ls
find .php files modified on the last 30 days
find / -type f -regex .*php$ -mtime -30 -exec ls -lt {} +
find files between 1 month ago and 3 days ago that ends with numbers in name
find / -type f -regex .*[0-9]$ -newermt $( date -d '-1 month' +%Y-%m-%d ) ! -newermt $( date -d '-3 day' +%Y-%m-%d ) -exec ls -la {} +
compress files created between two specified dates
find / -type f -newermt $( date -d '-1 month' +%Y-%m-%d ) ! -newermt $( date -d '-3 day' +%Y-%m-%d ) -exec gzip -f {} +
Append string to filename
cp /etc/fstab{,.backup}
Append string and date to filename
cp /etc/fstab{,.backup.$(date +%Y%m%d)}
number of days until / partition is full
df -m | awk '/\/$/ { printf "%d\n", $4-$3/1024 }' > /tmp/free_space; find / -path /proc -prune -o -type f -ctime -1 -ls | awk '{n+=$7} END {printf "%d\n", n/1024/1024}' | awk 'BEGIN{ORS=" "} {print}' /tmp/free_space /dev/stdin | awk '{printf "%d%s\n", $1/$2," days until partition / is full"}'; rm -f /tmp/free_space
excludes /proc dir in find search
find / -path /proc -prune -o -type f -ls
excludes /proc and /usr dirs from find search
find / -path /proc -prune -o -path /usr -prune -o -type f -ls
excludes any subdirectory on find search
find /tmp -maxdepth 1 -type f -ls
join pipe output with file content
grep ^root /etc/passwd | awk 'ORS=" " {print} END {print "\n"}' /dev/stdin /proc/uptime
Delete current rule if this exist
iptables -nvL FORWARD --line-number | grep 'yourDomainHere.ddns.net' | awk '{ print $1 }' | xargs -I% iptables -D INPUT %
Add a new rule with the current IP
getent ahostsv4 yourDomainHere.ddns.net | tail -1 | awk '/^[0-9]/ { print $1 }' | xargs -I% iptables -I FORWARD -s %/32 -p tcp -m multiport --dports 20,21 -m comment --comment "yourDomainHere.ddns.net" -j ACCEPT
Shell Script to change IP on firewall and log this
#!/bin/bash
# Allow FTP on Firewall for dynamic IPs
HOSTS=( insertHereHost1.ddns.net insertHereHost2.ddns.net )
for i in ${HOSTS[*]}; do
/sbin/iptables -nvL INPUT | grep $i > /dev/null
if [ $? -ne 0 ];then
echo '[' $( date '+%d/%m/%Y %R' ) '] There was no entry for HOST' $i 'on firewall' | tee -a /var/log/$( basename $0 ).log
/sbin/iptables -I INPUT -s $i/32 -p tcp -m multiport --dports 20,21 -m comment --comment "$i" -j ACCEPT
elif [ $( getent ahostsv4 $i | tail -1 | awk '/^[0-9]/ { print $1 }' ) != $( /sbin/iptables -nvL INPUT | grep $i | awk '{ print $8 }' ) ]; then
echo '[' $( date '+%d/%m/%Y %R' ) '] HOST' $i 'IP changed:' $( getent ahostsv4 $i | tail -1 | awk '/^[0-9]/ { print $1 }' ) 'traffic with old IP:' $( /sbin/iptables -nvL INPUT | grep $i | awk '{ print $1 }' ) | tee -a /var/log/$( basename $0 ).log
/sbin/iptables -nvL INPUT --line-number | grep $i | awk '{ print $1 }' | xargs -I% /sbin/iptables -D INPUT %
/sbin/iptables -I INPUT -s $i/32 -p tcp -m multiport --dports 20,21 -m comment --comment "$i" -j ACCEPT
fi
done
using content of echo command
echo ' pwd hostname ' | sh
using content of cat command
cat <<EOF | sh pwd hostname EOF
bash
convert unixtimestamp to date with date command
date -d @1601661600
awk
convert unixtimestamp to date with awk
awk '{ print strftime("%c", $1) }' <<<1601661600
convert unixtimestamp to date formating it
awk '{ print strftime("%Y-%m-%d %H:%M:%S", $1) }' <<<1601661600
convert unixtimestamp to date appending the output to end of line
awk '{ "date -d @1601661600" |& getline $(NF+1); print }' <<<'a b c'
perl
convert unixtimestamp to date with perl
perl -le 'print scalar localtime 1601661600'
create N files (suffix x0[1-N]piece) each with max 700 lines
split --numeric-suffixes=1 --additional-suffix=piece -l 700 fileTosplit
| fileA | fileB |
|---|---|
| fileA_line1 | fileB_line1 |
| fileA_line2 | fileB_line2 |
| fileA_line3 | fileB_line3 |
merge two files separate by TAB default field separator
paste fileA fileB
merge two files without field separator
paste -d'\0' fileA fileB
merge two files separate by New Line
paste -d'\n' fileA fileB
you should redirect the condition error test to /dev/null
sample 1, until $i LessThan 5
until [ $i -eq 5 2> /dev/null ]; do (( i++ )) echo $i done
sample 2, until $LATENCY_PING LessThan 50
until [ $LATENCY_PING -lt 50 2> /dev/null ]; do
LATENCY_PING=$( ping -i 0.5 -c 4 8.8.8.8 | awk -F'/' 'END { printf "%d\n", $5 }' )
echo -e "average latency ping:\e[33;1m $LATENCY_PING\e[m"
done
if file has regex ‘^ wpa. * default.c’ execute block followed by && if not execute block followed by ||
grep -q '^wpa.*default.c' /etc/network/interfaces && sed -i "/^wpa-conf/s/^/#/; /default_.conf$/s/^#// w/dev/stdout" /etc/network/interfaces || sed -i "/^wpa-conf/s/^/#/; /default.conf$/s/^#// w/dev/stdout" /etc/network/interfaces
bash
convert string to upper case
echo aBcD | tr '[:lower:]' '[:upper:]'
convert string to lower case
echo aBcD | tr '[:upper:]' '[:lower:]'
awk
convert only first line to uppercase using awk
echo -e "ab cd\nef gh" | awk 'NR==1 {print toupper($0)} NR>1'
convert two last chars to lowercase using awk
echo -e "a JAN\nb DEC" | awk '{ print substr($2,1,1) tolower( substr($2,2) ) }'
convert two last chars to lowercase of column 2 using awk
awk '{ $2=substr($2,1,1) tolower( substr($2,2) ); print }' <<<'JANM DEC'
convert range of characteres of column 2 to lowercase using awk
echo -e "A JANUARY\nB DECEMBER" | awk '{ $2=substr($2,1,1) tolower( substr($2,2,2) ) substr($2,4); print }'
sed
convert only first line to uppercase using sed
echo -e "ab cd\nef gh" | sed '1s/.*/\U&/'
convert two last chars to lowercase using sed
echo -e "JAN\nDEC" | sed -r 's/.{2}$/\L&/'
convert first char to uppercase using sed
echo -e "jan\ndec" | sed -r 's/.{1}/\U&\E/'
#!/bin/bash
# block countries with iptables based on list with range of ips
# download list with range of countries ips
rm -rf /tmp/all-zones*; wget -nc http://www.ipdeny.com/ipblocks/data/countries/all-zones.tar.gz -P /tmp
mkdir /tmp/all-zones; tar -xzvf /tmp/all-zones.tar.gz -C $_
COUNTRIES_ISO_CODE_LIST=(af cu mo)
for country_iso_code in ${COUNTRIES_ISO_CODE_LIST[*]}; do
for country_ip in $( cat /tmp/all-zones/$country_iso_code.zone ); do
echo creating rules to $country_iso_code $country_ip
/sbin/iptables -A INPUT -s $country_ip -m comment --comment "rule to $( echo $country_iso_code | tr '[:lower:]' '[:upper:]' ) country" -j DROP
done
done
#!/bin/bash
# reboot if ping test fails for 3 times
unset IVAR
for i in {1..3}; do
ping -c 1 8.8.8.8 > /dev/null || ((IVAR++))
done
if [ "$IVAR" == 3 ]; then
echo '[' $( date '+%d/%m/%Y %R' ) '] reboot' | tee -a /var/log/$( basename $0 ).log
/sbin/init 6
fi
echo 'www.google.com' | rev | cut -d'.' -f1 | rev
sorting by column 1
echo -e "2 44 B\n1 33 A\n4 22 A\n3 11 B" | sort
sorting by column 3
echo -e "2 44 B\n1 33 A\n4 22 A\n3 11 B" | sort -k3
sorting by column 3 and 2
echo -e "2 44 B\n1 33 A\n4 22 A\n3 11 B" | sort -k3 -k2
sorting by second field using comma(csv) as field separator
echo -e 'x,2\nb,1\nm,3' | sort -t, -k2
sorting by fifth field with numeric order
df -h | sort -k5n
sorting by fifth field with reverse numeric order
df -h | sort -k5nr
sort lines by length
awk '{ print length, $0 }' /etc/passwd | sort
count a chars in file order by last column
awk -F'a' '{print $0,NF-1}' /etc/passwd | rev | sort | rev
sort in numeric order
echo 1 2 10 | tr ' ' '\n' | sort -V
extract some columns of /etc/passwd, sort by second column, put header and format output
cut -d: -f1,3,6,7 /etc/passwd | sort -t: -k2n | cat <(echo user:id:home:shell) - | column -s: -t
list files and sort by size
ls -lh | sort -k5,5hr
list files and sort by date and size
ls -lh | sort -k6,8 -k5,5hr
basic test variables
| && | Execute if true |
| || | Execute if false |
| $? | exit status of last command executed. if true return 0 else retun > than 0 |
| -eq | equal |
| -ne | not equal |
Different ways to suppress grep output
grep -q root /etc/passwd && echo true || echo false
grep root /etc/passwd > /dev/null && echo true || echo false
test using logical operators && ||
echo 'abc' | grep -q a && echo true || echo false
test using test command
test -z "$( mount | grep sdb )" && mount /dev/sdb /mnt
echo 'abc' | grep -q a; test $? -eq 0 && echo true || echo false
test command directly inside of if
if grep -q root /etc/passwd; then echo true; else echo false; fi
test special var $? inside of if
echo 'abc' | grep -q a; if [ $? -eq 0 ]; then echo true; else echo false; fi
random test using if else command
if [ "$(shuf -i 1-2 -n 1 | grep 1)" == "1" ];then echo 1; else echo 2; fi
if a command not exist show a error message and exit
! type ssh &> /dev/null && echo erro install ssh && exit
define variable based on output command
whoami | grep -q nickollas && user=nick || user=other
multiple tests string and a number in the same time
test str == str -a 7 -eq 7 && echo ok
execute multiple commands if the execution of previous command failed
curl --connect-timeout 0.01 google.com || { echo 1; echo 2; }
execute multiple commands in a negation test with pipe
test ! "$( cat /etc/passwd | grep rootX )" && echo cmd1 && echo cmd2
! [[ $( export | grep DBUS ) ]] && echo true || echo false
execute multiple commands if a regex test fail
grep newroot /etc/passwd || { echo cmd1; exit; }
ouput with new line
echo -e "a\nb\nc"
after put the command inside the variable, you lose the new line
VAR=$(echo -e "a\nb\nc"); echo $VAR
put output in quotes ” to prevent losing new lines
VAR=$(echo -e "a\nb\nc"); echo "$VAR"
block multiple URLs
#!/bin/bash
# analyzes apache log and blocking ip than access url 3 times in 2 minutes
# block only access to url not to all urls
exec 2>> /var/log/$( basename $0 ).log.$( date +%F_%H%M )
# you can use | to separate multiple strings to block
URL_STRING_TO_BLOCK="insertPartOfUrl1Here|insertPartOfUrl2Here"
RANGE_TIME=$( date '+%H:%M' )
# make array with 2 past minutes using | separator. used as a separator field in the egrep regex
for i in {1..2};do
RANGE_TIME=${RANGE_TIME}\|$(date -d "-$i min" '+%H:%M:')
done
# list of ips with hits to the same url
IPs_LIST=$( egrep "$(date '+%d\/%b\/%Y:')($RANGE_TIME).*($URL_STRING_TO_BLOCK)" /var/log/httpd/access.log )
# group ips by number of hits to the same url
GROUP_BY_IPs=$( echo "$IPs_LIST" | cut -d' ' -f1 | sort | uniq -c )
# create iptables rule to block each ip that access url 3 times in N minutes than not starts with 192.168.0 range
for i in $( echo "$GROUP_BY_IPs" | awk '$1 >= 3 && $2 !~ "^192.168.0" {print $2}' );do
/sbin/iptables -nvL INPUT | grep " $i .*many" > /dev/null
if [ $? -ne 0 ]; then
for j in $( echo $URL_STRING_TO_BLOCK | sed 's/|/\n/g' ); do
echo $( date '+%d/%m/%Y %R' ) "blocking ip: $i than access part of URL: $j" | tee -a /var/log/$( basename $0 ).log
/sbin/iptables -A INPUT -s $i -p tcp -m tcp --dport 80 -m string --string "$j" --algo bm --to 65535 -m comment --comment "block IP than access this URL many times" -j DROP
done
fi
done
one liner code to blocking url access 3 times in 1 minute. (block only 1 url)
URL='insertUrlHere'; RANGE_TIME=$(date '+%H:%M'); for i in {1..1};do RANGE_TIME=${RANGE_TIME}\|$(date -d "-$i min" '+%H:%M:'); done; egrep "$(date '+%d\/%b\/%Y:')($RANGE_TIME).*$URL" /var/log/httpd/access.log | cut -d' ' -f1 | sort | uniq -c | awk '$1 >= 3 && $2 !~ "^192.168" {print $2}' | xargs -I% iptables -A INPUT -s % -p tcp -m tcp --dport 80 -m string --string "$URL" --algo bm --to 65535 -j DROP
Append/Add value to variable inside loop in shell script, make variable containing range numbers
unset RANGE_NUM; for i in {1..5};do RANGE_NUM=${RANGE_NUM}$i; echo $RANGE_NUM; done
variable containing range with the last 5 minutes
LASTMINS=$(date '+%H:%M'); for i in {1..4};do LASTMINS=${LASTMINS}\|$(date -d "-$i min" '+%H:%M'); echo $LASTMINS; done
variable containing range with the last 7 days / 1 week
DAYS=$(date +%d/%b); for i in {1..6};do DAYS=${DAYS}\|$(date -d "-$i day" +%d/%b ); echo $DAYS; done
sed
insert header
sed '1 i header' /etc/passwd
insert footer
sed '$ a footer' /etc/passwd
insert header and footer
sed -e '1 i header' -e '$ a footer' /etc/passwd
put header to uppercase and print all other lines
sed '1s/.*/\U&/' /etc/passwd
awk
insert header
awk 'BEGIN { print "###\nheader\n###" } { print }' /etc/passwd
insert footer
awk '{ print } END { print "###\nfooter\n###" }' /etc/passwd
insert header and footer
awk 'BEGIN { print "###\nheader\n###" } { print } END { print "###\nfooter\n###" }' /etc/passwd
insert header before output match
awk 'BEGIN { print "###\nheader\n###" } /www-data/ { print }' /etc/passwd
insert footer before output match
awk '/www-data/ { print } END { print "###\nfooter\n###" }' /etc/passwd
insert header and footer between output match
awk 'BEGIN { print "###\nheader\n###" } /www-data/ { print } END { print "###\nfooter\n###" }' /etc/passwd
put header to uppercase and print all other lines
awk 'NR==1 {print toupper($0)} NR>1' /etc/passwd
bash
insert header
echo header | cat - /etc/passwd
insert footer
echo footer | cat /etc/passwd -
insert header and footer
echo header | cat - /etc/passwd <(echo footer)
put header to uppercase and print all other lines
head -1 /etc/passwd | tr '[:lower:]' '[:upper:]' | cat - <(tail -n +2 /etc/passwd)
echo -e 'a\nb\nc\na' | sort | uniq -D
substitute whitespace by commas to generate csv output
echo a b c | tr -s ' ' ','
find way
search for tty string in all .sh files in /etc dir
find /etc -name '*.sh' -exec grep -i tty {} + 2>&-
search for database string in all .php files in local dir
find /etc -name '*.php' -exec grep -i database {} + 2>&-
search for log4j string in all .java files in local dir
find /etc -name '*.java' -exec grep -i log4j {} + 2>&-
shell script way
search for tty string in all .sh files in /etc dir
for i in $( find /etc -name '*.sh' );do OUT=$( grep -i 'tty' $i ); test "$OUT" && echo -e "\n$i\n$OUT"; done 2>&-
search for database string in all .php files in local dir
for i in $( find . -name '*.php' );do OUT=$( grep -i 'database' $i ); test "$OUT" && echo -e "\n$i\n$OUT"; done 2>&-
search for log4j string in all .java files in local dir
for i in $( find . -name '*.java' );do OUT=$( grep -i 'log4j' $i ); test "$OUT" && echo -e "\n$i\n$OUT"; done 2>&-
VAR=0
for i in {1..3}; do
((VAR++))
echo VAR: $VAR
done
#!/bin/bash
# Import Remote MySQL Database
exec 2>> /var/log/$( basename $0 ).log.$( date +%F_%H%M )
### LOCAL MYSQL INFO ###
LOCAL_MYSQL_USER='root'
LOCAL_MYSQL_PASSWD='localMysqlPasswordHere'
### REMOTE MYSQL INFO ###
REMOTE_MYSQL_HOST='192.168.0.7'
REMOTE_MYSQL_USER='root'
REMOTE_MYSQL_PASSWD='remoteMysqlPasswordHere'
REMOTE_DATABASES_TO_EXPORT=(dbName1Here dbName2Here)
echo -e "\n[ $( date '+%d/%m/%Y %R' ) ] START SCRIPT" | tee -a /var/log/$( basename $0 ).log
for i in ${REMOTE_DATABASES_TO_EXPORT[*]}; do
echo -e "\n[ $( date '+%d/%m/%Y %R' ) ] EXPORT REMOTE DB $i" | tee -a /var/log/$( basename $0 ).log
mysqldump -h $REMOTE_MYSQL_HOST -u $REMOTE_MYSQL_USER -p$REMOTE_MYSQL_PASSWD --databases $i --single-transaction > '/tmp/'$i'.sql'
echo "START IMPORT $i [ $( date '+%d/%m/%Y %R' ) ]" | tee -a /var/log/$( basename $0 ).log
mysql -u $LOCAL_MYSQL_USER -p$LOCAL_MYSQL_PASSWD -e "DROP DATABASE $i"
mysql -u $LOCAL_MYSQL_USER -p$LOCAL_MYSQL_PASSWD < '/tmp/'$i'.sql'
echo "END IMPORT $i [ $( date '+%d/%m/%Y %H:%M' ) ]" | tee -a /var/log/$( basename $0 ).log
# Delete dump file
rm -f '/tmp/'$i'.sql'
done
echo "[ $( date '+%d/%m/%Y %R' ) ] END SCRIPT" | tee -a /var/log/$( basename $0 ).log
1 – Create the file that will be executed after ssh login
vi /usr/local/bin/screen_script.sh
#!/bin/bash
# Connect in screen session if you have one open
SESSION_ID=$( screen -ls | grep -P '\t' | cut -d. -f1 | tr -d '\t' )
if [ -z $SESSION_ID ]; then
text='Started new session with screen!'; printf "%*s\n" "$(((${#text}+$(tput cols))/2))" "$text"
screen -d -R -S screenNameHere
exit
else
text='Connected in existing screen session!'; printf "%*s\n" "$(((${#text}+$(tput cols))/2))" "$text"
screen -wipe
screen -r $SESSION_ID
screen -d -R -S screenNameHere
exit
fi
*if you need execute command inside screen use
screen -d -R -S screenNameHere sh -c "clear; telnet 192.168.0.7; logout"
Limiting 1 login per user (optional step)
echo 'userNameHere hard maxlogins 1' >> /etc/security/limits.d/users.conf
connect on remote screen server and execute script after ssh login
ssh IpOfServerWithScreenHere -t 'screen_script.sh'
way 1
echo abc | printf "%*s\n" $(( $( tput cols )/2 )) "$(cat /dev/stdin)"
way 2
echo abc | printf "%*s\n" $((($COLUMNS)/2)) "$(cat /dev/stdin)"
way 3
text=abc; printf "%*s\n" $(((${#text}+$COLUMNS)/2)) "$text"
one liner to show erros in last 10 minutes
LAST10MIN=$(date -d '-20 min' '+%M'); grep "$(date '+%d/%b/%Y:%H:[$LAST10MIN-%M]').*erro" /var/log/apache2/error.log
#!/bin/bash
# Search for apache error in the last X minutes
MINUTES=30
DATA=$( date '+%d/%m/%Y %R' )
LAST_ERROR_LOG=$( grep 'erro' /var/log/apache2/error.log | tail -n 1 )
CUT_TIME_ERRO_LOG=$( echo $LAST_ERROR_LOG | egrep -o '.*([0-9]){2}:[0-9]{2}|.*([0-9]){2}:[0-9]{2} [0-9]{4}' | tr -d [ )
UNIX_TIMESTAMP_CUT_TIME_ERROR_LOG=$( date -d "$CUT_TIME_ERRO_LOG" +%s )
if [ ${#LAST_ERROR_LOG} -gt 0 ] && [ $(( `date +%s` - $UNIX_TIMESTAMP_CUT_TIME_ERROR_LOG )) -lt $(( $MINUTES*60 )) ]; then
echo -e "[$DATA] \e[31;1m[ ERROR FOUND ]\e[m $LAST_ERROR_LOG" | tee -a /var/log/$( basename $0 ).log
fi
cut last time error in kernel log
awk '/erro/ {print $1,$2,$3}' /var/log/messages | tail -1
cut last time error in apache log
awk '/erro/ {print $1,$2,$3,$4,$5}' /var/log/httpd/error_log | tail -1 | tr -d []
#!/bin/bash
# cut last time error in both kernel and apache log
for i in /var/log/messages /var/log/httpd/error_log; do
grep 'erro' $i | tail -1 | egrep -o '.*([0-9]){2}:[0-9]{2}|.*([0-9]){2}:[0-9]{2} [0-9]{4}' | tr -d [
done
bash
delete first line
nl /etc/passwd | tail -n +2
delete last line
nl /etc/passwd | head -n-1
delete first and last line
nl /etc/passwd | tail -n +2 | head -n-1
delete penultimate and last line
nl /etc/passwd | head -n-2
delete fist, second, penultimate and last line
nl /etc/passwd | tail -n +3 | head -n-2
sed
delete first line
nl /etc/passwd | sed 1d
delete last line
nl /etc/passwd | sed '$d'
delete first and last line
nl /etc/passwd | sed '1d; $d'
delete penultimate and last line
nl /etc/passwd | sed -n 'N;$!P;D'
delete fist, second, penultimate and last line
nl /etc/passwd | sed -n '1,2d; N;$!P;D'
awk
delete first line
nl /etc/passwd | awk 'NR>1'
delete last line
nl /etc/passwd | awk 'NR>1 { print last } { last=$0 }'
delete first and last line
nl /etc/passwd | awk 'NR>2 { print last } { last=$0 }'
syntax of mail command
echo body_msg | mail -s "subject" "mail@domain.com"
script to send mail if any file exceeds maximum allowed size
#!/bin/bash
# Send mail if any file exceeds maximum allowed size
exec 2>> /var/log/$( basename $0 ).log
FILES=(/etc/hosts /etc/passwd)
MAXIMUM_FILE_SIZE_BYTES=1024
EMAIL=mail@domain.com
for i in ${FILES[*]}; do
if [ $( ls -l $i | awk '{print $5}' ) -gt $MAXIMUM_FILE_SIZE_BYTES ]; then
echo -e "[ $( date '+%F %R' ) ] [ \e[31;1mERRO\e[m ] $i higher than allowed: $( ls -l $i | awk '{print $5}' )" bytes | tee -a /var/log/$( basename $0 ).log
echo | mail -s "$( hostname -s ) $i higher than allowed" $EMAIL
fi
done
the same one-liner script
ls -l /etc/{hosts,passwd} | awk '$5 > 1024 { print $NF, "higher than allowed" }' | xargs -I% mail -s "$( hostname -s ) %" "mail@domain.com"
time of last: access, modification, size and others
stat /etc/passwd
get name, permissions and size of file
stat -c '%n %A %s' /etc/passwd
Using stat
stat -c %s /etc/passwd
Using find
find /etc/passwd -printf %s
using ls + awk
ls -l /etc/passwd | awk '{print $5}'
1 – Install shc package
apt-get install shc
2 – Create the shell script file
vi file.sh
#!/bin/bash
for i in {1..3}; do echo $i: $RANDOM; done
3 – Encrypt shell script file using shc
shc -f file.sh Output generated files: file.sh.x file.sh.x.c
4 – Execute the encrypted shell script
./file.sh.x
5 – Specifying expiration date for your shell script ( dd/mm/yyyy format )
shc -e 30/07/2017 -f file.sh
6 – Add custom expiration message for your shell script
shc -e 30/07/2017 -m contact 'my@mail.com' -f file.sh
sum bash linux
echo $(( 3 + 4 ))
division integer
echo $(( 11 / 5 ))
rest of division
echo $(( 11 % 5 ))
get hexadecimal of ã latin letter
echo -en "ã" | hexdump -e '/1 "%X "'; echo
print hexadecimal char with echo
echo -e '\xC3\xA3'
get hexadecimal of nTABc latin letter
echo -en "n\tc" | hexdump -e '/1 "%X "'; echo
print hexadecimal string with echo
echo -e '\x6E\x9\x63'
UNICODE TABLE REFERENCE: http://unicode-table.com
Delete new line of input
grep root /etc/passwd | tr -d '\n'
Delete new line and o char of input
grep root /etc/passwd | tr -d '\n,o'
get all mac address and ip from your network gateway order by ip
nmap -sP $( ip route | grep '^default' | egrep -o '([0-9]{1,3}\.){3}' )0/24; arp -a | sort -n -t . -k 4 | awk '{print "mac:", $4, "#"$2}'
get all ip and mac address from your network gateway
nmap -sn -PO $( ip route | grep default | egrep -o '([0-9]{1,3}\.){3}' )0/24 | grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}|[[:xdigit:]:]{6,}' | awk 'BEGIN{ m=0 } { if ( $1 ~ /^[[:digit:].]{3,}/ ) m++; print m, $0 }' | awk '{ a[$1] = a[$1] FS substr( $0, index( $0,$2 ) ) } END{ for( i in a ) print i a[i] }'
get all ip and mac address from a network range with ssh open
nmap -sS -p 22 --open $( ip route | grep default | egrep -o '([0-9]{1,3}\.){3}' )0/24 | grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}|[[:xdigit:]:]{6,}' | awk 'BEGIN{ m=0 } { if ( $1 ~ /^[[:digit:].]{3,}/ ) m++; print m, $0 }' | awk '{ a[$1] = a[$1] FS substr( $0, index( $0,$2 ) ) } END{ for( i in a ) print i a[i] }'
A full script sample to get all IPs and MACs of an entire network:
https://github.com/nickollasc/get_all_ips_and_macs_of_network
nmap -sP 192.168.0.0/24; arp -a | sort -n -t . -k 4
loop 1 until 10
for i in {1..10}; do echo $i; done
list all files in directory
for i in /etc/*; do echo $i; done
loop strings
for i in /etc /opt; do echo $i; done
create one file with 1 million of entries
for i in {1..1000000}; do echo $i; done > file
#!/bin/bash
# Remove all mysql databases
echo 'Enter MySQL user'
read MYSQL_USER
echo 'Enter MySQL password'
stty -echo
read MYSQL_PASSWD
stty echo
DATABASES=$( mysql -u $MYSQL_USER -p$MYSQL_PASSWD -e "SHOW DATABASES;" | tr -d "| " | grep -v Database )
for i in ${DATABASES[*]}; do
if [ $i != 'mysql' ] && [ $i != 'information_schema' ] && [ $i != 'performance_schema' ]; then
echo $( date +%R ) drop database: $i | tee -a /var/log/$( basename $0 ).log
time mysql -u $MYSQL_USER -p$MYSQL_PASSWD -e "DROP DATABASE $i"
fi
done
test $# -eq 0 && echo try run: $0 -n param / -s param / -d / -h
while getopts n:s:dh OPT; do
case $OPT in
n | number ) echo number $OPTARG ;;
s | string ) echo string $OPTARG ;;
d | date ) date +%F ;;
h | help ) echo $0 -n param / -s param / -d / -h && exit ;;
esac
done
using only bash
sorting by column two maintaining first line as header
tail -n +2 file.csv | sort -t, -k2n | cat <(head -1 file.csv) -
sorting by column two maintaining first line as header and format output as tsv
tail -n +2 file.csv | sort -t, -k2n | cat <(head -1 file.csv) - | column -s, -t
using sed + bash
sorting by column two maintaining first line as header
sed -n '2,$p' file.csv | sort -t, -k2n | cat <(sed 1q file.csv) -
VAR=abc; echo ${VAR}def
for i in {1..5}; do echo x${i}x; done
#!/bin/bash
# Recalculate vmware thin disk size
exec 2>> /var/log/$( basename $0 ).log.$( date +%F_%H%M )
# Creates zero-block file in all free disk space
for i in $(df | grep '^/dev/' | awk '{print $6}'); do
if [ $i == '/' ];then
echo 'creating a file of blocks zeroed in:' ${i}zerofile
dd if=/dev/zero of=${i}zerofile bs=4096; rm -f ${i}zerofile
else
echo 'creating a file of blocks zeroed in:' $i/zerofile
dd if=/dev/zero of=$i/zerofile bs=4096; rm -f $i/zerofile
fi
done
After run the command with the machine turned off in ESXi Host
vmkfstools --punchzero /vmfs/volumes/path-to-disk.vmdk
APACHE PROCESSES THAT CPU OR MEMORY IS LARGER THAN 0
ps -A --sort -rss -o pid,comm,pcpu,pmem,lstart | grep httpd | awk '($3>0) || ($4>0)'
TOTAL CPU USED BY THE APACHE PROCESS
ps -A --sort -rss -o pid,comm,pcpu,pmem | grep httpd | awk '{n+=$3} END {print $2" total cpu: "n"%"}'
TOTAL MEMORY USED BY THE APACHE PROCESS
ps -A --sort -rss -o pid,comm,pcpu,pmem | grep httpd | awk '{n+=$4} END {print $2" total mem: "n"%"}'
TOTAL CPU AND MEMORY USED BY THE APACHE PROCESS
ps -A --sort -rss -o pid,comm,pcpu,pmem | grep httpd | awk '{x+=$3} {y+=$4} END {print $2" total cpu: "x"% mem: "y"%"}'
You can use server-status module to monitoring apache. Options are:
lynx http://localhost/server-status lynx http://localhost/server-status?auto lynx http://localhost/server-status?refresh=5 lynx http://localhost/server-status?auto&refresh=5
documentacoes.wordpess.com 