Monday, November 23, 2015

Shell (1): copy/count/cut/find/paste/sort/split: Usages.........

#########################
 cat
#For help with this command
cat --help
#Read the file
cat file
#Merge individual files into one
cat *_file >>merged_file
#Prints the content of file with line number (-n option)
cat -n file
#Change output delimiter 
cat file| cut -d':'  -s -f1,3 --output-delimiter='#'
cat file| cut -d':'  -s -f1,3 --output-delimiter=$'\n'
#Extract only field 2 and from the file
cat file|cut -d' ' -f2,3-
#Show contents of many files
cat file1 file2 file3
#Create  a file
cat > file
cat file |more
#Display line numbers in a file
cat -n file
#Display $ at the end of file (useful to find empty lines)
cat -e file
#Display multiple files
cat file1; cat file2
cat file1 file2
#Creates a file and appends content of another file to that
cat file >> file2
#To extract the lines common to all files
cat file1 file2 file3 | awk '{a[$0]++} END{for (i in a) if (a[i]==3) print i}'
#########################
cd
#Change to home
cd
cd~
#Move a directory up in the rank
cd ..
#Change to a directory
cd dir
#########################
clear, reset
#To clean screen
reset
clear
#########################
column
#To align data to legends for easy read . column -t file
column -t tabular_file
column -t tabular_file > aligned_tabular_file && mv aligned_tabular_file  tabular_file
#########################
 comm
#For comm comand to be executed, the rows must be in sorted manner
#It  has several options.-1 means ignore unique file1 lines, -2 means ignore unique file 2 lines, -3 means ignore lines common to both.

#Shows common proteins to file 1 and file2
comm -12  file1 file2
#These proteins occur only in file1

comm -23 file1 file2
#These proteins occur only in file2

comm -13  file1 file2
#Check what the option -3 does
#########################
cp
#Copy a file and  rename it
cp file1 file2
#Copy file1 and file2 to directory dir
cp file1 file2 dir/
#Recursive copy (-r or -R option)
cp -r
Copy the folder and its contents from the host to local PC folder
cp -R full_source_path  full_destination path

e.g. cp -R /share/projects/data/genomes /home/pseema/Desktop/new_hypothetical_IS
#########################
curl
#Opens the website
curl http://aphotoathought.blogspot.com/
curl -s 'http://twitter.com/users/username.json' | python -mjson.tool
#########################
cut
#Extracts the 3rd field if each field is separated by a comma
cut -d, -f3 file
#Keep only  character 1 to 5 of each line in the file
cut -c 1-5 file
#Extracts only first and second character
cut -c1-2 file
cut -c2 file
#select from character 2 to 5
cut -c2-5 file
#keep 20 characters
cut -c20- file
#select from field 1 to 3 separated by delimiter space
cut -d " " -f1,3 file
#Extracts column 1 and 3 separated by delimiter :
cut -d':' -f1,3 file
#To extract columns from 10000 to 100000
cut -d '\t' -f 10000-100000 file
#########################
diff
#The diff command compare files line by line and displays a list of changes between two file.
# Compare two folders
diff  /tmp/r/   /tmp/s/
#Compare multiple files and get the difference
for file in `ls -1 $dir/$files`
do
   echo $file
   diff standard.file $file

done > outputfile
#########################
echo
#Tilde expansion shows path of directory
echo ~
#Arithmetic expansion $((expression))
echo $((2 * 3))
echo $(($((5**2)) * 3))
#Brace expansion
It adds the prefix and suffix to each letter within the braces
echo Front-{A,B,C}-Back
echo Number{1..10}
echo {01..100}
echo {001..50}
echo {A..Z}
echo {Z..A}
echo a{A{1,2},B{3,4}}b
*Exactly two space between the letters in order to print desired result
#To add numbers
x=742   
y=1511   
add=$(($x + $y)) 
echo "Sum: $add"
 #Sum: 2253
Command substitution ($ use)
x="It is a sunny day"
echo $x
Length of as string stored in a variable
VAR="seema"
echo ${#VAR}
echo $(ls)
echo $(cal)
echo "$(cal)"
echo "The balance for user $USER is: \$5.00"
#Print the given statement
echo "The biggest protein is: "'

#Find the length of a string from the command line
expr length "seema."
expr length "seema patel"
echo -n "seema" | wc -c
echo "hello" | awk '{print length}'
# Copy files to multiple directories
echo /usr/dir1 /var/dir2 /nas/dir3 |  xargs -n 1 cp -v /path/to/file

#Filenames starting with a, followed by a single character, followed by c and a single
character, are matched and listed.
 ls a?c?
#Filenames starting with abc and followed by exactly three characters are expanded and
displayed by the echo command.
echo  abc???

echo $[ 5 + 4 − 2 ]
#states=(ME  [3]=CA  [2]=CT)
echo ${states[*]}
echo ${states[0]}

#function_name () { commands ; commands; }
function greet { echo "Hello $LOGNAME, today is $(date)"; }
greet

declare –i num
num=hello
echo $num

num=5+5
echo $num

num=4*6
echo $num
#########################
fc
#selects the last 16 commands from the history list
fc −l
#selects the last three commands from the history list
fc −l −3
#prints the history list without line numbers
fc −ln
#########################
find
#To produce a list of home directory
find ~
find ~ | wc -l
#To find type of list 
find ~ -type d | wc -l
find ~ -type f | wc -l
find ~ -type f -name "*.JPG" -size +1M | wc -l
#Output of the find command piped into xargs which  creates argument list for the ls command 
find ~ -type f -name 'file' -print | xargs ls -l
#To find files starting with a pattern  and move them to a directory
find . -type f -name "separate*" -type f | xargs -I '{}' mv {} full_path_of_directory
#Shows file sunder current directory (2 options)
find . -maxdepth 1 -type f -printf '%f\n'
for i in $(find . -maxdepth 1 -type f -printf '%f\n'); do echo $i; done
#Copy all files and put in the directory 'New'
find . -name \file* -exec cp {} New/ \;
# List all directories in your $HOME
find $HOME -type d -ls | less
#Find files in the directory that have pattern 'found' in their name
find path_name -name '*found*'
find path_name -name '*found*' |wc
#Find the files with the pattern '*coiled_motif*' in file name
 find path_name -name $isolate.'*coiled_motif*' | wc
#Find files with size0 in a particular directory
find . -size 0
#Find files with size0 in a particular directory
 find . -size 0 -delete
#Print contents of a number of files with common pattern in file name
find pattern_*
find pattern_* |  wc -l
cat `find pattern_*`  
 #To delete backup files (ending in ~_ in a folder. Its required to avoid mistakes in counting common lines between files.
find . -name '*~' -exec rm {} \;
########################
fmt
#Reformat each paragraph
fmt file
#Wrap long lines
fmt -s file
########################
locate
Locates all zip files in the system and then greps zip files only in bin folders
locate zip
locate zip | grep bin/
Counts the number of zip files
locate -c  zip
########################
ls
#List all files in a directory and output the data into a file
ls directory_name
ls directory_name > ls_file
#Tells rwr permission of the file
ls -l file
#To count the files in the directory (the first one is more correct)
ls -l | wc -l
ls -l | grep -v  ^l| wc -l
#file name with abc followed by either of any given number
ls abc[123]
abc1  abc2
ls abc[1–3]
ls [a–z][a–z][a–z]
ls [!f–z]???

ls abc12[23]
#To show path of cp (the both commands below do same thing)
ls -l $(which cp)
ls -l `which cp`
ls
echo *
echo */
ls -a
ls -l
ls --author -l
ls --block-size=M -l
ls --block-size=K -l
########################
man
#To see manual  of a particular command
man cp
########################
mkdir
mkdir pics
mkdir {2007..2009}-{01..05}

#File creation and execution permission
chmod g+wx dir
#Deletion option to members of the group
chmod +t dir
########################
mv
#Rename file
mv file1 file_new
#file abc has been changed to file xyz
mv abc xyz
cat file1 && mv file1  file2
########################
paste
option -d (to paste based on character not tab)
#join the lines with comma between each lines

paste -d, -s file2 > file3
#To add columns of two files side by side
paste file1 file2 > combined_file
paste -d file1 file2 > combined_file
paste -d"\t" file1 file2
#Paste field 1 from file1 to file2 and output the merged file
awk '{print $1}' < file1 | paste file2 - > merged_file
#Paste columns from multiple file into 1 column
awk '{print $1}'  file1 file2 > merged_file
#########################
printenv, printfpr
#To see variables
printenv | less
#To add legend to the data file
{ printf 'ISOLATE\tName\tLineage\tSmallest\tLargest\tCountry\n'; cat file; } > final_file_with_legend
#Print a word/string/pattern n times (here 5 times)
printf 'pattern\n%.0s' {1..5}
#Pastes side by side with tabs in between for better display purpose
pr -m -t file1 file2 > combined_file
#########################
read
echo 'Enter your name'
read name

#!/bin/bash
echo −e "Are you happy? \c"
read answer
echo "$answer is the right response."
echo −e "What is your full name? \c"
read first middle last
echo "Hello  $first"
echo –n "Where do you work? "
read
echo I hope you like $REPLY .

#!/bin/bash
read −p "Enter your job title: "
echo "I thought you might be an $REPLY."
echo −n "Who are your best friends? "
read −a friends

echo "Say hi to ${friends[2]}.
#########################
 rm
#Removes file 1 and 2
rm file 1 file2
#Delete intermediate files
rm intermediate_file1
rm intermediate_file2
#########################
screen
screen
#You can attach it or detach it
#See all commands on screen
Ctrl-A
?
#Re-attach the screen
screen -r
#To see the number of screen
screen -ls
#Restore a specific screen 
screen -r 7849
#Logging feature
screen -L
#Leaving Screen
Ctrl-A  and  K
#########################
seq
#Prints 1 to 10 (in 10 lines)
seq 10
#Prints 3 to 9 (in 7 lines)
seq 3 9
#Prints 10 to 110 (in 10 intervals)
seq 10 10 110
#Generate some numbers (18 here) and print them on groups of 5
seq 18 | xargs -n5
########################
sort
sort -k1 means sort starting on key 1 till the end of the line.  
sort -k1,1 means sort from key 1 to key 1
#sort alphabetically

sort file1 > file2
#Sort file numerically
sort -n file > sorted_length_file
#Sort file alphabetically
sort -u file > sorted_length_file
#Sort the file to find lines in the order of maximum frequency
sort filepath  | uniq -c | sort -n -r > outfile
#A csv (comma-separated rows) file sorted based on text of field 4 and number of field1
sort -t, -k 4,4 -k 1,1nr file
#Sorted as per column 2
cat  file| sort -k2,2
#Sort based on column 2
sort -k 2 file
#Sorting numerically based on column 4
sort -k4 -n file
#Sorting primarily and also secondarily
sort -k4n -k2 file
sort -k4n -k2 -o outfile infile
sort -k4n -k2 infile > outfile
#Join two files (Where one file is not sorted) Using sort and join
$ sort m1.txt | join - m2.txt
#Find all the unique 4-letter words in a text
 $ cat x.txt | perl -ne 'print map("$_\n", m/\w+/g);' | tr A-Z a-z | sort | uniq | awk 'length($1) == 4 {print}'
#########################
split
Split a multi-row file into many small files (suppose each of 20 rows)
split -l 20 bigfile smallfiles
#########################
tr
#Replace dot with comma
 tr "." "," < file
#Remove dot
tr -d "."  < file
#replace dots with line breaks
tr "." "\n" < file
# Removes the carriage returns in the file
#cat $file | tr -d "\r" > new_file
#convert all words to upper case
 tr "[:lower:]" "[:upper:]" < file
#Print all comma-separated items in a row 
echo "winter, rain, autumn, spring" | tr , "\n" | sort | tr "\n" , | sed 's@,$@\n@'
#Print all comma-separated items in a column 
echo "winter, rain, autumn, spring" | tr , "\n" | sed 's@,$@\n@'
#Alphabetically arrange the items
echo "winter, rain, autumn, spring" | tr , "\n" | sort |  sed 's@,$@\n@'
echo "winter, rain, autumn, summer, spring" | tr , "\n" | sort
#Convert all lower case letters into n or N
tr actg n < file
tr actg N < file
#Remove blank lines from a file
tr -s '\n' < file

#########################
touch
#Create a new file
touch file
> file
#########################
wc
#Count line numbers in the file
wc -l file
#Count number of file in a directory
ls -l |wc -l
#########################
uniq
#Extracts only unique rows in a file
uniq file
awk '!NF || !seen[$0]++' file
#Prints each unique entry one time
sort file | uniq
#Prints entries those exist only once
sort file | uniq -u
#Finding duplicate lines (cound instances of duplication)
sort file | uniq -c
#sort -f and uniq -i can be used to ignore case
#Extract unique values between 2 files
awk 'FNR==NR {a[$0]++; next} !a[$0]' file1 file2
#prints unique lines for column 2 only
 uniq -f 2 input
#Compare multiple files and get the difference
cat $dir/$files|uniq -u|while read line
do
   echo "\^${line}\$"
done > outfile
grep -f outfile $dir/$files
#########################
whatis, whereis, whoami, who
#Describes keyword
whatis man
#Locates files
whereis file
#Displays user id
whoami
#########################
zip, tar, rar
#Compress file
gzip file
#Uncompress file
gzip -d file.gz
gunzip file.gz
#List the files in the archive archive.tar verbosely. The options ‘t’ (Display the contents), ‘v’ (Verbose), ‘f’ (files)
tar -tvf archive.tar
#Extract the files from archive archive.tar. x tells tar to extract files from an archive
tar -xf archive.tar
tar -xvf  file.tar.gz
tar -zxvf asciiquarium.tar.gz
tar -xvf regex-markup-0.10.0.tar.gz
unrar e file.rar
***************************************************
Some other commands
#Gives user identification
id
#Tells calendar and date
cal
date
#To abort a shell script before it is successfully executed
exit -1
#To cache console output
script my.terminal.session

No comments:

Post a Comment