+

Handy Bash oneliner commands for tsv file editing

    @@ -65,7 +46,12 @@
    grep lines without word (e.g. bbo)
    -
    grep -v bbo
    +
    grep -v bbo filename
    + +
    +grep only one/first match (e.g. bbo)
    + +
    grep -m 1 bbo filename
    grep and count (e.g. bbo)
    @@ -126,6 +112,32 @@ 
    grep $'\t'
    +
    +grep variable from variable
    + +
    $echo "$long_str"|grep -q "$short_str"
+if [ $? -eq 0 ]; then echo 'found'; fi
    + +

    //grep -q will output 0 if match found +//remember to add space between []!

    + +
    +grep strings between a bracket()
    + +
    grep -oP '\(\K[^\)]+'
    + +
    +grep number of characters with known strings in between(e.g. AAEL000001-RA)
    + +
    grep -o -w "\w\{10\}\-R\w\{1\}"
    + +

    // \w word character [0-9a-zA-Z_] \W not word character

    + +
    +a lot examples here
    + +

    http://www.cyberciti.biz/faq/grep-regular-expressions/

    +

    Sed

    @@ -165,11 +177,31 @@ 
    sed '$d'
    +
    +delete last character from end of file
    + +
    sed -i '$ s/.$//' filename
    + +
    +add string to end of file (e.g. "]")
    + +
    sed '$s/$/]/' filename
    + +
    +add string to end of each line (e.g. "}")
    + +
    sed -e 's/$/\}\]/' filename
    +
    add \n every nth character (e.g. every 4th character)
    sed 's/.\{4\}/&\n/g'
    +
    +concatenate/combine/join files with a seperator and next line (e.g seperate by ",")
    + +
    sed -s '$a,' *.json > all.json
    +
    substitution (e.g. replace A by B)
    @@ -231,6 +263,11 @@ e.g. add the filename to every last column of the file

    for i in $(ls);do sed -i "s/$/\t$i/" $i;done
    +
    +add extension of filename to last column
    + +
    for i in T000086_1.02.n T000086_1.02.p;do sed "s/$/\t${i/*./}/" $i;done >T000086_1.02.np
    +
    remove newline\ nextline
    @@ -241,6 +278,11 @@ e.g. add the filename to every last column of the file

    sed -n '10,33p' <filename
    +
    +change delimiter
    + +
    sed 's=/=\\/=g'
    +

    Awk

    @@ -292,6 +334,11 @@ awk -v a="$a 
    awk -v N=7 '{print}/bbo/&& --N<=0 {exit}'
    +
    +print filename and last line of all files in directory
    + +
    ls|xargs -n1 -I file awk '{s=$0};END{print FILENAME,s}' file'
    +
    add string to the beginning of a column (e.g add "chr" to column $3)
    @@ -375,6 +422,44 @@ fileA: 
    awk '{printf("%s\t%s\n",NR,$0)}'
    +
    +break combine column data into rows
    + +

    e.g. +seperate

    + +

    David cat,dog

    + +

    into

    + +

    David cat

    + +

    David dog

    + +

    detail here:　http://stackoverflow.com/questions/33408762/bash-turning-single-comma-separated-column-into-multi-line-string

    + +
    awk '{split($2,a,",");for(i in a)print $1"\t"a[i]}' file
    + +
    +sum up a file (each line in file contains only one number)
    + +
    awk '{s+=$1} END {print s}' filename
    + +
    +average a file (each line in file contains only one number)
    + +
    awk '{s+=$1}END{print s/NR}'
    + +
    +print field start with string (e.g Linux)
    + +
    awk '$1 ~ /^Linux/'
    + +
    +sort a row (e.g. 1 40 35 12 23 --> 1 12 23 35 40)
    + +
    awk ' {split( $0, a, "\t" ); asort( a ); for( i = 1; i <= length(a); i++ ) printf( "%s\t", a[i] ); printf( "\n" ); }'
    +

    Xargs

    @@ -977,6 +1062,11 @@ group= 5 
    echo ${#foo}
    +
    +echo tab
    + +
    echo -e ' \t '
    +
    array
    @@ -987,6 +1077,11 @@ group= 5 
    scp -r directoryname user@ip:/path/to/send
    +
    +split file into lines (e.g. 1000 lines/smallfile)
    + +
    $ split -d -l 1000 bigfilename
    +

    System

    @@ -1325,11 +1420,34 @@ chown -R user_name /path/to/directory/