awk 'pattern { action }' file
awk -F delimiter 'pattern { action }' file
awk -f script.awk file

BEGIN { # Executed before processing }
pattern { # Executed for matching lines }
END { # Executed after processing }

$0      # Entire line
$1      # First field
$2      # Second field
$NF     # Last field
$(NF-1) # Second to last field

NF      # Number of fields in current record
NR      # Current record number (line number)
FNR     # Record number in current file
FS      # Input field separator (default: whitespace)
OFS     # Output field separator (default: space)
RS      # Input record separator (default: newline)
ORS     # Output record separator (default: newline)
FILENAME # Current filename

# Print specific fields
awk '{print $1}' file.txt                # First field
awk '{print $1, $3}' file.txt            # First and third fields
awk '{print $NF}' file.txt               # Last field

# Print with custom separator
awk '{print $1 ":" $2}' file.txt         # Custom separator
awk -v OFS='\t' '{print $1, $2}' file    # Tab-separated

# Print entire line with line number
awk '{print NR, $0}' file.txt

# Use comma as separator
awk -F',' '{print $1}' file.csv

# Use colon as separator (like /etc/passwd)
awk -F':' '{print $1}' /etc/passwd

# Multiple character separator
awk -F'::' '{print $2}' file.txt

# Regex as separator
awk -F'[,:]' '{print $1}' file.txt       # Comma or colon

# Match lines containing pattern
awk '/pattern/ {print}' file.txt
awk '/error/ {print $0}' logfile.txt

# Case-insensitive match
awk 'tolower($0) ~ /pattern/' file.txt

# Regex on specific field
awk '$2 ~ /pattern/' file.txt
awk '$2 !~ /pattern/' file.txt  # Not matching

# Exact match
awk '$1 == "value"' file.txt
awk '$2 != "value"' file.txt

# Greater than
awk '$3 > 100' file.txt

# Less than or equal
awk '$2 <= 50' file.txt

# Range
awk '$1 >= 10 && $1 <= 20' file.txt

# Complex conditions
awk '$1 > 100 && $2 == "active"' file.txt
awk '$1 > 100 || $2 > 200' file.txt

# Header and footer
awk 'BEGIN {print "Name\tAge"} {print $1, $2} END {print "---"}' file

# Initialize variables
awk 'BEGIN {count=0} {count++} END {print count}' file

# Set field separator in BEGIN
awk 'BEGIN {FS=","} {print $1}' file.csv

# Sum a column
awk '{sum += $1} END {print sum}' file.txt

# Average
awk '{sum += $1; count++} END {print sum/count}' file

# Count records
awk 'END {print NR}' file.txt

# Max value
awk 'BEGIN {max=0} {if ($1 > max) max=$1} END {print max}' file

# Count occurrences
awk '{count[$1]++} END {for (key in count) print key, count[key]}' file

# If-else
awk '{if ($1 > 100) print "High"; else print "Low"}' file

# Ternary operator
awk '{print ($1 > 100) ? "High" : "Low"}' file

# Multiple conditions
awk '{
    if ($1 > 100) print "High"
    else if ($1 > 50) print "Medium"
    else print "Low"
}' file

# Count occurrences
awk '{count[$1]++} END {for (key in count) print key, count[key]}' file

# Group by key
awk '{sum[$1] += $2} END {for (key in sum) print key, sum[key]}' file

# Check if key exists
awk '{if ($1 in array) print "Duplicate"}' file

# Count unique values
awk '{a[$1]++} END {print length(a)}' file

# Find duplicates
awk '{count[$1]++} END {for (k in count) if (count[k] > 1) print k}' file

# Store and print in order
awk '{lines[NR] = $0} END {for (i=1; i<=NR; i++) print lines[i]}' file

length(string)           # String length
substr(string, start, len) # Substring
index(string, substring) # Find substring position
split(string, array, sep) # Split string into array
sub(regex, replacement, string) # Replace first match
gsub(regex, replacement, string) # Replace all matches
tolower(string)          # Convert to lowercase
toupper(string)          # Convert to uppercase
match(string, regex)     # Test regex match

# String length
awk '{print length($1)}' file

# Substring
awk '{print substr($1, 1, 3)}' file  # First 3 characters

# Replace
awk '{gsub(/old/, "new"); print}' file

# Convert case
awk '{print toupper($1)}' file

# Split and process
awk '{split($0, a, ":"); print a[1]}' /etc/passwd

int(x)      # Integer part
sqrt(x)     # Square root
sin(x)      # Sine
cos(x)      # Cosine
atan2(y,x)  # Arctangent
log(x)      # Natural logarithm
exp(x)      # Exponential
rand()      # Random number [0,1)
srand()     # Seed random number generator

# Count HTTP status codes
awk '{print $9}' access.log | sort | uniq -c

# Sum response times
awk '{sum += $10; count++} END {print sum/count}' access.log

# Filter by time range
awk '$4 > "[01/Jan/2025:10:00:00" && $4 < "[01/Jan/2025:11:00:00"' access.log

# Extract and count IPs
awk '{print $1}' access.log | sort | uniq -c | sort -rn | head -10

# Print specific columns from CSV
awk -F',' '{print $1, $3}' file.csv

# Skip header
awk -F',' 'NR > 1 {print $1, $2}' file.csv

# Convert CSV to tab-separated
awk -F',' -v OFS='\t' '{print $1, $2, $3}' file.csv

# Filter rows
awk -F',' '$3 > 100 {print $0}' file.csv

# Parse /etc/passwd
awk -F':' '{print $1, $6}' /etc/passwd  # username, home directory
awk -F':' '$3 >= 1000 {print $1}' /etc/passwd  # Regular users

# Disk usage analysis
df -h | awk '$5 > 80 {print $0}'  # > 80% full

# Process monitoring
ps aux | awk '$3 > 50 {print $2, $11}'  # High CPU processes

# Network stats
netstat -an | awk '/ESTABLISHED/ {print $5}' | cut -d: -f1 | sort | uniq -c

# Swap columns
awk '{print $2, $1}' file

# Add line numbers
awk '{print NR ":", $0}' file

# Remove duplicates (keeping first occurrence)
awk '!seen[$0]++' file

# Join lines
awk '{printf "%s ", $0} END {print ""}' file

# Transpose rows to columns
awk '{for (i=1; i<=NF; i++) a[NR,i]=$i; max=(NF>max?NF:max)}
     END {for (i=1; i<=max; i++) {
         for (j=1; j<=NR; j++) printf "%s ", a[j,i]
         print ""
     }}' file

awk '
BEGIN {
    FS = ","
    print "Processing..."
}
{
    sum += $2
    count++
}
END {
    print "Average:", sum/count
}
' file.csv

# script.awk
BEGIN {
    FS = ","
}
{
    sum += $2
}
END {
    print "Total:", sum
}

# Execute
awk -f script.awk file.csv

awk 'NF > 0' file
awk '/./' file

awk 'NR>=10 && NR<=20' file  # Lines 10-20

awk '!seen[$0]++' file

awk '/pattern/ {count++} END {print count}' file

# Fixed-width columns
awk '{printf "%-20s %10s\n", $1, $2}' file

# Align numbers
awk '{printf "%s: %8.2f\n", $1, $2}' file

# Faster
awk '{print $NF}' file

# Slower
awk '{print $length($0)}' file

# Good
awk '$1 > 100' file

# Wasteful
awk '{if ($1 > 100) print $0}' file

# Compile regex once
awk 'BEGIN {pattern = /error/} $0 ~ pattern' file

# Wrong - shell expands $1
awk {print $1} file

# Right
awk '{print $1}' file

# Dangerous
awk '{print $1/$2}' file

# Safe
awk '{if ($2 != 0) print $1/$2; else print "N/A"}' file

# Problematic
awk '$1 == 0.1' file

# Better
awk 'function abs(x){return x<0?-x:x} abs($1 - 0.1) < 0.001' file

# Check field existence
awk 'NF >= 3 {print $3}' file

# awk with grep
grep "error" log | awk '{print $1, $NF}'

# awk with sort
awk '{print $2}' file | sort -n

# Pipeline
cat file | awk '$1 > 100' | sort | uniq

Bash Reference Guide

Overview

Bash (Bourne Again SHell) is a Unix shell and command language. This guide covers bash-specific features, syntax, and best practices.

Official Documentation: https://www.gnu.org/software/bash/manual/

Bash vs POSIX Shell (sh)

Bash is a superset of POSIX sh with many extensions. Not all bash scripts are POSIX-compliant.

Bash-Specific Features (NOT in POSIX sh)

1. Arrays

   # Bash only
   array=(one two three)
   echo "${array[0]}"
   declare -a indexed_array
   declare -A associative_array

2. [[ ]] Test Construct

   # Bash only - more powerful than [ ]
   if [[ "$var" == pattern* ]]; then
       echo "Matches pattern"
   fi
   
   # POSIX sh - use [ ]
   if [ "$var" = "exact" ]; then
       echo "Exact match"
   fi

3. Process Substitution

   # Bash only
   diff <(ls dir1) <(ls dir2)

4. Brace Expansion

   # Bash only
   echo {1..10}  # Outputs: 1 2 3 4 5 6 7 8 9 10
   mv file.{txt,bak}  # Renames file.txt to file.bak

5. Function Keyword

   # Bash style (function keyword optional)
   function myfunction {
       echo "Hello"
   }
   
   # POSIX sh style (no function keyword)
   myfunction() {
       echo "Hello"
   }

6. Local Variables

   # Bash only
   function myfunc {
       local var="value"
       echo "$var"
   }

7. Extended Pattern Matching

   shopt -s extglob
   # Bash only
   ?(pattern-list)  # Matches zero or one occurrence
   *(pattern-list)  # Matches zero or more occurrences
   +(pattern-list)  # Matches one or more occurrences
   @(pattern-list)  # Matches one occurrence
   !(pattern-list)  # Matches anything except pattern

8. Advanced Parameter Expansion

   # Bash supports more advanced parameter expansion
   ${var,,}    # Lowercase
   ${var^^}    # Uppercase
   ${var:0:5}  # Substring
   ${var/pattern/replacement}  # Replace first
   ${var//pattern/replacement} # Replace all

9. Source vs Dot

   source script.sh  # Bash (also works: . script.sh)
   . script.sh       # POSIX sh

10. Bash Built-in Variables

    $RANDOM     # Random number
    $SECONDS    # Seconds since script started
    $BASH_VERSION
    $BASH_SOURCE
    $FUNCNAME
    $DIRSTACK

Core Bash Syntax

Variables

# Assignment (no spaces around =)
var="value"
readonly CONST="constant"
declare -i integer=42
declare -r readonly_var="const"
declare -x export_var="exported"

# Reading variables
echo "$var"
echo "${var}"  # Preferred for clarity

# Command substitution
result=$(command)
result=`command`  # Deprecated, use $() instead

# Arithmetic
result=$((5 + 3))
((var++))
((var += 5))

Quoting Rules

# Double quotes: Preserve literal value except $, `, \, and !
echo "Value: $var"

# Single quotes: Preserve literal value of all characters
echo 'Value: $var'  # Outputs: Value: $var

# No quotes: Word splitting and pathname expansion
files=$var  # Dangerous if var contains spaces

# Always quote variable expansions unless you need word splitting
cp "$file" "$destination"

Control Structures

# If statement
if [[ condition ]]; then
    # commands
elif [[ condition ]]; then
    # commands
else
    # commands
fi

# Case statement
case "$var" in
    pattern1)
        # commands
        ;;
    pattern2|pattern3)
        # commands
        ;;
    *)
        # default
        ;;
esac

# For loops
for item in list; do
    echo "$item"
done

for ((i=0; i<10; i++)); do
    echo "$i"
done

# While loop
while [[ condition ]]; do
    # commands
done

# Until loop
until [[ condition ]]; do
    # commands
done

Functions

# Function definition
function_name() {
    local local_var="value"
    echo "$1"  # First argument
    return 0   # Exit status
}

# Call function
function_name arg1 arg2

# Function with return value (via stdout)
get_value() {
    echo "returned value"
}
result=$(get_value)

Error Handling

# Exit on error
set -e
set -o errexit

# Exit on undefined variable
set -u
set -o nounset

# Pipe failure detection
set -o pipefail

# Combining options
set -euo pipefail

# Trap errors
trap 'echo "Error on line $LINENO"' ERR

# Trap exit
trap cleanup EXIT
cleanup() {
    # Cleanup code
    rm -f "$temp_file"
}

Input/Output Redirection

# Redirect stdout
command > file

# Redirect stderr
command 2> errors.txt

# Redirect both
command &> output.txt
command > output.txt 2>&1

# Append
command >> file

# Here document
cat <<EOF
multiple
lines
of text
EOF

# Here string
grep pattern <<< "$variable"

Best Practices

1. Use SheBang

#!/usr/bin/env bash
# Portable shebang that finds bash in PATH

2. Enable Strict Mode

#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'

3. Quote Variables

# Good
cp "$source" "$destination"

# Bad (fails with spaces)
cp $source $destination

4. Use $() Instead of Backticks

# Good
result=$(command)

# Bad
result=`command`

5. Check Command Existence

if command -v shellcheck &>/dev/null; then
    echo "ShellCheck is installed"
fi

6. Use [[ ]] for Tests

# Preferred in bash
if [[ "$var" == "value" ]]; then
    # ...
fi

# Use [ ] only for POSIX compliance
if [ "$var" = "value" ]; then
    # ...
fi

7. Handle Errors Appropriately

if ! command; then
    echo "Command failed" >&2
    exit 1
fi

command || { echo "Failed" >&2; exit 1; }

8. Use Meaningful Variable Names

# Good
user_count=10
max_retries=3

# Bad
n=10
x=3

9. Add Comments

# Explain complex logic
# Document function parameters
# Clarify non-obvious behavior

10. Use Functions for Reusability

log_error() {
    echo "[ERROR] $*" >&2
}

log_info() {
    echo "[INFO] $*"
}

Common Pitfalls

1. Unquoted Variables

# Wrong
file=/path/with spaces/file.txt
cat $file  # Fails!

# Right
file="/path/with spaces/file.txt"
cat "$file"

2. Not Checking Return Codes

# Wrong
cd /some/directory
rm -rf *  # Dangerous if cd fails!

# Right
cd /some/directory || exit 1
rm -rf *

3. Using [ ] with Bash Features

# Wrong
if [ "$var" == pattern* ]; then  # == not in POSIX

# Right (bash)
if [[ "$var" == pattern* ]]; then

# Right (POSIX)
if [ "$var" = "exact" ]; then

4. Word Splitting Issues

# Wrong
files=$(ls *.txt)
for file in $files; do  # Breaks on spaces
    echo "$file"
done

# Right
for file in *.txt; do
    echo "$file"
done

5. Not Using Local in Functions

# Wrong - pollutes global scope
function bad {
    var="value"
}

# Right
function good {
    local var="value"
}

Parameter Expansion Reference

${var}              # Value of var
${var:-default}     # Use default if var is unset
${var:=default}     # Assign default if var is unset
${var:?error}       # Error if var is unset
${var:+alternate}   # Use alternate if var is set
${#var}             # Length of var
${var:offset:length} # Substring
${var#pattern}      # Remove shortest match from beginning
${var##pattern}     # Remove longest match from beginning
${var%pattern}      # Remove shortest match from end
${var%%pattern}     # Remove longest match from end
${var/pattern/replacement}   # Replace first match
${var//pattern/replacement}  # Replace all matches
${var^}             # Uppercase first character
${var^^}            # Uppercase all
${var,}             # Lowercase first character
${var,,}            # Lowercase all

Special Variables

$0      # Script name
$1-$9   # Positional parameters
${10}   # 10th parameter (braces required)
$#      # Number of positional parameters
$*      # All positional parameters (as single word)
$@      # All positional parameters (as separate words)
$$      # Process ID of shell
$!      # PID of last background command
$?      # Exit status of last command
$_      # Last argument of previous command

Resources

• Official GNU Bash Manual
• Bash Guide for Beginners
• Advanced Bash-Scripting Guide
• ShellCheck - Linting tool

Common Shell Scripting Mistakes

This guide covers frequent mistakes made in bash and shell scripts, their consequences, and how to fix them.

1. Unquoted Variables

Problem

# Wrong
file=/path/with spaces/file.txt
cat $file  # Breaks into multiple arguments

Consequence

Word splitting and glob expansion cause unexpected behavior.

Solution

# Right
file="/path/with spaces/file.txt"
cat "$file"

Rule

Always quote variable expansions unless you explicitly need word splitting.

---

2. Not Checking Command Success

Problem

# Wrong
cd /some/directory
rm -rf *  # DANGEROUS if cd fails!

Consequence

Commands execute even if previous commands fail, potentially catastrophic.

Solution

# Right
cd /some/directory || exit 1
rm -rf *

# Or use set -e
set -e
cd /some/directory
rm -rf *

# Or check explicitly
if ! cd /some/directory; then
    echo "Failed to change directory" >&2
    exit 1
fi
rm -rf *

---

3. Using [ ] with Bash Features

Problem

# Wrong (== not POSIX, may fail in sh)
if [ "$var" == "value" ]; then
    echo "match"
fi

Solution

# POSIX sh - use single =
if [ "$var" = "value" ]; then
    echo "match"
fi

# Or use bash [[ ]] (bash only)
if [[ "$var" == "value" ]]; then
    echo "match"
fi

---

4. Useless Use of cat (UUOC)

Problem

# Wrong - unnecessary cat
cat file.txt | grep pattern
cat file.txt | awk '{print $1}'

Consequence

Wastes a process, less efficient.

Solution

# Right
grep pattern file.txt
awk '{print $1}' file.txt

# Or use redirection
< file.txt grep pattern

---

5. Not Using -r with read

Problem

# Wrong
while read line; do
    echo "$line"
done < file

Consequence

Backslashes are interpreted, leading character may be removed.

Solution

# Right
while IFS= read -r line; do
    echo "$line"
done < file

• -r prevents backslash interpretation
• IFS= prevents leading/trailing whitespace trimming

---

6. Testing $? After Multiple Commands

Problem

# Wrong
command1
command2
if [ $? -eq 0 ]; then  # Tests command2, not command1!
    echo "Success"
fi

Solution

# Right - test immediately
command1
if [ $? -eq 0 ]; then
    echo "command1 succeeded"
fi

# Better - test directly
if command1; then
    echo "Success"
fi

---

7. Arrays in POSIX sh Scripts

Problem

#!/bin/sh
# Wrong - arrays not in POSIX sh
array=(one two three)
echo "${array[0]}"

Solution

# Use bash
#!/bin/bash
array=(one two three)
echo "${array[0]}"

# Or use POSIX alternatives
set -- one two three
echo "$1"

---

8. Not Declaring Functions Before Use

Problem

# Wrong - function not defined yet
my_function

my_function() {
    echo "Hello"
}

Solution

# Right - define first
my_function() {
    echo "Hello"
}

my_function

---

9. Using eval Unsafely

Problem

# DANGEROUS
user_input="$1"
eval "$user_input"  # Command injection risk!

Consequence

Security vulnerability - arbitrary code execution.

Solution

# Avoid eval when possible
# If necessary, sanitize input thoroughly
# Or use safer alternatives

# Example: dynamic variable names
var_name="my_var"
# Don't: eval "echo \$$var_name"
# Do: Use indirect expansion (bash)
echo "${!var_name}"

---

10. Forgetting set -u

Problem

# Wrong - typo goes unnoticed
nmae="John"  # Typo
echo "Hello, $name"  # Prints "Hello, " (empty)

Solution

# Right - use set -u
set -u
nmae="John"  # Typo
echo "Hello, $name"  # Error: name: unbound variable

---

11. Incorrect String Comparison

Problem

# Wrong - numeric comparison on strings
if [ "$version" -gt "2.0" ]; then
    echo "New version"
fi

Solution

# Right - string comparison
if [ "$version" = "2.0" ]; then
    echo "Exact match"
fi

# Or use proper version comparison
if [[ "$version" > "2.0" ]]; then
    echo "Greater"
fi

---

12. Not Handling Spaces in Filenames

Problem

# Wrong
for file in $(ls *.txt); do
    echo "$file"
done

Consequence

Files with spaces break into multiple items.

Solution

# Right - use glob directly
for file in *.txt; do
    echo "$file"
done

# Or use find with -print0
while IFS= read -r -d '' file; do
    echo "$file"
done < <(find . -name "*.txt" -print0)

---

13. Backticks Instead of $()

Problem

# Deprecated
result=`command arg1 arg2`

Solution

# Modern
result=$(command arg1 arg2)

Why

• Better nesting: $(cmd1 $(cmd2))
• Better readability
• Fewer escaping issues

---

14. Using = Instead of == in [[ ]]

Not really a mistake, but inconsistent:

# Both work in [[ ]]
[[ "$var" = "value" ]]   # POSIX style (works)
[[ "$var" == "value" ]]  # Bash style (also works)

# Only = works in [ ]
[ "$var" = "value" ]     # Works
[ "$var" == "value" ]    # May fail in POSIX sh

Recommendation: Use = for portability, or stick to == in bash with [[ ]].

---

15. Not Quoting $@

Problem

# Wrong
script.sh "$@"  # Right
command $@      # Wrong if args have spaces

Solution

# Right
command "$@"  # Preserves argument boundaries

---

16. Using ls to Process Files

Problem

# Wrong
files=$(ls *.txt)
for file in $files; do
    process "$file"
done

Issues

• Breaks on spaces
• Breaks on newlines in filenames
• Breaks on glob characters

Solution

# Right
for file in *.txt; do
    process "$file"
done

# Or with find
find . -name "*.txt" -exec process {} \;

---

17. Incorrect Exit Codes

Problem

# Wrong
function check_file() {
    if [ -f "$1" ]; then
        echo "File exists"
        return 1  # Success should be 0!
    fi
    return 0
}

Solution

# Right - 0 is success, non-zero is failure
function check_file() {
    if [ -f "$1" ]; then
        echo "File exists"
        return 0
    fi
    return 1
}

---

18. Using -a and -o in [ ]

Problem

# Deprecated and error-prone
[ "$a" = "x" -a "$b" = "y" ]
[ "$a" = "x" -o "$b" = "y" ]

Solution

# Right - use && and ||
[ "$a" = "x" ] && [ "$b" = "y" ]
[ "$a" = "x" ] || [ "$b" = "y" ]

# Or use [[ ]] in bash
[[ "$a" = "x" && "$b" = "y" ]]
[[ "$a" = "x" || "$b" = "y" ]]

---

19. Not Making Scripts Executable

Problem

# Wrong
bash script.sh  # Works but not ideal

Solution

# Right
chmod +x script.sh
./script.sh

And include proper shebang:

#!/usr/bin/env bash

---

20. Forgetting Final Newline

Problem

Some tools expect files to end with a newline.

Solution

Ensure your editor adds a final newline, or:

echo "" >> file

---

21. Using grep -q Without Knowing Implications

Problem

# Potentially inefficient
if [ "$(grep pattern file)" ]; then
    echo "Found"
fi

Solution

# Better - grep -q exits on first match
if grep -q pattern file; then
    echo "Found"
fi

---

22. Incorrect glob Pattern

Problem

# Wrong - doesn't match hidden files
for file in *; do
    process "$file"
done

Solution

# Include hidden files (bash)
shopt -s dotglob
for file in *; do
    process "$file"
done
shopt -u dotglob

# Or explicitly
for file in * .[!.]* ..?*; do
    [ -e "$file" ] && process "$file"
done

---

23. Not Handling Empty Globs

Problem

# Fails if no .txt files
for file in *.txt; do
    process "$file"  # Processes literal "*.txt"
done

Solution

# Bash - fail gracefully
shopt -s nullglob
for file in *.txt; do
    process "$file"
done
shopt -u nullglob

# POSIX - check existence
for file in *.txt; do
    [ -e "$file" ] || continue
    process "$file"
done

---

24. Not Sanitizing Input

Problem

# Dangerous
rm -rf "/$1"  # What if $1 is empty or manipulated?

Solution

# Safer
if [ -z "$1" ]; then
    echo "Error: No argument provided" >&2
    exit 1
fi

# Validate
case "$1" in
    /*)
        echo "Error: Absolute paths not allowed" >&2
        exit 1
        ;;
esac

rm -rf "$1"

---

25. Using -e for File Existence

Not a mistake, but be specific:

[ -e "$file" ]  # Exists (any type)
[ -f "$file" ]  # Regular file
[ -d "$file" ]  # Directory
[ -L "$file" ]  # Symbolic link
[ -r "$file" ]  # Readable
[ -w "$file" ]  # Writable
[ -x "$file" ]  # Executable

---

Quick Checklist

Before running a script, verify:

• Proper shebang (#!/bin/bash or #!/bin/sh)
• set -euo pipefail (strict mode)
• All variables quoted
• Error handling for critical commands
• Using $() not backticks
• Not using ls for file processing
• Functions defined before use
• Proper exit codes (0 = success)
• Input validation
• ShellCheck passes

---

Resources

• ShellCheck - Catches most of these
• Bash Pitfalls
• POSIX Shell

GNU grep Reference Guide

Overview

grep (Global Regular Expression Print) searches for patterns in text files. It’s one of the most commonly used Unix tools.

Official Manual: https://www.gnu.org/software/grep/manual/ Man Page: man grep

Basic Syntax

grep [OPTIONS] PATTERN [FILE...]
grep [OPTIONS] -e PATTERN ... [FILE...]
grep [OPTIONS] -f PATTERN_FILE ... [FILE...]

Common Options

Basic Options

-i, --ignore-case          # Case-insensitive search
-v, --invert-match         # Invert match (select non-matching lines)
-w, --word-regexp          # Match whole words only
-x, --line-regexp          # Match whole lines only
-c, --count                # Count matching lines
-n, --line-number          # Show line numbers
-H, --with-filename        # Print filename with matches
-h, --no-filename          # Suppress filename output
-l, --files-with-matches   # Print only filenames with matches
-L, --files-without-match  # Print only filenames without matches

Context Options

-A NUM, --after-context=NUM     # Print NUM lines after match
-B NUM, --before-context=NUM    # Print NUM lines before match
-C NUM, --context=NUM           # Print NUM lines before and after

Regular Expression Options

-E, --extended-regexp      # Use Extended Regular Expressions (ERE)
-F, --fixed-strings        # Treat PATTERN as fixed strings, not regex
-G, --basic-regexp         # Use Basic Regular Expressions (BRE) - default
-P, --perl-regexp          # Use Perl-compatible regex (PCRE)

Output Options

-o, --only-matching        # Print only matched parts
-q, --quiet, --silent      # Suppress output, just return exit code
--color[=WHEN]             # Colorize output (auto, always, never)
-s, --no-messages          # Suppress error messages

File Selection

-r, --recursive            # Recursive search
-R, --dereference-recursive # Recursive, following symlinks
--include=PATTERN          # Search only files matching PATTERN
--exclude=PATTERN          # Skip files matching PATTERN
--exclude-dir=PATTERN      # Skip directories matching PATTERN

Common Usage Patterns

Basic Searches

# Simple string search
grep "error" logfile.txt

# Case-insensitive search
grep -i "error" logfile.txt

# Search for whole word
grep -w "error" logfile.txt

# Count matches
grep -c "error" logfile.txt

# Show line numbers
grep -n "error" logfile.txt

Multiple Files

# Search in multiple files
grep "pattern" file1.txt file2.txt

# Search in all txt files
grep "pattern" *.txt

# Recursive search
grep -r "pattern" /path/to/dir

# Recursive with file pattern
grep -r --include="*.log" "error" /var/log

Context Display

# Show 3 lines after match
grep -A 3 "error" logfile.txt

# Show 3 lines before match
grep -B 3 "error" logfile.txt

# Show 3 lines before and after
grep -C 3 "error" logfile.txt

Invert Match

# Show lines that DON'T contain pattern
grep -v "debug" logfile.txt

# Exclude multiple patterns
grep -v "debug\|info" logfile.txt

Multiple Patterns

# Match any pattern (OR)
grep -e "error" -e "warning" file.txt
grep "error\|warning" file.txt

# Match all patterns (AND) - requires pipeline
grep "error" file.txt | grep "critical"

File Selection

# List files containing match
grep -l "pattern" *.txt

# List files NOT containing match
grep -L "pattern" *.txt

# Recursive with excludes
grep -r --exclude-dir=".git" "pattern" .
grep -r --exclude="*.min.js" "pattern" .

Regular Expressions in grep

Basic Regular Expressions (BRE) - Default

# BRE Metacharacters (no escaping needed)
.       # Any single character
^       # Start of line
$       # End of line
[...]   # Character class
[^...]  # Negated character class
*       # Zero or more of previous

# BRE Metacharacters (MUST be escaped)
\+      # One or more (requires \)
\?      # Zero or one (requires \)
\{m,n\} # Between m and n occurrences (requires \)
\(...\) # Group (requires \)
\|      # Alternation (requires \)

BRE Examples

# Match lines starting with "Error"
grep "^Error" file.txt

# Match lines ending with "failed"
grep "failed$" file.txt

# Match any character
grep "a.c" file.txt  # Matches abc, aXc, a5c

# Character class
grep "[0-9]" file.txt  # Match any digit
grep "[A-Za-z]" file.txt  # Match any letter

# One or more (escaped)
grep "a\+b" file.txt  # Matches ab, aab, aaab

# Groups and alternation (escaped)
grep "\(error\|warning\)" file.txt

Extended Regular Expressions (ERE) - grep -E

# ERE - No escaping needed for +, ?, |, (), {}
+       # One or more
?       # Zero or one
{m,n}   # Between m and n occurrences
(...)   # Group
|       # Alternation

ERE Examples

# One or more (no escape)
grep -E "a+b" file.txt

# Zero or one
grep -E "colou?r" file.txt  # Matches color or colour

# Alternation
grep -E "(error|warning)" file.txt

# Quantifiers
grep -E "[0-9]{3}-[0-9]{4}" file.txt  # Phone: 123-4567
grep -E "[0-9]{1,3}" file.txt  # 1 to 3 digits

# Groups
grep -E "(http|https)://[^ ]+" file.txt  # URLs

Character Classes

POSIX Character Classes

[:alnum:]   # Alphanumeric [A-Za-z0-9]
[:alpha:]   # Alphabetic [A-Za-z]
[:digit:]   # Digits [0-9]
[:lower:]   # Lowercase [a-z]
[:upper:]   # Uppercase [A-Z]
[:space:]   # Whitespace [ \t\n\r\f\v]
[:blank:]   # Space and tab [ \t]
[:punct:]   # Punctuation
[:xdigit:]  # Hex digits [0-9A-Fa-f]
[:word:]    # Word characters [A-Za-z0-9_]

Usage

# Match any digit
grep "[[:digit:]]" file.txt

# Match any whitespace
grep "[[:space:]]" file.txt

# Match uppercase letters
grep "[[:upper:]]" file.txt

Practical Examples for Shell Scripts

Log File Analysis

# Find errors in last hour
grep "$(date -d '1 hour ago' '+%Y-%m-%d %H')" /var/log/app.log | grep -i error

# Count error types
grep -i "error" logfile.log | cut -d: -f2 | sort | uniq -c | sort -rn

# Extract IP addresses
grep -oE '\b([0-9]{1,3}\.){3}[0-9]{1,3}\b' access.log

Configuration Validation

# Find uncommented lines in config
grep -v "^#" config.file | grep -v "^$"

# Check for specific settings
if grep -q "debug = true" config.ini; then
    echo "Debug mode is enabled"
fi

Code Search

# Find function definitions
grep -n "^function " script.sh

# Find TODO comments
grep -rn "TODO" --include="*.sh" .

# Find unquoted variables (simple check)
grep -n '\$[A-Za-z_][A-Za-z0-9_]*\s' script.sh

Performance Tips

1. Use -F for Fixed Strings

# Faster when not using regex
grep -F "literal string" large_file.txt

2. Use -q When Only Checking Existence

# Don't need output, just exit code
if grep -q "pattern" file.txt; then
    echo "Found"
fi

3. Limit Search Depth

# Don't recurse too deep
grep -r --max-depth=2 "pattern" /path

4. Exclude Unnecessary Directories

grep -r --exclude-dir={.git,.svn,node_modules} "pattern" .

Exit Codes

• 0: Match found
• 1: No match found
• 2: Error occurred

Common Pitfalls in Shell Scripts

1. Not Quoting Patterns with Spaces

# Wrong
grep $pattern file.txt

# Right
grep "$pattern" file.txt

2. Using grep in Tests Without -q

# Inefficient
if [ "$(grep pattern file)" ]; then

# Better
if grep -q pattern file; then

3. Useless Use of cat

# Wrong (UUOC)
cat file | grep pattern

# Right
grep pattern file
# or
< file grep pattern

4. Not Handling No Match Case

# grep returns 1 if no match, can cause set -e to exit
grep "pattern" file || true

# Or check explicitly
if grep -q "pattern" file; then
    echo "Found"
else
    echo "Not found"
fi

5. Forgetting to Escape Regex Metacharacters

# Wrong - . matches any character
grep "192.168.1.1" file

# Right - escape the dots
grep "192\.168\.1\.1" file

# Or use -F for literal match
grep -F "192.168.1.1" file

Useful Combinations

# Case-insensitive recursive search with line numbers
grep -rni "pattern" /path

# Count total matches across files
grep -r "pattern" . | wc -l

# Find and highlight matches
grep --color=always "pattern" file.txt | less -R

# Search compressed files
zgrep "pattern" file.gz

# Search with extended regex and only show matches
grep -Eo "pattern" file.txt

Resources

• GNU grep Manual
• grep(1) Linux Man Page
• Regular Expressions in grep

Regular Expressions Reference Guide

Overview

Regular expressions (regex) are patterns used to match character combinations in strings. POSIX defines two flavors: Basic (BRE) and Extended (ERE).

POSIX Specification: https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html

BRE vs ERE

Feature	BRE	ERE
One or more	\+	+
Zero or one	\?	?
Alternation	|	`
Grouping	\(...\)	(...)
Quantifiers	\{m,n\}	{m,n}

Tool Usage

# BRE (Basic)
grep 'pattern' file          # BRE by default
sed 's/pattern/repl/' file   # BRE by default
awk '/pattern/' file         # ERE by default

# ERE (Extended)
grep -E 'pattern' file       # ERE
egrep 'pattern' file         # ERE (deprecated, use grep -E)
sed -E 's/pattern/repl/' file # ERE

Basic Metacharacters (Both BRE and ERE)

Single Character Matchers

.           # Any single character except newline
[abc]       # Any character in set (a, b, or c)
[^abc]      # Any character NOT in set
[a-z]       # Any character in range
[0-9]       # Any digit

Anchors

^           # Start of line
$           # End of line
\<          # Start of word (GNU extension)
\>          # End of word (GNU extension)
\b          # Word boundary (some tools)
\B          # Not a word boundary (some tools)

Quantifiers (Zero or More)

*           # Zero or more of previous (both BRE and ERE)

Extended Metacharacters

ERE Quantifiers

+           # One or more (ERE: +) (BRE: \+)
?           # Zero or one (ERE: ?) (BRE: \?)
{n}         # Exactly n (ERE: {n}) (BRE: \{n\})
{n,}        # n or more (ERE: {n,}) (BRE: \{n,\})
{n,m}       # Between n and m (ERE: {n,m}) (BRE: \{n,m\})

Grouping and Alternation

# ERE
(pattern)   # Group
pattern1|pattern2  # Alternation (OR)

# BRE (requires backslashes)
\(pattern\)        # Group
pattern1\|pattern2  # Alternation (OR)

POSIX Character Classes

Must be used inside bracket expressions [[:class:]]:

[:alnum:]   # Alphanumeric [A-Za-z0-9]
[:alpha:]   # Alphabetic [A-Za-z]
[:digit:]   # Digits [0-9]
[:lower:]   # Lowercase [a-z]
[:upper:]   # Uppercase [A-Z]
[:space:]   # Whitespace [ \t\n\r\f\v]
[:blank:]   # Space and tab [ \t]
[:punct:]   # Punctuation
[:xdigit:]  # Hexadecimal [0-9A-Fa-f]
[:word:]    # Word characters [A-Za-z0-9_] (GNU extension)
[:graph:]   # Visible characters (not space)
[:print:]   # Printable characters (including space)
[:cntrl:]   # Control characters

Usage

# Match any digit
grep '[[:digit:]]' file

# Match any whitespace
grep '[[:space:]]' file

# Match alphanumeric
grep '[[:alnum:]]' file

# Negation
grep '[^[:digit:]]' file    # Not a digit

Common Patterns

Numbers

# BRE
[0-9]                       # Single digit
[0-9]\+                     # One or more digits
[0-9]\{3\}                  # Exactly 3 digits
[0-9]\{3,5\}                # 3 to 5 digits

# ERE
[0-9]                       # Single digit
[0-9]+                      # One or more digits
[0-9]{3}                    # Exactly 3 digits
[0-9]{3,5}                  # 3 to 5 digits

IP Addresses

# Simple (BRE)
grep '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}' file

# Simple (ERE)
grep -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' file

# More strict (ERE)
grep -E '\b([0-9]{1,3}\.){3}[0-9]{1,3}\b' file

Email Addresses

# Simple (ERE)
grep -E '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' file

# BRE
grep '[a-zA-Z0-9._%+-]\+@[a-zA-Z0-9.-]\+\.[a-zA-Z]\{2,\}' file

URLs

# Simple HTTP/HTTPS (ERE)
grep -E 'https?://[a-zA-Z0-9./?=_-]+' file

# BRE
grep 'https\?://[a-zA-Z0-9./?=_-]\+' file

Phone Numbers

# Format: 123-456-7890 (ERE)
grep -E '[0-9]{3}-[0-9]{3}-[0-9]{4}' file

# Format: (123) 456-7890 (ERE)
grep -E '\([0-9]{3}\) [0-9]{3}-[0-9]{4}' file

# BRE
grep '\([0-9]\{3\}\) [0-9]\{3\}-[0-9]\{4\}' file

Dates

# YYYY-MM-DD (ERE)
grep -E '[0-9]{4}-[0-9]{2}-[0-9]{2}' file

# MM/DD/YYYY (ERE)
grep -E '[0-9]{2}/[0-9]{2}/[0-9]{4}' file

# BRE
grep '[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}' file

Shell Script Patterns

Variable Names

# Valid bash variable name (ERE)
grep -E '^[a-zA-Z_][a-zA-Z0-9_]*=' file

# Find variable usage
grep -E '\$[a-zA-Z_][a-zA-Z0-9_]*' file
grep -E '\$\{[a-zA-Z_][a-zA-Z0-9_]*\}' file

Function Definitions

# POSIX function (ERE)
grep -E '^[a-zA-Z_][a-zA-Z0-9_]*\s*\(\)' file

# Bash function keyword (ERE)
grep -E '^function [a-zA-Z_][a-zA-Z0-9_]*' file

Comments

# Shell comments
grep '^[[:space:]]*#' file

# Uncommented lines
grep -v '^[[:space:]]*#' file

Escaping Special Characters

Characters that need escaping (context-dependent):

.  *  [  ]  ^  $  \  (  )  {  }  +  ?  |

Literal Matching

# Match literal dot (BRE and ERE)
grep '\.' file

# Match literal asterisk
grep '\*' file

# Match literal dollar sign
grep '\$' file

# Match literal brackets
grep '\[' file
grep '\]' file

Backreferences

Capture and Reuse

# BRE - capture with \(...\), reference with \1, \2, etc.
sed 's/\([0-9]\+\)-\([0-9]\+\)/\2-\1/' file  # Swap numbers

# ERE - capture with (...), reference with \1, \2, etc.
sed -E 's/([0-9]+)-([0-9]+)/\2-\1/' file

# Find duplicate words
grep -E '\b([a-z]+) \1\b' file

Examples

# Extract and rearrange (BRE)
sed 's/\([A-Z][a-z]*\), \([A-Z][a-z]*\)/\2 \1/' file

# ERE
sed -E 's/([A-Z][a-z]*), ([A-Z][a-z]*)/\2 \1/' file

# Find repeated lines
grep -E '^(.*)$\n\1$' file

Greedy vs Non-Greedy

POSIX regex is always greedy (matches longest possible string):

# Always greedy in POSIX
echo "foo bar baz" | grep -o 'f.*b'  # Matches "foo bar b"

# For non-greedy, you need to be creative
echo "foo bar baz" | grep -o 'f[^b]*b'  # Matches "foo b"

Lookahead and Lookbehind

NOT supported in POSIX BRE/ERE. Only available in PCRE (Perl-Compatible Regular Expressions):

# PCRE only (not POSIX)
grep -P '(?<=foo)bar' file      # Lookbehind
grep -P 'foo(?=bar)' file       # Lookahead

Common Mistakes

1. Forgetting to Escape in BRE

# Wrong (BRE)
grep '(foo|bar)' file

# Right (BRE)
grep '\(foo\|bar\)' file

# Or use ERE
grep -E '(foo|bar)' file

2. Using + in BRE Without Escape

# Wrong (BRE)
grep '[0-9]+' file

# Right (BRE)
grep '[0-9]\+' file

# Or use ERE
grep -E '[0-9]+' file

3. Not Escaping Dots for Literal Match

# Wrong - matches any character
grep '192.168.1.1' file

# Right - matches literal dots
grep '192\.168\.1\.1' file

4. Greedy Matching Issues

# Matches too much
echo '<tag>content</tag>' | sed 's/<.*>//'  # Empty!

# Better
echo '<tag>content</tag>' | sed 's/<[^>]*>//'

5. Character Class Mistakes

# Wrong - not a range
grep '[a-Z]' file  # Undefined behavior

# Right
grep '[a-zA-Z]' file

# Or use POSIX class
grep '[[:alpha:]]' file

Testing Regex

Online Tools

• regex101.com (supports PCRE, not POSIX)
• regexr.com
• regexpal.com

Command Line Testing

# Test with echo
echo "test string" | grep 'pattern'

# Show matches only
echo "test string" | grep -o 'pattern'

# Test with multiple lines
printf 'line1\nline2\nline3\n' | grep 'pattern'

# Color highlighting
grep --color=always 'pattern' file | less -R

Quick Reference Table

Pattern	BRE	ERE	Matches
Literal	abc	abc	abc
Any char	.	.	any single character
Start	^	^	start of line
End	$	$	end of line
Zero or more	*	*	0+ of previous
One or more	\+	+	1+ of previous
Zero or one	\?	?	0 or 1 of previous
Exactly n	\{n\}	{n}	exactly n
n or more	\{n,\}	{n,}	n or more
n to m	\{n,m\}	{n,m}	between n and m
Group	\(...\)	(...)	capture group
Alternation	|	`	`
Character class	[abc]	[abc]	a, b, or c
Negated class	[^abc]	[^abc]	not a, b, or c
Range	[a-z]	[a-z]	lowercase letters

Resources

• POSIX Regular Expressions
• Regular-Expressions.info
• GNU grep Manual - Regular Expressions

GNU sed Reference Guide

Overview

sed (Stream EDitor) is a powerful text processing tool that performs basic text transformations on an input stream (file or pipeline).

Official Manual: https://www.gnu.org/software/sed/manual/ Man Page: man sed

Basic Syntax

sed [OPTIONS] 'command' file
sed [OPTIONS] -e 'command1' -e 'command2' file
sed [OPTIONS] -f script.sed file

Common Options

-n, --quiet, --silent    # Suppress automatic output
-e SCRIPT               # Add script to commands
-f FILE                 # Add contents of FILE as commands
-i[SUFFIX]              # Edit files in-place
-r, -E                  # Use extended regular expressions
--debug                 # Annotate program execution

Basic Commands

Substitution (s)

# Basic substitution
sed 's/old/new/' file              # Replace first occurrence
sed 's/old/new/g' file             # Replace all occurrences
sed 's/old/new/2' file             # Replace second occurrence
sed 's/old/new/gi' file            # Case-insensitive, all

# With different delimiter
sed 's|/old/path|/new/path|g' file
sed 's#old#new#g' file

Deletion (d)

# Delete lines
sed '5d' file                       # Delete line 5
sed '5,10d' file                    # Delete lines 5-10
sed '/pattern/d' file               # Delete matching lines
sed '/^$/d' file                    # Delete empty lines
sed '/^#/d' file                    # Delete comment lines

Print (p)

# Print lines
sed -n '5p' file                    # Print only line 5
sed -n '5,10p' file                 # Print lines 5-10
sed -n '/pattern/p' file            # Print matching lines

Append (a), Insert (i), Change (c)

# Append after line
sed '5a\New line' file

# Insert before line
sed '5i\New line' file

# Change line
sed '5c\Replacement line' file

# With pattern
sed '/pattern/a\New line after match' file

Address Ranges

Line Numbers

sed '5s/old/new/' file              # Line 5 only
sed '5,10s/old/new/' file           # Lines 5-10
sed '5,$s/old/new/' file            # Line 5 to end
sed '1,5d' file                     # Delete first 5 lines

Patterns

sed '/start/,/end/d' file           # Delete from start to end pattern
sed '/pattern/s/old/new/' file      # Substitute in matching lines
sed '1,/pattern/d' file             # Delete from line 1 to first match

Special Addresses

sed '$d' file                       # Delete last line
sed '1d' file                       # Delete first line
sed '$s/old/new/' file              # Substitute in last line

Advanced Substitution

Backreferences

# Capture and reuse
sed 's/\([0-9]\+\)/Number: \1/' file

# Multiple captures
sed 's/\([a-z]\+\) \([0-9]\+\)/\2 \1/' file

# With ERE (-E or -r)
sed -E 's/([0-9]+)/Number: \1/' file
sed -E 's/([a-z]+) ([0-9]+)/\2 \1/' file

Flags

s/old/new/      # Replace first
s/old/new/g     # Replace all (global)
s/old/new/2     # Replace 2nd occurrence
s/old/new/i     # Case-insensitive
s/old/new/I     # Case-insensitive (same as i)
s/old/new/p     # Print if substitution made
s/old/new/w file # Write if substitution made

Special Characters in Replacement

&               # Matched string
\1, \2, etc     # Backreferences
\L, \U          # Convert to lower/upper (GNU sed)
\n              # Newline (in replacement)
\\              # Literal backslash

Multiple Commands

Multiple -e Options

sed -e 's/old/new/g' -e 's/foo/bar/g' file

Semicolon Separator

sed 's/old/new/g; s/foo/bar/g' file

Multi-line Script

sed '
s/old/new/g
s/foo/bar/g
/pattern/d
' file

In-place Editing

# Edit file in-place
sed -i 's/old/new/g' file

# Create backup
sed -i.bak 's/old/new/g' file

# Multiple files
sed -i 's/old/new/g' *.txt

Pattern Matching

BRE (Basic Regular Expressions) - Default

sed 's/^/#/' file                  # Add # at beginning
sed 's/$/;/' file                  # Add ; at end
sed 's/[0-9]\+/X/g' file          # Replace numbers (BRE)
sed 's/\<word\>/WORD/g' file      # Word boundaries (BRE)

ERE (Extended Regular Expressions)

sed -E 's/[0-9]+/X/g' file        # Replace numbers (ERE)
sed -E 's/(foo|bar)/baz/g' file   # Alternation
sed -E 's/\s+/ /g' file           # Multiple spaces to one

Practical Examples for Shell Scripts

Configuration File Editing

# Change a config value
sed -i 's/^Port .*/Port 2222/' /etc/ssh/sshd_config

# Uncomment a line
sed -i 's/^#\(.*option.*\)/\1/' config.file

# Comment out a line
sed -i 's/^\(.*dangerous.*\)/#\1/' config.file

# Add line after pattern
sed -i '/\[section\]/a new_setting = value' config.ini

Text Processing

# Remove trailing whitespace
sed 's/[[:space:]]*$//' file

# Remove leading whitespace
sed 's/^[[:space:]]*//' file

# Remove empty lines
sed '/^$/d' file

# Remove comments and empty lines
sed '/^#/d; /^$/d' file

# Double-space file
sed 'G' file

# Remove duplicate lines (consecutive)
sed '$!N; /^\(.*\)\n\1$/!P; D' file

Path Manipulation

# Change paths
sed 's|/old/path|/new/path|g' file

# Extract filename from path
echo "/path/to/file.txt" | sed 's|.*/||'

# Extract directory from path
echo "/path/to/file.txt" | sed 's|/[^/]*$||'

Log File Processing

# Extract IP addresses
sed -n 's/.*\([0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\).*/\1/p' log

# Filter by date
sed -n '/2025-01-01/,/2025-01-31/p' log

# Remove timestamp
sed 's/^[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\} [0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\} //' log

Code Refactoring

# Rename function
sed -i 's/\boldFunctionName\b/newFunctionName/g' *.sh

# Change variable
sed -i 's/\$old_var/\$new_var/g' script.sh

# Update shebang
sed -i '1s|^#!/bin/sh|#!/bin/bash|' *.sh

Advanced Features

Hold Space

# Hold space commands
h               # Copy pattern space to hold space
H               # Append pattern space to hold space
g               # Copy hold space to pattern space
G               # Append hold space to pattern space
x               # Exchange pattern and hold spaces

Hold Space Examples

# Reverse file
sed '1!G;h;$!d' file

# Print last line
sed -n '$p' file
sed -n 'h;$p' file

# Append next line to current
sed 'N;s/\n/ /' file

Branching

# Label and branch
:label          # Define label
b label         # Branch to label
t label         # Branch if substitution succeeded
T label         # Branch if substitution failed

Branching Examples

# Remove C-style comments
sed -n '
/\/\*/ {
    :loop
    /\*\// {
        s|/\*.*\*/||g
        p
        b
    }
    N
    b loop
}
p
' file

Common Patterns

Join Lines

# Join all lines
sed ':a;N;$!ba;s/\n/ /g' file

# Join lines ending with backslash
sed -e :a -e '/\\$/N; s/\\\n//; ta' file

Number Lines

sed = file | sed 'N;s/\n/\t/'

Reverse Lines

sed '1!G;h;$!d' file

Print Specific Lines

# Print line 5
sed -n '5p' file

# Print first 10 lines
sed -n '1,10p' file

# Print last 10 lines
sed -n -e :a -e '1,10!{P;N;D;};N;ba' file

Common Pitfalls in Shell Scripts

1. Special Characters in Pattern

# Wrong - . matches any character
sed 's/192.168.1.1/new/' file

# Right - escape dots
sed 's/192\.168\.1\.1/new/' file

# Or use different delimiter
sed 's|192.168.1.1|new|' file

2. Not Escaping Backreferences in BRE

# Wrong (BRE)
sed 's/([0-9]+)/\1/' file

# Right (BRE)
sed 's/\([0-9]\+\)/\1/' file

# Right (ERE)
sed -E 's/([0-9]+)/\1/' file

3. In-place Editing Without Backup

# Dangerous
sed -i 's/old/new/' important_file

# Safer
sed -i.backup 's/old/new/' important_file

4. Using sed for Line Counting

# Inefficient
sed -n '$=' file

# Better
wc -l < file

5. Not Quoting Variables Properly

# Wrong
sed "s/$old/$new/g" file  # Dangerous if vars contain /

# Better
old_escaped=$(printf '%s\n' "$old" | sed 's:[\\/&]:\\&:g')
new_escaped=$(printf '%s\n' "$new" | sed 's:[\\/&]:\\&:g')
sed "s/$old_escaped/$new_escaped/g" file

# Or use different delimiter
sed "s|$old|$new|g" file

Performance Tips

1. Use Appropriate Tools

# For simple replacements, consider using other tools
# sed is great for complex patterns, but for simple tasks:

# Instead of
sed 's/old/new/g' file

# Consider
tr 'old' 'new' < file  # For single-character replacement

2. Minimize Pattern Matching

# Less efficient
sed '/pattern/s/old/new/g' large_file

# More efficient if pattern is rare
grep 'pattern' large_file | sed 's/old/new/g'

3. Combine Commands

# Less efficient
sed 's/old/new/g' file | sed 's/foo/bar/g'

# More efficient
sed 's/old/new/g; s/foo/bar/g' file

Testing sed Commands

# Test before in-place edit
sed 's/old/new/g' file | head

# Show only changes
sed -n 's/old/new/gp' file

# Count changes
sed -n 's/old/new/gp' file | wc -l

Resources

• GNU sed Manual
• sed(1) Man Page
• sed One-Liners
• Grymoire sed Tutorial

POSIX Shell (sh) Reference Guide

Overview

POSIX sh is the portable shell specification defined by POSIX standards. Scripts written for POSIX sh should work across different Unix-like systems (bash, dash, ksh, etc.).

Official Specification: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html

Why POSIX Shell Matters

• Portability: Works across different Unix systems and shells
• Minimal dependencies: Available in minimal environments (containers, embedded systems)
• Faster: Shells like dash are faster than bash for simple scripts
• Compatibility: /bin/sh may not be bash (Ubuntu/Debian use dash)

Key Differences: sh vs bash

Features NOT Available in POSIX sh

1. Arrays

   # Bash only - NOT POSIX
   array=(one two three)
   echo "${array[0]}"

2. [[ ]] Test Construct

   # Bash only - NOT POSIX
   if [[ "$var" == "value" ]]; then
   
   # POSIX sh - use [ ]
   if [ "$var" = "value" ]; then

3. == Operator

   # Bash style - NOT POSIX
   [ "$a" == "$b" ]
   
   # POSIX sh - use single =
   [ "$a" = "$b" ]

4. Process Substitution

   # Bash only - NOT POSIX
   diff <(ls dir1) <(ls dir2)

5. Brace Expansion

   # Bash only - NOT POSIX
   echo {1..10}

6. function Keyword

   # Bash style - NOT in original POSIX
   function myfunc {
       echo "hello"
   }
   
   # POSIX sh style
   myfunc() {
       echo "hello"
   }

7. local Keyword

   # Common but not in POSIX standard
   local var="value"
   
   # POSIX alternative: use function scope carefully
   # or use naming conventions
   _func_var="value"

8. source Command

   # Bash style - NOT POSIX
   source script.sh
   
   # POSIX sh
   . script.sh

POSIX Shell Syntax

Variables

# Assignment
var="value"
readonly CONST="constant"

# Reading variables
echo "$var"
echo "${var}"

# Command substitution (POSIX)
result=$(command)

# Old-style command substitution (works but deprecated)
result=`command`

# Arithmetic (POSIX way)
result=$((5 + 3))

Quoting

# Double quotes: Preserve literal value except $, `, and \
echo "Value: $var"

# Single quotes: Preserve everything literally
echo 'Value: $var'

# Always quote variables
cp "$file" "$destination"

Control Structures

# If statement
if [ condition ]; then
    # commands
elif [ condition ]; then
    # commands
else
    # commands
fi

# Case statement
case "$var" in
    pattern1)
        # commands
        ;;
    pattern2|pattern3)
        # commands
        ;;
    *)
        # default
        ;;
esac

# For loop
for item in list; do
    echo "$item"
done

# While loop
while [ condition ]; do
    # commands
done

# Until loop
until [ condition ]; do
    # commands
done

Test Constructs

POSIX sh uses [ ] (also known as test command):

# String comparisons
[ "$a" = "$b" ]      # Equal
[ "$a" != "$b" ]     # Not equal
[ -z "$a" ]          # String is empty
[ -n "$a" ]          # String is not empty

# Numeric comparisons
[ "$a" -eq "$b" ]    # Equal
[ "$a" -ne "$b" ]    # Not equal
[ "$a" -lt "$b" ]    # Less than
[ "$a" -le "$b" ]    # Less than or equal
[ "$a" -gt "$b" ]    # Greater than
[ "$a" -ge "$b" ]    # Greater than or equal

# File tests
[ -e "$file" ]       # File exists
[ -f "$file" ]       # Regular file exists
[ -d "$file" ]       # Directory exists
[ -r "$file" ]       # File is readable
[ -w "$file" ]       # File is writable
[ -x "$file" ]       # File is executable
[ -s "$file" ]       # File is not empty

# Logical operators
[ condition1 ] && [ condition2 ]  # AND
[ condition1 ] || [ condition2 ]  # OR
[ ! condition ]                   # NOT
[ condition1 -a condition2 ]      # AND (inside test)
[ condition1 -o condition2 ]      # OR (inside test)

Functions

# POSIX function definition
function_name() {
    # No 'local' in strict POSIX
    # Use careful scoping or naming conventions
    echo "$1"  # First argument
    return 0   # Exit status
}

# Call function
function_name arg1 arg2

Input/Output Redirection

# Redirect stdout
command > file

# Redirect stderr
command 2> errors.txt

# Redirect both
command > output.txt 2>&1

# Append
command >> file

# Here document
cat <<EOF
multiple
lines
of text
EOF

# Read from stdin
while read -r line; do
    echo "$line"
done < file.txt

POSIX Best Practices

1. Proper Shebang

#!/bin/sh
# Use /bin/sh for POSIX scripts, not /bin/bash

2. Quote All Variables

# Good
cp "$source" "$destination"

# Bad
cp $source $destination

3. Use = Not ==

# POSIX compliant
if [ "$var" = "value" ]; then

# NOT POSIX (bash-specific)
if [ "$var" == "value" ]; then

4. Use $() for Command Substitution

# Preferred (POSIX)
result=$(command)

# Old style (works but less readable)
result=`command`

5. Avoid Bashisms

Don’t use:

• Arrays: array=(one two)
• [[ test construct
• Process substitution: <(command)
• Brace expansion: {1..10}
• function keyword
• source command (use . instead)
• == operator (use =)
• $RANDOM variable

6. Check Command Existence

if command -v shellcheck >/dev/null 2>&1; then
    echo "ShellCheck is installed"
fi

7. Handle Errors

# Set errexit
set -e

# Or check manually
if ! command; then
    echo "Command failed" >&2
    exit 1
fi

8. Use set -u for Undefined Variables

set -u
# Now accessing undefined variables causes error

Common Portability Issues

1. echo Command

# Portable way to echo without newline
printf '%s' "text without newline"

# echo -n is not portable
echo -n "text"  # Don't use in POSIX sh

# echo with backslashes
printf '%s\n' "text\twith\ttabs"
echo "text\twith\ttabs"  # Behavior varies

2. Array Alternatives

# Instead of arrays, use:

# 1. Positional parameters
set -- one two three
echo "$1"  # one

# 2. Delimited strings
items="one:two:three"
IFS=:
for item in $items; do
    echo "$item"
done

3. String Manipulation

# POSIX parameter expansion
${var#pattern}   # Remove shortest match from beginning
${var##pattern}  # Remove longest match from beginning
${var%pattern}   # Remove shortest match from end
${var%%pattern}  # Remove longest match from end

# NOT POSIX (bash-specific)
${var/pattern/replacement}
${var,,}  # lowercase
${var^^}  # uppercase

4. Arithmetic

# POSIX way
result=$((a + b))

# NOT POSIX (bash-specific)
((a++))
let "a = a + 1"

5. read Command

# POSIX
while IFS= read -r line; do
    echo "$line"
done < file

# Bash-specific flags to avoid:
read -p "prompt"     # Not in POSIX
read -a array        # Not in POSIX
read -t timeout      # Not in POSIX

POSIX Parameter Expansion

${var}              # Value of var
${var:-default}     # Use default if var is unset or null
${var:=default}     # Assign default if var is unset or null
${var:?error}       # Error if var is unset or null
${var:+alternate}   # Use alternate if var is set and not null
${#var}             # Length of var
${var#pattern}      # Remove shortest match from beginning
${var##pattern}     # Remove longest match from beginning
${var%pattern}      # Remove shortest match from end
${var%%pattern}     # Remove longest match from end

Special Variables (POSIX)

$0      # Script name
$1-$9   # Positional parameters
${10}   # Parameters beyond 9 (braces required)
$#      # Number of positional parameters
$*      # All positional parameters (as single word)
$@      # All positional parameters (as separate words)
$$      # Process ID of shell
$!      # PID of last background command
$?      # Exit status of last command

Testing for POSIX Compliance

Use checkbashisms

# Install checkbashisms (Debian/Ubuntu)
apt-get install devscripts

# Check script
checkbashisms script.sh

Use ShellCheck with sh

# Validate as sh script
shellcheck -s sh script.sh

Test with Different Shells

# Test with dash (common /bin/sh)
dash script.sh

# Test with ash
ash script.sh

# Test with ksh
ksh script.sh

Common POSIX Utilities

These utilities are standardized and safe to use in POSIX scripts:

• cat, echo, printf
• grep, sed, awk
• cut, sort, uniq, tr
• head, tail, wc
• find, xargs
• test (same as [ ])
• cd, pwd, ls
• cp, mv, rm, mkdir
• chmod, chown
• read, shift, set, export

Resources

• POSIX Shell Command Language
• Dash as /bin/sh
• Autoconf Portable Shell
• Rich’s sh (POSIX shell) tricks