string-manipulation-mastery

CRITICAL GUIDELINES

Windows File Path Requirements

MANDATORY: Always Use Backslashes on Windows for File Paths

When using Edit or Write tools on Windows, you MUST use backslashes (\) in file paths, NOT forward slashes (/).

---

Bash String Manipulation Mastery (2025)

Overview

Comprehensive guide to string manipulation in bash using parameter expansion, pattern matching, regular expressions, and built-in transformations. Master these techniques to avoid spawning external processes like sed, awk, or cut for simple operations.

Parameter Expansion Basics

String Length

#!/usr/bin/env bash
set -euo pipefail

str="Hello, World!"

# Get length
echo "${#str}"  # 13

# Length of array element
arr=("short" "much longer string")
echo "${#arr[0]}"  # 5
echo "${#arr[1]}"  # 18

# Number of array elements (not string length)
echo "${#arr[@]}"  # 2

Substring Extraction

#!/usr/bin/env bash
set -euo pipefail

str="Hello, World!"

# ${var:offset} - from offset to end
echo "${str:7}"     # World!

# ${var:offset:length} - from offset, length chars
echo "${str:0:5}"   # Hello
echo "${str:7:5}"   # World

# Negative offset (from end) - note the space or parentheses
echo "${str: -6}"      # World!
echo "${str:(-6)}"     # World!
echo "${str: -6:5}"    # World

# Last N characters
last_n() {
    local str="$1" n="$2"
    echo "${str: -$n}"
}
last_n "Hello" 3  # llo

# Extract between positions
between() {
    local str="$1" start="$2" end="$3"
    echo "${str:start:$((end - start))}"
}
between "0123456789" 3 7  # 3456

Default Values

#!/usr/bin/env bash
set -euo pipefail

# ${var:-default} - use default if unset or empty
name="${1:-Anonymous}"
echo "Hello, $name"

# ${var:=default} - assign default if unset or empty
: "${CONFIG_FILE:=/etc/app.conf}"

# ${var:+alternate} - use alternate if var IS set
debug_flag="${DEBUG:+--verbose}"

# ${var:?error} - exit with error if unset or empty
: "${REQUIRED_VAR:?REQUIRED_VAR must be set}"

# Without colon - only checks if unset (empty is OK)
# ${var-default}, ${var=default}, ${var+alt}, ${var?err}

# Practical example: config with defaults
setup_config() {
    : "${DB_HOST:=localhost}"
    : "${DB_PORT:=5432}"
    : "${DB_NAME:=myapp}"
    : "${DB_USER:=postgres}"
}

Indirect Expansion

#!/usr/bin/env bash
set -euo pipefail

# ${!var} - indirect reference
config_host="server.example.com"
config_port="8080"

key="config_host"
echo "${!key}"  # server.example.com

# Iterate over related variables
for suffix in host port; do
    var="config_$suffix"
    echo "$suffix = ${!var}"
done

# Get all variable names matching pattern
# ${!prefix@} or ${!prefix*}
for var in "${!config_@}"; do
    echo "$var = ${!var}"
done

# Array indirection
arr=(a b c d e)
idx=2
echo "${arr[$idx]}"  # c

# Indirect array reference (Bash 4.3+ nameref)
get_array_element() {
    local -n arr_ref="$1"
    local idx="$2"
    echo "${arr_ref[$idx]}"
}
get_array_element arr 3  # d

Pattern Matching

Prefix Removal

#!/usr/bin/env bash
set -euo pipefail

path="/home/user/documents/file.tar.gz"

# ${var#pattern} - remove shortest prefix match
echo "${path#*/}"     # home/user/documents/file.tar.gz

# ${var##pattern} - remove longest prefix match
echo "${path##*/}"    # file.tar.gz (basename)

# Remove extension
filename="archive.tar.gz"
echo "${filename#*.}"   # tar.gz (first . onwards)
echo "${filename##*.}"  # gz (last extension only)

# Remove prefix string
url="https://example.com/path"
echo "${url#https://}"  # example.com/path

# Practical: Get file extension
get_extension() {
    local file="$1"
    echo "${file##*.}"
}

# Practical: Strip leading zeros
strip_leading_zeros() {
    local num="$1"
    echo "${num#"${num%%[!0]*}"}"
}
strip_leading_zeros "000123"  # 123

Suffix Removal

#!/usr/bin/env bash
set -euo pipefail

path="/home/user/documents/file.tar.gz"

# ${var%pattern} - remove shortest suffix match
echo "${path%/*}"     # /home/user/documents (dirname)

# ${var%%pattern} - remove longest suffix match
echo "${path%%/*}"    # (empty - removes everything)

# Remove extension
filename="archive.tar.gz"
echo "${filename%.*}"   # archive.tar
echo "${filename%%.*}"  # archive

# Practical: Get directory
dirname="${path%/*}"

# Practical: Remove file extension
basename="${path##*/}"
name_without_ext="${basename%.*}"

# Combined: Change extension
change_extension() {
    local file="$1" new_ext="$2"
    echo "${file%.*}.$new_ext"
}
change_extension "doc.txt" "md"  # doc.md

Pattern Substitution

#!/usr/bin/env bash
set -euo pipefail

str="hello hello hello"

# ${var/pattern/replacement} - replace first match
echo "${str/hello/hi}"     # hi hello hello

# ${var//pattern/replacement} - replace all matches
echo "${str//hello/hi}"    # hi hi hi

# ${var/#pattern/replacement} - replace if at start
echo "${str/#hello/hi}"    # hi hello hello

# ${var/%pattern/replacement} - replace if at end
echo "${str/%hello/goodbye}"  # hello hello goodbye

# Delete pattern (empty replacement)
echo "${str//hello/}"      # "   " (just spaces)

# Practical: Sanitize filename
sanitize_filename() {
    local name="$1"
    # Replace spaces with underscores
    name="${name// /_}"
    # Remove special characters
    name="${name//[^a-zA-Z0-9._-]/}"
    echo "$name"
}
sanitize_filename "My File (2024).txt"  # My_File_2024.txt

# Practical: Path manipulation
normalize_path() {
    local path="$1"
    # Remove double slashes
    while [[ "$path" == *//* ]]; do
        path="${path//\/\//\/}"
    done
    # Remove trailing slash
    echo "${path%/}"
}

Case Transformation

Basic Case Changes

#!/usr/bin/env bash
set -euo pipefail

str="Hello World"

# Lowercase first character
echo "${str,}"      # hello World

# Lowercase all
echo "${str,,}"     # hello world

# Uppercase first character
echo "${str^}"      # Hello World (already uppercase)

str2="hello world"
echo "${str2^}"     # Hello world

# Uppercase all
echo "${str,,}"     # hello world
echo "${str2^^}"    # HELLO WORLD

# Toggle case (Bash 4.4+)
echo "${str~~}"     # hELLO wORLD

Pattern-Based Case Changes

#!/usr/bin/env bash
set -euo pipefail

str="hello world"

# Uppercase only matching pattern
echo "${str^^[aeiou]}"  # hEllO wOrld

# Lowercase only matching pattern
str2="HELLO WORLD"
echo "${str2,,[AEIOU]}"  # HeLLo WoRLD

# Practical: Title case
title_case() {
    local str="$1"
    local result=""
    local capitalize=true

    for ((i=0; i<${#str}; i++)); do
        local char="${str:$i:1}"
        if [[ "$char" == " " ]]; then
            result+="$char"
            capitalize=true
        elif $capitalize; then
            result+="${char^}"
            capitalize=false
        else
            result+="${char,}"
        fi
    done

    echo "$result"
}
title_case "hello WORLD from BASH"  # Hello World From Bash

Regular Expressions

Bash Regex Matching

#!/usr/bin/env bash
set -euo pipefail

# =~ operator for regex matching
str="Hello World 123"

if [[ "$str" =~ ^Hello ]]; then
    echo "Starts with Hello"
fi

if [[ "$str" =~ [0-9]+ ]]; then
    echo "Contains numbers"
fi

# Capture groups with BASH_REMATCH
email="user@example.com"
if [[ "$email" =~ ^([^@]+)@(.+)$ ]]; then
    echo "User: ${BASH_REMATCH[1]}"    # user
    echo "Domain: ${BASH_REMATCH[2]}"  # example.com
    echo "Full match: ${BASH_REMATCH[0]}"  # user@example.com
fi

# Store regex in variable (avoids quoting issues)
pattern='^[0-9]{4}-[0-9]{2}-[0-9]{2}$'
date="2024-03-15"
if [[ "$date" =~ $pattern ]]; then
    echo "Valid date format"
fi

# Multiple capture groups
log_line='2024-03-15 10:30:45 ERROR Connection failed'
pattern='^([0-9-]+) ([0-9:]+) ([A-Z]+) (.+)$'
if [[ "$log_line" =~ $pattern ]]; then
    date="${BASH_REMATCH[1]}"
    time="${BASH_REMATCH[2]}"
    level="${BASH_REMATCH[3]}"
    message="${BASH_REMATCH[4]}"
fi

Practical Regex Patterns

#!/usr/bin/env bash
set -euo pipefail

# Email validation
is_valid_email() {
    local email="$1"
    local pattern='^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
    [[ "$email" =~ $pattern ]]
}

# IP address validation
is_valid_ip() {
    local ip="$1"
    local octet='(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)'
    local pattern="^${octet}\.${octet}\.${octet}\.${octet}$"
    [[ "$ip" =~ $pattern ]]
}

# URL parsing
parse_url() {
    local url="$1"
    local pattern='^(https?|ftp)://([^/:]+)(:([0-9]+))?(/.*)?$'

    if [[ "$url" =~ $pattern ]]; then
        echo "Protocol: ${BASH_REMATCH[1]}"
        echo "Host: ${BASH_REMATCH[2]}"
        echo "Port: ${BASH_REMATCH[4]:-default}"
        echo "Path: ${BASH_REMATCH[5]:-/}"
    else
        echo "Invalid URL" >&2
        return 1
    fi
}

# Semantic version parsing
parse_semver() {
    local version="$1"
    local pattern='^v?([0-9]+)\.([0-9]+)\.([0-9]+)(-([a-zA-Z0-9.-]+))?(\+([a-zA-Z0-9.-]+))?$'

    if [[ "$version" =~ $pattern ]]; then
        echo "Major: ${BASH_REMATCH[1]}"
        echo "Minor: ${BASH_REMATCH[2]}"
        echo "Patch: ${BASH_REMATCH[3]}"
        echo "Prerelease: ${BASH_REMATCH[5]:-none}"
        echo "Build: ${BASH_REMATCH[7]:-none}"
    fi
}

String Splitting and Joining

Split String to Array

#!/usr/bin/env bash
set -euo pipefail

# Using IFS and read
str="one,two,three,four"

IFS=',' read -ra arr <<< "$str"
echo "${arr[0]}"  # one
echo "${arr[2]}"  # three

# Multi-character delimiter (using parameter expansion)
str="one||two||three"
arr=()
while [[ "$str" == *"||"* ]]; do
    arr+=("${str%%||*}")
    str="${str#*||}"
done
arr+=("$str")

# Using mapfile (newline-delimited)
mapfile -t lines <<< "$(echo -e "line1\nline2\nline3")"

# Split on any whitespace
str="one   two     three"
read -ra arr <<< "$str"  # IFS defaults to whitespace

Join Array to String

#!/usr/bin/env bash
set -euo pipefail

arr=("one" "two" "three" "four")

# Join with delimiter using IFS
join_by() {
    local IFS="$1"
    shift
    echo "$*"
}
join_by ',' "${arr[@]}"  # one,two,three,four
join_by ' | ' "${arr[@]}"  # one | two | three | four

# Alternative using printf
join_array() {
    local delim="$1"
    shift
    local first="$1"
    shift
    printf '%s' "$first" "${@/#/$delim}"
}
join_array ',' "${arr[@]}"

# Join with custom format
printf '"%s" ' "${arr[@]}"  # "one" "two" "three" "four"
printf '%s\n' "${arr[@]}"   # One per line

Text Processing Without External Commands

Detailed examples replacing common sed, awk, cut, tr, and basename patterns with Bash parameter expansion, loops, arrays, and built-ins live in references/text-processing-without-external-commands.md. Load that reference when optimizing hot loops or avoiding process-spawn overhead.

Extended Globbing

Enable and Use

#!/usr/bin/env bash
set -euo pipefail
shopt -s extglob

# Extended patterns:
# ?(pattern) - 0 or 1 occurrence
# *(pattern) - 0 or more occurrences
# +(pattern) - 1 or more occurrences
# @(pattern) - exactly 1 occurrence
# !(pattern) - anything except pattern

# Match files
ls *.@(jpg|png|gif)      # Images only
ls !(*.bak|*.tmp)        # Exclude backup/temp files
ls +([0-9]).txt          # Files starting with digits

# String manipulation
str="   hello   world   "

# Remove leading whitespace
echo "${str##+([[:space:]])}"  # "hello   world   "

# Remove all whitespace
echo "${str//+([[:space:]])/ }"  # " hello world "

# Remove multiple extensions
file="archive.tar.gz.bak"
echo "${file%.@(tar|gz|bak)*}"  # archive

# Match alternatives
case "$response" in
    @(yes|y|Y|YES))
        echo "Affirmative"
        ;;
    @(no|n|N|NO))
        echo "Negative"
        ;;
esac

Practical Extended Glob Patterns

#!/usr/bin/env bash
set -euo pipefail
shopt -s extglob

# Clean backup files
rm -f *.@(bak|backup|orig|~)

# Find source files only
ls *.@(c|cpp|h|hpp|cc)

# Exclude certain patterns
for file in !(test_*|_*).py; do
    process "$file"
done

# Match complex version strings
version_pattern='+([0-9]).+([0-9]).+([0-9])?(-+([a-z0-9]))'
if [[ "$version" == $version_pattern ]]; then
    echo "Valid version"
fi

# Remove redundant characters
clean_string() {
    local str="$1"
    # Remove repeated spaces
    echo "${str//+([[:space:]])/ }"
}

# Match optional parts
file_pattern='*.@(test|spec)?.@(js|ts)'
# Matches: file.js, file.ts, file.test.js, file.spec.ts, etc.

Bash 5.3+ String Features

In-Shell Substitution for Strings

#!/usr/bin/env bash
# Requires Bash 5.3+
set -euo pipefail

# No-fork string operations
result=${ echo "${str^^}"; }  # Uppercase without subshell

# REPLY syntax for string building
build_path() {
    local parts=("$@")
    REPLY=""

    for part in "${parts[@]}"; do
        ${| REPLY+="${REPLY:+/}$part"; }
    done
}

# Efficient string accumulation
accumulate() {
    local -n result="$1"
    shift

    for item in "$@"; do
        ${| result+="$item"; }
    done
}

Performance Tips

Avoid Subshells for Simple Operations

#!/usr/bin/env bash
set -euo pipefail

str="hello world"

# ✗ SLOW - spawns external process
basename=$(basename "$path")
dirname=$(dirname "$path")
upper=$(echo "$str" | tr 'a-z' 'A-Z')
len=$(echo -n "$str" | wc -c)

# ✓ FAST - pure bash
basename="${path##*/}"
dirname="${path%/*}"
upper="${str^^}"
len="${#str}"

Batch String Operations

#!/usr/bin/env bash
set -euo pipefail

# ✗ SLOW - multiple expansions
str="$input"
str="${str//  / }"
str="${str#"${str%%[![:space:]]*}"}"
str="${str%"${str##*[![:space:]]}"}"
str="${str,,}"

# ✓ BETTER - single function call
normalize_string() {
    local str="$1"
    str="${str//  / }"
    str="${str#"${str%%[![:space:]]*}"}"
    str="${str%"${str##*[![:space:]]}"}"
    echo "${str,,}"
}
result=$(normalize_string "$input")

Resources

• Bash Reference - Shell Parameter Expansion
• Bash Reference - Pattern Matching
• BashFAQ/100 - String manipulation
• Bash Hackers - Parameter Expansion

---

Master bash string manipulation to write efficient scripts without external dependencies.