Text Processing: grep, sed, awk, and jq¶

command 2>/dev/null            # discard errors
command > out.log 2>&1         # stdout and stderr to one file
command 2>&1 | tee run.log     # see output and save it
command | head -5              # the pipeline stops early; later lines are never read

grep -i 'error' app.log                     # case-insensitive
grep -n -C 3 'Traceback' app.log            # line numbers with 3 lines of context
grep -v 'healthcheck' access.log            # invert: exclude matches
grep -c ' 500 ' access.log                  # count matching lines
grep -E 'timeout|refused|reset' app.log     # extended regex: alternation
grep -o 'user_id=[0-9]\+' app.log           # print only the match
grep -rIl 'password' /etc/myapp/            # recursive, skip binaries, filenames only
grep -F '[ERROR]' app.log                   # fixed string: no regex, no escaping brackets
zgrep 'ERROR' app.log.2.gz                  # search compressed rotated logs

sed -n '100,120p' app.log                           # print a line range
sed 's/http:/https:/g' urls.txt                     # substitute every match on each line
sed -i.bak 's/^max_connections = .*/max_connections = 500/' postgresql.conf   # in place, keep a backup
sed '/^\s*#/d; /^\s*$/d' nginx.conf                 # strip comments and blank lines
sed -n '/2026-09-14T10:00/,/2026-09-14T10:15/p' app.log   # lines between two timestamps

# Requests per status code
awk '{print $9}' access.log | sort | uniq -c | sort -rn

# Top 10 client IPs
awk '{print $1}' access.log | sort | uniq -c | sort -rn | head

# Only 5xx responses, printing time, path, and status
awk '$9 ~ /^5/ {print $4, $7, $9}' access.log

# Average response time (last field) per path
awk '{sum[$7] += $NF; n[$7]++} END {for (p in sum) printf "%.3f %s\n", sum[p]/n[p], p}' access.log | sort -rn | head

# Custom separator: users with a login shell from /etc/passwd
awk -F: '$7 !~ /nologin|false/ {print $1, $7}' /etc/passwd

cut -d, -f1,3 inventory.csv            # columns 1 and 3 of a CSV (no quoted commas)
sort -t, -k3,3n inventory.csv          # sort numerically by the third column
sort -u hosts.txt                      # sort and de-duplicate
uniq -c                                # count adjacent duplicates — always sort first
comm -13 <(sort old.txt) <(sort new.txt)   # lines only in new.txt
tr '[:upper:]' '[:lower:]' < names.txt
xargs -n 1 -P 4 ./check-host.sh < hosts.txt   # run 4 checks in parallel

sudo apt install -y jq

# Pretty-print and pick fields
curl -s https://api.github.com/repos/prometheus/prometheus/releases/latest | jq '{tag: .tag_name, published: .published_at}'

# Raw strings for use in shell variables
TAG=$(curl -s https://api.github.com/repos/prometheus/prometheus/releases/latest | jq -r .tag_name)

# Pods that are not Running, with their node
kubectl get pods -A -o json \
  | jq -r '.items[] | select(.status.phase != "Running") | [.metadata.namespace, .metadata.name, .status.phase, .spec.nodeName] | @tsv'

# Container images across all pods, counted
kubectl get pods -A -o json | jq -r '.items[].spec.containers[].image' | sort | uniq -c | sort -rn

# Count structured log events by level
jq -r '.level' app.json.log | sort | uniq -c

# Filter JSON log lines by a field
jq -c 'select(.level == "ERROR" and .service == "orders-api")' app.json.log

# Build JSON safely from shell variables — never by string concatenation
jq -n --arg host "$HOSTNAME" --argjson disk "$(df --output=pcent / | tail -1 | tr -dc 0-9)" \
  '{host: $host, disk_used_percent: $disk}'

since=$(date -u -d '15 minutes ago' '+%d/%b/%Y:%H:%M')
awk -v since="$since" '
  { ts = substr($4, 2, 17) }            # 14/Sep/2026:10:02
  ts >= since && $9 ~ /^5/ { count[$7" "$9]++ }
  END { for (k in count) print count[k], k }
' /var/log/nginx/access.log | sort -rn | head