Kubernetes/OpenShift Troubleshooting Guide

1. IDENTIFY → What is the problem?
2. GATHER  → Collect relevant information
3. ANALYZE → Examine logs, events, metrics
4. ISOLATE → Narrow down the root cause
5. RESOLVE → Fix and verify the solution

$ kubectl get pods
NAME                    READY   STATUS    RESTARTS   AGE
myapp-5d4b8c7f9-abc12   0/1     Pending   0          5m

# Check pod events
kubectl describe pod myapp-5d4b8c7f9-abc12

# Common causes in events:
# - "Insufficient cpu/memory"
# - "No nodes available"
# - "PersistentVolumeClaim not found"
# - "ImagePullBackOff"

# Check node resources
kubectl top nodes
kubectl describe nodes

# Check resource requests
kubectl get pod myapp-5d4b8c7f9-abc12 -o yaml | grep -A 5 resources

# Solutions:
# - Add more nodes
# - Reduce resource requests
# - Delete unused pods
# - Scale down other deployments

# Check node labels
kubectl get nodes --show-labels

# Check pod node selector
kubectl get pod myapp-5d4b8c7f9-abc12 -o yaml | grep -A 5 nodeSelector

# Solution: Add required labels to nodes
kubectl label nodes node1 disktype=ssd

# Check PVC status
kubectl get pvc

# Check PV availability
kubectl get pv

# Solution: Create PV or fix storage class

$ kubectl get pods
NAME                    READY   STATUS             RESTARTS   AGE
myapp-5d4b8c7f9-abc12   0/1     CrashLoopBackOff   5          10m

# Check logs
kubectl logs myapp-5d4b8c7f9-abc12
kubectl logs myapp-5d4b8c7f9-abc12 --previous

# Check events
kubectl describe pod myapp-5d4b8c7f9-abc12

# Common causes:
# - Application error
# - Missing dependencies
# - Configuration error
# - Failed health checks
# - Insufficient permissions

# Debug with shell access
kubectl run debug --rm -it --image=busybox -- sh

# Or debug the actual pod
kubectl debug myapp-5d4b8c7f9-abc12 -it --image=busybox

# Check application configuration
kubectl get configmap myapp-config -o yaml
kubectl get secret myapp-secret -o yaml

# Adjust probe settings
livenessProbe:
  httpGet:
    path: /health
    port: 8080
  initialDelaySeconds: 60  # Increase if app takes time to start
  periodSeconds: 10
  timeoutSeconds: 5
  failureThreshold: 3

# Check security context
kubectl get pod myapp-5d4b8c7f9-abc12 -o yaml | grep -A 10 securityContext

# For OpenShift, check SCC
oc describe pod myapp-5d4b8c7f9-abc12 | grep scc

$ kubectl get pods
NAME                    READY   STATUS             RESTARTS   AGE
myapp-5d4b8c7f9-abc12   0/1     ImagePullBackOff   0          2m

# Check events
kubectl describe pod myapp-5d4b8c7f9-abc12

# Common errors:
# - "Failed to pull image"
# - "manifest unknown"
# - "unauthorized"
# - "connection refused"

# Verify image exists
docker pull myapp:v1.0

# Check image name in deployment
kubectl get deployment myapp -o yaml | grep image:

# Fix: Update to correct image
kubectl set image deployment/myapp myapp=myapp:v1.0

# Create docker registry secret
kubectl create secret docker-registry regcred \
  --docker-server=docker.io \
  --docker-username=myuser \
  --docker-password=mypass \
  --docker-email=myemail@example.com

# Add to deployment
kubectl patch deployment myapp -p '
{
  "spec": {
    "template": {
      "spec": {
        "imagePullSecrets": [{"name": "regcred"}]
      }
    }
  }
}'

# Test connectivity from node
ssh node1
curl -I https://registry.example.com

# Check proxy settings
kubectl get configmap -n kube-system kube-proxy -o yaml

$ kubectl get pods
NAME                    READY   STATUS      RESTARTS   AGE
myapp-5d4b8c7f9-abc12   0/1     OOMKilled   3          5m

# Check memory usage
kubectl top pod myapp-5d4b8c7f9-abc12

# Check memory limits
kubectl get pod myapp-5d4b8c7f9-abc12 -o yaml | grep -A 5 resources

# Check events
kubectl describe pod myapp-5d4b8c7f9-abc12

# Increase memory limits
resources:
  requests:
    memory: "512Mi"
  limits:
    memory: "1Gi"  # Increased from 512Mi

# Or investigate memory leak in application

# Check service
kubectl get svc myapp-service
kubectl describe svc myapp-service

# Check endpoints
kubectl get endpoints myapp-service

# If no endpoints, pods don't match selector
kubectl get pods --show-labels
kubectl get svc myapp-service -o yaml | grep selector

# Fix pod labels
kubectl label pods myapp-5d4b8c7f9-abc12 app=myapp

# Or fix service selector
kubectl patch svc myapp-service -p '{"spec":{"selector":{"app":"myapp"}}}'

# Check service ports
kubectl get svc myapp-service -o yaml

# Check container ports
kubectl get pod myapp-5d4b8c7f9-abc12 -o yaml | grep containerPort

# Fix service port
kubectl patch svc myapp-service -p '
{
  "spec": {
    "ports": [{
      "port": 80,
      "targetPort": 8080
    }]
  }
}'

# Check network policies
kubectl get networkpolicy

# Test connectivity
kubectl run test --rm -it --image=busybox -- wget -O- http://myapp-service

# Test DNS from pod
kubectl run test --rm -it --image=busybox -- nslookup myapp-service

# Check CoreDNS pods
kubectl get pods -n kube-system -l k8s-app=kube-dns

# Check CoreDNS logs
kubectl logs -n kube-system -l k8s-app=kube-dns

# Restart CoreDNS
kubectl rollout restart deployment/coredns -n kube-system

# Check CoreDNS ConfigMap
kubectl get configmap coredns -n kube-system -o yaml

# Verify service exists
kubectl get svc myapp-service

# Check ingress
kubectl get ingress
kubectl describe ingress myapp-ingress

# Check ingress controller
kubectl get pods -n ingress-nginx
kubectl logs -n ingress-nginx -l app.kubernetes.io/name=ingress-nginx

# Test backend service
kubectl run test --rm -it --image=curlimages/curl -- \
  curl http://myapp-service.default.svc.cluster.local

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: myapp-ingress
spec:
  ingressClassName: nginx  # Add this
  rules:
  - host: myapp.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: myapp-service
            port:
              number: 80

# Get ingress IP
kubectl get ingress myapp-ingress

# Verify DNS
nslookup myapp.example.com

# Update DNS A record to point to ingress IP

# Check PVC
kubectl get pvc
kubectl describe pvc myapp-pvc

# Check storage class
kubectl get storageclass

# Check PV availability
kubectl get pv

# Set default storage class
kubectl patch storageclass standard -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

# Or specify in PVC
kubectl patch pvc myapp-pvc -p '{"spec":{"storageClassName":"standard"}}'

# Create PV manually
kubectl apply -f - <<EOF
apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-manual
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  hostPath:
    path: /mnt/data
EOF

# Check pod events
kubectl describe pod myapp-5d4b8c7f9-abc12

# Common errors:
# - "Unable to mount volumes"
# - "Volume is already attached"
# - "Permission denied"

# Find which pod is using the volume
kubectl get pods -o json | jq -r '.items[] | 
  select(.spec.volumes[]?.persistentVolumeClaim.claimName=="myapp-pvc") | 
  .metadata.name'

# Delete the old pod
kubectl delete pod <old-pod-name>

# Set fsGroup in security context
securityContext:
  fsGroup: 2000
  runAsUser: 1000

# Check node status
kubectl get nodes
kubectl describe node node1

# Check node conditions
kubectl get node node1 -o yaml | grep -A 10 conditions

# SSH to node and check
ssh node1
systemctl status kubelet
journalctl -u kubelet -n 100

# On the node
systemctl start kubelet
systemctl enable kubelet

# Check kubelet logs
journalctl -u kubelet -f

# Check disk usage
df -h

# Clean up
docker system prune -a
kubectl delete pods --field-selector=status.phase=Failed -A

# Increase disk or add new node

# Check CNI plugin
kubectl get pods -n kube-system | grep -E 'calico|flannel|weave'

# Restart CNI pods
kubectl delete pods -n kube-system -l k8s-app=calico-node

# Check node resources
kubectl top nodes
kubectl describe node node1

# Find resource-hungry pods
kubectl top pods --all-namespaces --sort-by=memory
kubectl top pods --all-namespaces --sort-by=cpu

# Set resource limits
kubectl set resources deployment myapp --limits=cpu=500m,memory=512Mi

# Implement HPA
kubectl autoscale deployment myapp --min=2 --max=10 --cpu-percent=70

# Add more nodes or scale down workloads

# Check rollout status
kubectl rollout status deployment/myapp

# Check deployment
kubectl describe deployment myapp

# Check replica sets
kubectl get rs
kubectl describe rs myapp-5d4b8c7f9

# Check events
kubectl get events --sort-by=.metadata.creationTimestamp

# Scale down or add resources
kubectl scale deployment myapp --replicas=2

# Check new pods
kubectl get pods -l app=myapp

# Fix image or add pull secret
kubectl set image deployment/myapp myapp=myapp:v1.0

# Check pod logs
kubectl logs -l app=myapp

# Adjust probe or fix application
kubectl patch deployment myapp -p '
{
  "spec": {
    "template": {
      "spec": {
        "containers": [{
          "name": "myapp",
          "readinessProbe": {
            "initialDelaySeconds": 60
          }
        }]
      }
    }
  }
}'

# View rollout history
kubectl rollout history deployment/myapp

# Rollback to previous version
kubectl rollout undo deployment/myapp

# Rollback to specific revision
kubectl rollout undo deployment/myapp --to-revision=2

# Pause rollout
kubectl rollout pause deployment/myapp

# Resume rollout
kubectl rollout resume deployment/myapp

# Check current user
kubectl auth whoami

# Check permissions
kubectl auth can-i create pods
kubectl auth can-i create pods --as=system:serviceaccount:default:myapp-sa

# Check role bindings
kubectl get rolebindings
kubectl describe rolebinding myapp-binding

apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: pod-reader
  namespace: default
rules:
- apiGroups: [""]
  resources: ["pods", "pods/log"]
  verbs: ["get", "list", "watch"]

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: read-pods
  namespace: default
subjects:
- kind: ServiceAccount
  name: myapp-sa
  namespace: default
roleRef:
  kind: Role
  name: pod-reader
  apiGroup: rbac.authorization.k8s.io

# Check SCC
oc describe pod myapp-5d4b8c7f9-abc12 | grep scc

# Add SCC to service account
oc adm policy add-scc-to-user anyuid -z myapp-sa

# Check resource usage
kubectl top pods
kubectl top nodes

# Check HPA status
kubectl get hpa

# Check pod distribution
kubectl get pods -o wide

# Check network latency
kubectl run test --rm -it --image=nicolaka/netshoot -- \
  ping myapp-service.default.svc.cluster.local

# Manual scaling
kubectl scale deployment myapp --replicas=5

# Enable HPA
kubectl autoscale deployment myapp --min=3 --max=10 --cpu-percent=70

resources:
  requests:
    cpu: "500m"
    memory: "512Mi"
  limits:
    cpu: "1000m"
    memory: "1Gi"

# Add Redis sidecar
containers:
- name: redis
  image: redis:6
  ports:
  - containerPort: 6379

# Check if pod is running
kubectl get pods

# Try different log commands
kubectl logs myapp-5d4b8c7f9-abc12
kubectl logs myapp-5d4b8c7f9-abc12 --previous
kubectl logs myapp-5d4b8c7f9-abc12 -c container-name

# Check logging infrastructure
kubectl get pods -n kube-system | grep -E 'fluentd|filebeat|logstash'

# Ensure application logs to stdout/stderr
# Check log rotation settings
# Verify logging agent is running
kubectl get daemonset -n kube-system

# Check metrics server
kubectl get deployment metrics-server -n kube-system
kubectl top nodes  # Should work if metrics server is running

# Check metrics server logs
kubectl logs -n kube-system -l k8s-app=metrics-server

# Install metrics server
kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

# For development clusters, may need to disable TLS
kubectl patch deployment metrics-server -n kube-system --type='json' \
  -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--kubelet-insecure-tls"}]'

# Get all resources
kubectl get all -A

# Check events
kubectl get events --sort-by=.metadata.creationTimestamp -A

# Check resource usage
kubectl top nodes
kubectl top pods -A

# Check cluster info
kubectl cluster-info
kubectl version

# Check component status
kubectl get componentstatuses

# Check API server
kubectl get --raw /healthz
kubectl get --raw /readyz

# Get pod YAML
kubectl get pod myapp-5d4b8c7f9-abc12 -o yaml

# Get pod JSON with jq
kubectl get pod myapp-5d4b8c7f9-abc12 -o json | jq '.status'

# Watch resources
kubectl get pods -w
kubectl get events -w

# Port forward for debugging
kubectl port-forward pod/myapp-5d4b8c7f9-abc12 8080:8080

# Execute commands in pod
kubectl exec -it myapp-5d4b8c7f9-abc12 -- /bin/bash
kubectl exec myapp-5d4b8c7f9-abc12 -- env
kubectl exec myapp-5d4b8c7f9-abc12 -- cat /etc/resolv.conf

# Copy files
kubectl cp myapp-5d4b8c7f9-abc12:/app/logs/app.log ./app.log

# Debug with ephemeral container (K8s 1.23+)
kubectl debug myapp-5d4b8c7f9-abc12 -it --image=busybox

# Debug containers
kubectl run debug --rm -it --image=nicolaka/netshoot -- bash
kubectl run debug --rm -it --image=busybox -- sh

# Useful images:
# - nicolaka/netshoot (network debugging)
# - busybox (basic utilities)
# - curlimages/curl (HTTP testing)
# - alpine (lightweight with package manager)

# Inside debug pod
nslookup service-name
ping service-name
curl http://service-name
traceroute service-name
netstat -tulpn
ss -tulpn
tcpdump -i any port 80