Home Monitoring a Kubernetes cluster
Post
Cancel

Monitoring a Kubernetes cluster

While there are more complete monitoring solutions (Portainer, Rancher), I just needed a simple script to show information on a single screen. This is a work in progress.

Running this script with the ‘watch’ command allows you to see a constantly updated screen.

1
watch -n 1 script_name.sh

Script

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
#!/bin/bash

K=kubectl
G=grep
POD_file="pods.dat"
NODE_file="nodes.dat"
current_date=$(date)
all="--all-namespaces"

$K get pods -o wide --sort-by="spec.nodeName" > $POD_file

function bargraph () {
  label="$1"
  value="$2"
  hide_if_zero="$3"
  bar=$(printf "%${value}s\n" | tr " " "=")
  printf "%20s %4s %s\n"  $label $value $bar
}

function status_info () {
  info="$1"
  status_to_look_for="$2"
  hide_if_zero="$3"
  count=$(echo "$info" | sort | grep -i $status_to_look_for | wc -l)
  count2=${count:-"0"}
  if ! [[ $count2 == "0" && "$hide_if_zero" == "HIDE" ]]; then
    bargraph "$status_to_look_for" "$count2"
  fi
  #bargraph "$status_to_look_for" "$count2"
}

function banner1 {
  echo "============================================================================================"
  echo "$1"
  echo "--------------------------------------------------------------------------------------------"
}

function display_status {
  banner1 "Status"
  STATS=$(cat pods.dat | awk '{print $3}' | sed 's/^[[:space:]]*//' | tail -n +2)
  status_info "$STATS" "ContainerCreating"
  status_info "$STATS" "Pending"
  status_info "$STATS" "Running"
  status_info "$STATS" "Terminating"
  status_info "$STATS" "Unknown" "HIDE"
  status_info "$STATS" "Failed" "HIDE"
  status_info "$STATS" "CrashLoopBackOff" "HIDE"
  status_info "$STATS" "Completed" "HIDE"
  status_info "$STATS" "ImagePullBackOff" "HIDE"
  status_info "$STATS" "Succeeded" "HIDE"

  echo -e
}

function display_pods {
  banner1 "Pods"
  cat $POD_file
  echo -e
}


function display_num_pods {
  banner1 "Number of pods running on each node"
  $K get nodes $all --no-headers -o custom-columns=NAME:.metadata.name | sort | while read node; do
        cmd="$G -c $node $POD_file"
        count=$($cmd)
        bargraph  "$node" "$count"
  done
  echo -e
}


function display_nodes {
  banner1 "Nodes"
  $K get nodes -o wide > $NODE_file > $NODE_file
  cat $NODE_file
  echo -e
}


function display_node_metrics {
  banner1 "Node Metrics"
  $K top nodes --use-protocol-buffers
  echo -e
}


function display_node_deployments {
  banner1 "Deployments"
  $K get deployments $all
  echo -e
}

function display_service {
  banner1 "Service"
  $K get service $all
  echo -e
}

function display_hpa
{
  banner1 "HPAs"
  $K get hpa -o wide
  echo -e
}

#----------------------------------------------------------------------------------------------------------
echo "Today is $(date)"
echo -e

# Comment out sections or reorder them as needed
display_status
display_num_pods
#display_nodes
display_node_metrics
display_node_deployments
#display_pods
display_hpa
display_service

Example output

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
Today is Sat 19 Nov 2022 09:15:06 PM UTC

============================================================================================
Status
--------------------------------------------------------------------------------------------
   ContainerCreating    0
             Pending    6 ======
             Running   58 ==========================================================
         Terminating    0
             Unknown    0
              Failed    0
    CrashLoopBackOff    0
           Completed    0
    ImagePullBackOff    0
           Succeeded    0

============================================================================================
Number of pods running on each node
--------------------------------------------------------------------------------------------
             master1    0
             worker1   19 ===================
             worker2   20 ====================
             worker3   19 ===================

============================================================================================
Node Metrics
--------------------------------------------------------------------------------------------
NAME      CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%
master1   396m         9%     1699Mi          44%
worker1   300m         7%     656Mi           17%
worker2   265m         6%     660Mi           17%
worker3   193m         4%     677Mi           17%

============================================================================================
HPAs
--------------------------------------------------------------------------------------------
NAME         REFERENCE               TARGETS   MINPODS   MAXPODS   REPLICAS   AGE
php-apache   Deployment/php-apache   5%/5%     1         100       63         4m51s
This post is licensed under CC BY 4.0 by the author.