From 714cf09f1c01d43f1d1095bef11e2d696a6184e3 Mon Sep 17 00:00:00 2001 From: Jordi Sanfeliu Date: Tue, 20 Dec 2016 10:13:57 +0100 Subject: [PATCH] added a complete graph for IPMI sensors using the 'ipmitool' command --- Changes | 2 + lib/ipmi.pm | 462 +++++++++++++++++++++++++++++++++++++++++++++++++ monitorix | 8 + monitorix.conf | 27 ++- 4 files changed, 498 insertions(+), 1 deletion(-) create mode 100644 lib/ipmi.pm diff --git a/Changes b/Changes index 754671a..2448e59 100644 --- a/Changes +++ b/Changes @@ -1,5 +1,7 @@ 3.N.N - DD-MMM-YYYY ==================== +- Added a complete graph for IPMI sensors using the 'ipmitool' command. + (suggested by Frank Dijcks, fd AT fdsystems.nl) - Improved the 'system.pm' graph with more detailed information about processes (sleeping, waiting I/O, zombie, stopped, paging and running). It also includes two new graphs: entropy and uptime. diff --git a/lib/ipmi.pm b/lib/ipmi.pm new file mode 100644 index 0000000..0a3ae5d --- /dev/null +++ b/lib/ipmi.pm @@ -0,0 +1,462 @@ +# +# Monitorix - A lightweight system monitoring tool. +# +# Copyright (C) 2005-2016 by Jordi Sanfeliu +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +package ipmi; + +use strict; +use warnings; +use Monitorix; +use RRDs; +use POSIX qw(strftime); +use Exporter 'import'; +our @EXPORT = qw(ipmi_init ipmi_update ipmi_cgi); + +sub ipmi_init { + my $myself = (caller(0))[3]; + my ($package, $config, $debug) = @_; + my $rrd = $config->{base_lib} . $package . ".rrd"; + my $ipmi = $config->{ipmi}; + + my $info; + my @ds; + my @rra; + my @tmp; + my $n; + + my @average; + my @min; + my @max; + my @last; + + if(-e $rrd) { + $info = RRDs::info($rrd); + for my $key (keys %$info) { + if(index($key, 'ds[') == 0) { + if(index($key, '.type') != -1) { + push(@ds, substr($key, 3, index($key, ']') - 3)); + } + } + if(index($key, 'rra[') == 0) { + if(index($key, '.rows') != -1) { + push(@rra, substr($key, 4, index($key, ']') - 4)); + } + } + } + if(scalar(@ds) / 9 != scalar(my @fl = split(',', $ipmi->{list}))) { + logger("$myself: Detected size mismatch between 'list' (" . scalar(my @fl = split(',', $ipmi->{list})) . ") and $rrd (" . scalar(@ds) / 9 . "). Resizing it accordingly. All historical data will be lost. Backup file created."); + rename($rrd, "$rrd.bak"); + } + if(scalar(@rra) < 12 + (4 * $config->{max_historic_years})) { + logger("$myself: Detected size mismatch between 'max_historic_years' (" . $config->{max_historic_years} . ") and $rrd (" . ((scalar(@rra) -12) / 4) . "). Resizing it accordingly. All historical data will be lost. Backup file created."); + rename($rrd, "$rrd.bak"); + } + } + + if(!(-e $rrd)) { + logger("Creating '$rrd' file."); + for($n = 1; $n <= $config->{max_historic_years}; $n++) { + push(@average, "RRA:AVERAGE:0.5:1440:" . (365 * $n)); + push(@min, "RRA:MIN:0.5:1440:" . (365 * $n)); + push(@max, "RRA:MAX:0.5:1440:" . (365 * $n)); + push(@last, "RRA:LAST:0.5:1440:" . (365 * $n)); + } + for($n = 0; $n < scalar(my @fl = split(',', $ipmi->{list})); $n++) { + push(@tmp, "DS:ipmi" . $n . "_s1:GAUGE:120:0:U"); + push(@tmp, "DS:ipmi" . $n . "_s2:GAUGE:120:0:U"); + push(@tmp, "DS:ipmi" . $n . "_s3:GAUGE:120:0:U"); + push(@tmp, "DS:ipmi" . $n . "_s4:GAUGE:120:0:U"); + push(@tmp, "DS:ipmi" . $n . "_s5:GAUGE:120:0:U"); + push(@tmp, "DS:ipmi" . $n . "_s6:GAUGE:120:0:U"); + push(@tmp, "DS:ipmi" . $n . "_s7:GAUGE:120:0:U"); + push(@tmp, "DS:ipmi" . $n . "_s8:GAUGE:120:0:U"); + push(@tmp, "DS:ipmi" . $n . "_s9:GAUGE:120:0:U"); + } + eval { + RRDs::create($rrd, + "--step=60", + @tmp, + "RRA:AVERAGE:0.5:1:1440", + "RRA:AVERAGE:0.5:30:336", + "RRA:AVERAGE:0.5:60:744", + @average, + "RRA:MIN:0.5:1:1440", + "RRA:MIN:0.5:30:336", + "RRA:MIN:0.5:60:744", + @min, + "RRA:MAX:0.5:1:1440", + "RRA:MAX:0.5:30:336", + "RRA:MAX:0.5:60:744", + @max, + "RRA:LAST:0.5:1:1440", + "RRA:LAST:0.5:30:336", + "RRA:LAST:0.5:60:744", + @last, + ); + }; + my $err = RRDs::error; + if($@ || $err) { + logger("$@") unless !$@; + if($err) { + logger("ERROR: while creating $rrd: $err"); + if($err eq "RRDs::error") { + logger("... is the RRDtool Perl package installed?"); + } + } + return; + } + } + + push(@{$config->{func_update}}, $package); + logger("$myself: Ok") if $debug; +} + +sub ipmi_update { + my $myself = (caller(0))[3]; + my ($package, $config, $debug) = @_; + my $rrd = $config->{base_lib} . $package . ".rrd"; + my $ipmi = $config->{ipmi}; + my $args = $ipmi->{extra_args} || ""; + + my @sens; + + my $n; + my $str; + my $rrdata = "N"; + + if(!open(IN, "ipmitool sdr $args |")) { + logger("$myself: unable to execute 'ipmitool'. $!"); + return; + } + my @data = ; + close(IN); + + my $e = 0; + while($e < scalar(my @sl = split(',', $ipmi->{list}))) { + my $e2 = 0; + foreach my $i (split(',', $ipmi->{desc}->{$e})) { + my $unit; + + $sens[$e][$e2] = 0 unless defined $sens[$e][$e2]; + $str = trim($i); + $unit = $ipmi->{units}->{$e}; + foreach(@data) { + if(/^($str)\s+\|\s+(\d+\.*\d*)\s+$unit\s+/) { + print "'$1' '$2'\n"; + $sens[$e][$e2] = $2; + } + } + $e2++; + } + $e++; + } + + $e = 0; + while($e < scalar(my @sl = split(',', $ipmi->{list}))) { + for($n = 0; $n < 9; $n++) { + $sens[$e][$n] = 0 unless defined $sens[$e][$n]; + $rrdata .= ":" . $sens[$e][$n]; + } + $e++; + } + + RRDs::update($rrd, $rrdata); + logger("$myself: $rrdata") if $debug; + my $err = RRDs::error; + logger("ERROR: while updating $rrd: $err") if $err; +} + +sub ipmi_cgi { + my ($package, $config, $cgi) = @_; + + my $ipmi = $config->{ipmi}; + my @rigid = split(',', ($ipmi->{rigid} || "")); + my @limit = split(',', ($ipmi->{limit} || "")); + my $tf = $cgi->{tf}; + my $colors = $cgi->{colors}; + my $graph = $cgi->{graph}; + my $silent = $cgi->{silent}; + my $zoom = "--zoom=" . $config->{global_zoom}; + my %rrd = ( + 'new' => \&RRDs::graphv, + 'old' => \&RRDs::graph, + ); + my $version = "new"; + my $pic; + my $picz; + my $picz_width; + my $picz_height; + + my $u = ""; + my $width; + my $height; + my @riglim; + my @IMG; + my @IMGz; + my @tmp; + my @tmpz; + my @CDEF; + my $n; + my $n2; + my $str; + my $err; + my @LC = ( + "#4444EE", + "#EEEE44", + "#44EEEE", + "#EE44EE", + "#888888", + "#E29136", + "#44EE44", + "#448844", + "#EE4444", + ); + + $version = "old" if $RRDs::VERSION < 1.3; + my $rrd = $config->{base_lib} . $package . ".rrd"; + my $title = $config->{graph_title}->{$package}; + my $IMG_DIR = $config->{base_dir} . "/" . $config->{imgs_dir}; + my $imgfmt_uc = uc($config->{image_format}); + my $imgfmt_lc = lc($config->{image_format}); + + $title = !$silent ? $title : ""; + + + # text mode + # + if(lc($config->{iface_mode}) eq "text") { + if($title) { + main::graph_header($title, 2); + print(" \n"); + print(" \n"); + } + my (undef, undef, undef, $data) = RRDs::fetch("$rrd", + "--start=-$tf->{nwhen}$tf->{twhen}", + "AVERAGE", + "-r $tf->{res}"); + $err = RRDs::error; + print("ERROR: while fetching $rrd: $err\n") if $err; + my $line1; + my $line2; + my $line3; + print("
\n");
+		print("    ");
+		for($n = 0; $n < scalar(my @sl = split(',', $ipmi->{list})); $n++) {
+			$line1 = "";
+			foreach my $i (split(',', $ipmi->{desc}->{$n})) {
+				$i = trim($i);
+				$str = $ipmi->{map}->{$i} || $i;
+				$str = sprintf("%7s", substr($str, 0, 5));
+				$line1 .= "        ";
+				$line2 .= sprintf(" %7s", $str);
+				$line3 .= "--------";
+			}
+			if($line1) {
+				my $i = length($line1);
+				printf(sprintf("%${i}s", sprintf("%s", trim($sl[$n]))));
+			}
+		}
+		print("\n");
+		print("Time$line2\n");
+		print("----$line3 \n");
+		my $line;
+		my @row;
+		my $time;
+		my $n2;
+		my $n3;
+		my $from;
+		my $to;
+		for($n = 0, $time = $tf->{tb}; $n < ($tf->{tb} * $tf->{ts}); $n++) {
+			$line = @$data[$n];
+			$time = $time - (1 / $tf->{ts});
+			printf(" %2d$tf->{tc} ", $time);
+			for($n2 = 0; $n2 < scalar(my @sl = split(',', $ipmi->{list})); $n2++) {
+				$n3 = 0;
+				foreach my $i (split(',', $ipmi->{desc}->{$n2})) {
+					$from = $n2 * 9 + $n3++;
+					$to = $from + 1;
+					my ($j) = @$line[$from..$to];
+					@row = ($j);
+					printf("%7.1lf ", @row);
+				}
+			}
+			print("\n");
+		}
+		print("    
\n"); + if($title) { + print(" \n"); + print(" \n"); + main::graph_footer(); + } + print("
\n"); + return; + } + + + # graph mode + # + if($silent eq "yes" || $silent eq "imagetag") { + $colors->{fg_color} = "#000000"; # visible color for text mode + $u = "_"; + } + if($silent eq "imagetagbig") { + $colors->{fg_color} = "#000000"; # visible color for text mode + $u = ""; + } + + for($n = 0; $n < scalar(my @sl = split(',', $ipmi->{list})); $n++) { + $str = $u . $package . $n . "." . $tf->{when} . ".$imgfmt_lc"; + push(@IMG, $str); + unlink("$IMG_DIR" . $str); + if(lc($config->{enable_zoom}) eq "y") { + $str = $u . $package . $n . "z." . $tf->{when} . ".$imgfmt_lc"; + push(@IMGz, $str); + unlink("$IMG_DIR" . $str); + } + } + + @riglim = @{setup_riglim($rigid[0], $limit[0])}; + $n = 0; + while($n < scalar(my @sl = split(',', $ipmi->{list}))) { + if($title) { + if($n == 0) { + main::graph_header($title, $ipmi->{graphs_per_row}); + } + print(" \n"); + } + for($n2 = 0; $n2 < $ipmi->{graphs_per_row}; $n2++) { + last unless $n < scalar(my @sl = split(',', $ipmi->{list})); + if($title) { + print(" \n"); + } + undef(@tmp); + undef(@tmpz); + undef(@CDEF); + my $e = 0; + my $unit = $ipmi->{units}->{$n}; + foreach my $i (split(',', $ipmi->{desc}->{$n})) { + $i = trim($i); + $str = $ipmi->{map}->{$i} || $i; + $str = sprintf("%-40s", substr($str, 0, 40)); + push(@tmp, "LINE2:s" . ($e + 1) . $LC[$e] . ":$str"); + push(@tmp, "GPRINT:s" . ($e + 1) . ":LAST: Current\\:%7.1lf\\n"); + push(@tmpz, "LINE2:s" . ($e + 1) . $LC[$e] . ":$str"); + $e++; + } + while($e < 9) { + push(@tmp, "COMMENT: \\n"); + $e++; + } + if(lc($config->{show_gaps}) eq "y") { + push(@tmp, "AREA:wrongdata#$colors->{gap}:"); + push(@tmpz, "AREA:wrongdata#$colors->{gap}:"); + push(@CDEF, "CDEF:wrongdata=allvalues,UN,INF,UNKN,IF"); + } + ($width, $height) = split('x', $config->{graph_size}->{medium}); + $str = substr(trim($sl[$n]), 0, 25); + $pic = $rrd{$version}->("$IMG_DIR" . "$IMG[$n]", + "--title=$str ($tf->{nwhen}$tf->{twhen})", + "--start=-$tf->{nwhen}$tf->{twhen}", + "--imgformat=$imgfmt_uc", + "--vertical-label=$unit", + "--width=$width", + "--height=$height", + @riglim, + $zoom, + @{$cgi->{version12}}, + @{$cgi->{version12_small}}, + @{$colors->{graph_colors}}, + "DEF:s1=$rrd:ipmi" . $n . "_s1:AVERAGE", + "DEF:s2=$rrd:ipmi" . $n . "_s2:AVERAGE", + "DEF:s3=$rrd:ipmi" . $n . "_s3:AVERAGE", + "DEF:s4=$rrd:ipmi" . $n . "_s4:AVERAGE", + "DEF:s5=$rrd:ipmi" . $n . "_s5:AVERAGE", + "DEF:s6=$rrd:ipmi" . $n . "_s6:AVERAGE", + "DEF:s7=$rrd:ipmi" . $n . "_s7:AVERAGE", + "DEF:s8=$rrd:ipmi" . $n . "_s8:AVERAGE", + "DEF:s9=$rrd:ipmi" . $n . "_s9:AVERAGE", + "CDEF:allvalues=s1,s2,s3,s4,s5,s6,s7,s8,s9,+,+,+,+,+,+,+,+", + @CDEF, + @tmp); + $err = RRDs::error; + print("ERROR: while graphing $IMG_DIR" . "$IMG[$n]: $err\n") if $err; + if(lc($config->{enable_zoom}) eq "y") { + ($width, $height) = split('x', $config->{graph_size}->{zoom}); + $picz = $rrd{$version}->("$IMG_DIR" . "$IMGz[$n]", + "--title=$str ($tf->{nwhen}$tf->{twhen})", + "--start=-$tf->{nwhen}$tf->{twhen}", + "--imgformat=$imgfmt_uc", + "--vertical-label=$unit", + "--width=$width", + "--height=$height", + @riglim, + $zoom, + @{$cgi->{version12}}, + @{$cgi->{version12_small}}, + @{$colors->{graph_colors}}, + "DEF:s1=$rrd:ipmi" . $n . "_s1:AVERAGE", + "DEF:s2=$rrd:ipmi" . $n . "_s2:AVERAGE", + "DEF:s3=$rrd:ipmi" . $n . "_s3:AVERAGE", + "DEF:s4=$rrd:ipmi" . $n . "_s4:AVERAGE", + "DEF:s5=$rrd:ipmi" . $n . "_s5:AVERAGE", + "DEF:s6=$rrd:ipmi" . $n . "_s6:AVERAGE", + "DEF:s7=$rrd:ipmi" . $n . "_s7:AVERAGE", + "DEF:s8=$rrd:ipmi" . $n . "_s8:AVERAGE", + "DEF:s9=$rrd:ipmi" . $n . "_s9:AVERAGE", + "CDEF:allvalues=s1,s2,s3,s4,s5,s6,s7,s8,s9,+,+,+,+,+,+,+,+", + @CDEF, + @tmpz); + $err = RRDs::error; + print("ERROR: while graphing $IMG_DIR" . "$IMGz[$n]: $err\n") if $err; + } + if($title || ($silent =~ /imagetag/ && $graph =~ /ipmi$n/)) { + if(lc($config->{enable_zoom}) eq "y") { + if(lc($config->{disable_javascript_void}) eq "y") { + print(" {url} . "/" . $config->{imgs_dir} . $IMGz[$n] . "\">\n"); + } else { + if($version eq "new") { + $picz_width = $picz->{image_width} * $config->{global_zoom}; + $picz_height = $picz->{image_height} * $config->{global_zoom}; + } else { + $picz_width = $width + 115; + $picz_height = $height + 100; + } + print(" {url} . "/" . $config->{imgs_dir} . $IMGz[$n] . "','','width=" . $picz_width . ",height=" . $picz_height . ",scrollbars=0,resizable=0'))\">\n"); + } + } else { + print(" \n"); + } + } + if($title) { + print(" \n"); + } + $n++; + } + if($title) { + print(" \n"); + } + } + if($title) { + main::graph_footer(); + } + print("
\n"); + return; +} + +1; diff --git a/monitorix b/monitorix index c2c3034..be7f9b2 100755 --- a/monitorix +++ b/monitorix @@ -275,6 +275,14 @@ EOF } next; } + if($g eq "ipmi") { + for($n = 0; $n < scalar(my @ipmi_list = split(',', $config{ipmi}->{list})); $n++) { + my $name = trim($ipmi_list[$n]); + $gname = "_" . $g; + print(OUT " \n"); + } + next; + } if($g eq "fs" || $g eq "process") { $n = 0; foreach my $k (sort keys %{$config{graphs}}) { diff --git a/monitorix.conf b/monitorix.conf index daca68b..e082e35 100644 --- a/monitorix.conf +++ b/monitorix.conf @@ -78,6 +78,7 @@ secure_log_date_format = %b %e hptemp = n lmsens = n gensens = n + ipmi = n nvidia = n disk = n fs = y @@ -222,6 +223,28 @@ secure_log_date_format = %b %e +# IPMI graph +# ----------------------------------------------------------------------------- + + list = Temperatures, Fans, Voltages + + 0 = CPU Temp, System Temp + 1 = FAN 1 + 2 = Vcore, 3.3VCC, 12V, VDIMM, 5VCC, CPU VTT, VBAT, VSB, AVCC + + + 0 = degrees C + 1 = RPM + 2 = Volts + + + + graphs_per_row = 2 + rigid = 0 + limit = 100 + + + # NVIDIA graph # ----------------------------------------------------------------------------- @@ -812,7 +835,7 @@ logo_bottom = logo_bot.png remote = 300x100 -graph_name = system, kern, proc, hptemp, lmsens, gensens, nvidia, disk, fs, zfs, du, net, netstat, tc, libvirt, process, serv, mail, port, user, ftp, apache, nginx, lighttpd, mysql, varnish, pagespeed, squid, nfss, nfsc, bind, ntp, chrony, fail2ban, icecast, raspberrypi, phpapc, memcached, apcupsd, nut, wowza, int, verlihub +graph_name = system, kern, proc, hptemp, lmsens, gensens, ipmi, nvidia, disk, fs, zfs, du, net, netstat, tc, libvirt, process, serv, mail, port, user, ftp, apache, nginx, lighttpd, mysql, varnish, pagespeed, squid, nfss, nfsc, bind, ntp, chrony, fail2ban, icecast, raspberrypi, phpapc, memcached, apcupsd, nut, wowza, int, verlihub system = System load average and usage @@ -821,6 +844,7 @@ graph_name = system, kern, proc, hptemp, lmsens, gensens, nvidia, disk, fs, zfs, hptemp = HP ProLiant System Health lmsens = LM-Sensors and GPU temperatures gensens = Generic sensor statistics + ipmi = IPMI sensor statistics nvidia = NVIDIA temperatures and usage disk = Disk drive temperatures and health fs = Filesystem usage and I/O activity @@ -880,6 +904,7 @@ graph_name = system, kern, proc, hptemp, lmsens, gensens, nvidia, disk, fs, zfs, _lmsens5 = GPU temperatures _gensens1 = Temperatures _gensens2 = CPU frequency + _ipmi = IPMI sensors _nvidia1 = NVIDIA temperatures _nvidia2 = CPU usage _nvidia3 = Memory usage