added a complete graph for IPMI sensors using the 'ipmitool' command

This commit is contained in:
Jordi Sanfeliu 2016-12-20 10:13:57 +01:00
parent e48cfa1aac
commit 714cf09f1c
4 changed files with 498 additions and 1 deletions

View File

@ -1,5 +1,7 @@
3.N.N - DD-MMM-YYYY
====================
- Added a complete graph for IPMI sensors using the 'ipmitool' command.
(suggested by Frank Dijcks, fd AT fdsystems.nl)
- Improved the 'system.pm' graph with more detailed information about processes
(sleeping, waiting I/O, zombie, stopped, paging and running). It also
includes two new graphs: entropy and uptime.

462
lib/ipmi.pm Normal file
View File

@ -0,0 +1,462 @@
#
# Monitorix - A lightweight system monitoring tool.
#
# Copyright (C) 2005-2016 by Jordi Sanfeliu <jordi@fibranet.cat>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
package ipmi;
use strict;
use warnings;
use Monitorix;
use RRDs;
use POSIX qw(strftime);
use Exporter 'import';
our @EXPORT = qw(ipmi_init ipmi_update ipmi_cgi);
sub ipmi_init {
my $myself = (caller(0))[3];
my ($package, $config, $debug) = @_;
my $rrd = $config->{base_lib} . $package . ".rrd";
my $ipmi = $config->{ipmi};
my $info;
my @ds;
my @rra;
my @tmp;
my $n;
my @average;
my @min;
my @max;
my @last;
if(-e $rrd) {
$info = RRDs::info($rrd);
for my $key (keys %$info) {
if(index($key, 'ds[') == 0) {
if(index($key, '.type') != -1) {
push(@ds, substr($key, 3, index($key, ']') - 3));
}
}
if(index($key, 'rra[') == 0) {
if(index($key, '.rows') != -1) {
push(@rra, substr($key, 4, index($key, ']') - 4));
}
}
}
if(scalar(@ds) / 9 != scalar(my @fl = split(',', $ipmi->{list}))) {
logger("$myself: Detected size mismatch between 'list' (" . scalar(my @fl = split(',', $ipmi->{list})) . ") and $rrd (" . scalar(@ds) / 9 . "). Resizing it accordingly. All historical data will be lost. Backup file created.");
rename($rrd, "$rrd.bak");
}
if(scalar(@rra) < 12 + (4 * $config->{max_historic_years})) {
logger("$myself: Detected size mismatch between 'max_historic_years' (" . $config->{max_historic_years} . ") and $rrd (" . ((scalar(@rra) -12) / 4) . "). Resizing it accordingly. All historical data will be lost. Backup file created.");
rename($rrd, "$rrd.bak");
}
}
if(!(-e $rrd)) {
logger("Creating '$rrd' file.");
for($n = 1; $n <= $config->{max_historic_years}; $n++) {
push(@average, "RRA:AVERAGE:0.5:1440:" . (365 * $n));
push(@min, "RRA:MIN:0.5:1440:" . (365 * $n));
push(@max, "RRA:MAX:0.5:1440:" . (365 * $n));
push(@last, "RRA:LAST:0.5:1440:" . (365 * $n));
}
for($n = 0; $n < scalar(my @fl = split(',', $ipmi->{list})); $n++) {
push(@tmp, "DS:ipmi" . $n . "_s1:GAUGE:120:0:U");
push(@tmp, "DS:ipmi" . $n . "_s2:GAUGE:120:0:U");
push(@tmp, "DS:ipmi" . $n . "_s3:GAUGE:120:0:U");
push(@tmp, "DS:ipmi" . $n . "_s4:GAUGE:120:0:U");
push(@tmp, "DS:ipmi" . $n . "_s5:GAUGE:120:0:U");
push(@tmp, "DS:ipmi" . $n . "_s6:GAUGE:120:0:U");
push(@tmp, "DS:ipmi" . $n . "_s7:GAUGE:120:0:U");
push(@tmp, "DS:ipmi" . $n . "_s8:GAUGE:120:0:U");
push(@tmp, "DS:ipmi" . $n . "_s9:GAUGE:120:0:U");
}
eval {
RRDs::create($rrd,
"--step=60",
@tmp,
"RRA:AVERAGE:0.5:1:1440",
"RRA:AVERAGE:0.5:30:336",
"RRA:AVERAGE:0.5:60:744",
@average,
"RRA:MIN:0.5:1:1440",
"RRA:MIN:0.5:30:336",
"RRA:MIN:0.5:60:744",
@min,
"RRA:MAX:0.5:1:1440",
"RRA:MAX:0.5:30:336",
"RRA:MAX:0.5:60:744",
@max,
"RRA:LAST:0.5:1:1440",
"RRA:LAST:0.5:30:336",
"RRA:LAST:0.5:60:744",
@last,
);
};
my $err = RRDs::error;
if($@ || $err) {
logger("$@") unless !$@;
if($err) {
logger("ERROR: while creating $rrd: $err");
if($err eq "RRDs::error") {
logger("... is the RRDtool Perl package installed?");
}
}
return;
}
}
push(@{$config->{func_update}}, $package);
logger("$myself: Ok") if $debug;
}
sub ipmi_update {
my $myself = (caller(0))[3];
my ($package, $config, $debug) = @_;
my $rrd = $config->{base_lib} . $package . ".rrd";
my $ipmi = $config->{ipmi};
my $args = $ipmi->{extra_args} || "";
my @sens;
my $n;
my $str;
my $rrdata = "N";
if(!open(IN, "ipmitool sdr $args |")) {
logger("$myself: unable to execute 'ipmitool'. $!");
return;
}
my @data = <IN>;
close(IN);
my $e = 0;
while($e < scalar(my @sl = split(',', $ipmi->{list}))) {
my $e2 = 0;
foreach my $i (split(',', $ipmi->{desc}->{$e})) {
my $unit;
$sens[$e][$e2] = 0 unless defined $sens[$e][$e2];
$str = trim($i);
$unit = $ipmi->{units}->{$e};
foreach(@data) {
if(/^($str)\s+\|\s+(\d+\.*\d*)\s+$unit\s+/) {
print "'$1' '$2'\n";
$sens[$e][$e2] = $2;
}
}
$e2++;
}
$e++;
}
$e = 0;
while($e < scalar(my @sl = split(',', $ipmi->{list}))) {
for($n = 0; $n < 9; $n++) {
$sens[$e][$n] = 0 unless defined $sens[$e][$n];
$rrdata .= ":" . $sens[$e][$n];
}
$e++;
}
RRDs::update($rrd, $rrdata);
logger("$myself: $rrdata") if $debug;
my $err = RRDs::error;
logger("ERROR: while updating $rrd: $err") if $err;
}
sub ipmi_cgi {
my ($package, $config, $cgi) = @_;
my $ipmi = $config->{ipmi};
my @rigid = split(',', ($ipmi->{rigid} || ""));
my @limit = split(',', ($ipmi->{limit} || ""));
my $tf = $cgi->{tf};
my $colors = $cgi->{colors};
my $graph = $cgi->{graph};
my $silent = $cgi->{silent};
my $zoom = "--zoom=" . $config->{global_zoom};
my %rrd = (
'new' => \&RRDs::graphv,
'old' => \&RRDs::graph,
);
my $version = "new";
my $pic;
my $picz;
my $picz_width;
my $picz_height;
my $u = "";
my $width;
my $height;
my @riglim;
my @IMG;
my @IMGz;
my @tmp;
my @tmpz;
my @CDEF;
my $n;
my $n2;
my $str;
my $err;
my @LC = (
"#4444EE",
"#EEEE44",
"#44EEEE",
"#EE44EE",
"#888888",
"#E29136",
"#44EE44",
"#448844",
"#EE4444",
);
$version = "old" if $RRDs::VERSION < 1.3;
my $rrd = $config->{base_lib} . $package . ".rrd";
my $title = $config->{graph_title}->{$package};
my $IMG_DIR = $config->{base_dir} . "/" . $config->{imgs_dir};
my $imgfmt_uc = uc($config->{image_format});
my $imgfmt_lc = lc($config->{image_format});
$title = !$silent ? $title : "";
# text mode
#
if(lc($config->{iface_mode}) eq "text") {
if($title) {
main::graph_header($title, 2);
print(" <tr>\n");
print(" <td bgcolor='$colors->{title_bg_color}'>\n");
}
my (undef, undef, undef, $data) = RRDs::fetch("$rrd",
"--start=-$tf->{nwhen}$tf->{twhen}",
"AVERAGE",
"-r $tf->{res}");
$err = RRDs::error;
print("ERROR: while fetching $rrd: $err\n") if $err;
my $line1;
my $line2;
my $line3;
print(" <pre style='font-size: 12px; color: $colors->{fg_color}';>\n");
print(" ");
for($n = 0; $n < scalar(my @sl = split(',', $ipmi->{list})); $n++) {
$line1 = "";
foreach my $i (split(',', $ipmi->{desc}->{$n})) {
$i = trim($i);
$str = $ipmi->{map}->{$i} || $i;
$str = sprintf("%7s", substr($str, 0, 5));
$line1 .= " ";
$line2 .= sprintf(" %7s", $str);
$line3 .= "--------";
}
if($line1) {
my $i = length($line1);
printf(sprintf("%${i}s", sprintf("%s", trim($sl[$n]))));
}
}
print("\n");
print("Time$line2\n");
print("----$line3 \n");
my $line;
my @row;
my $time;
my $n2;
my $n3;
my $from;
my $to;
for($n = 0, $time = $tf->{tb}; $n < ($tf->{tb} * $tf->{ts}); $n++) {
$line = @$data[$n];
$time = $time - (1 / $tf->{ts});
printf(" %2d$tf->{tc} ", $time);
for($n2 = 0; $n2 < scalar(my @sl = split(',', $ipmi->{list})); $n2++) {
$n3 = 0;
foreach my $i (split(',', $ipmi->{desc}->{$n2})) {
$from = $n2 * 9 + $n3++;
$to = $from + 1;
my ($j) = @$line[$from..$to];
@row = ($j);
printf("%7.1lf ", @row);
}
}
print("\n");
}
print(" </pre>\n");
if($title) {
print(" </td>\n");
print(" </tr>\n");
main::graph_footer();
}
print(" <br>\n");
return;
}
# graph mode
#
if($silent eq "yes" || $silent eq "imagetag") {
$colors->{fg_color} = "#000000"; # visible color for text mode
$u = "_";
}
if($silent eq "imagetagbig") {
$colors->{fg_color} = "#000000"; # visible color for text mode
$u = "";
}
for($n = 0; $n < scalar(my @sl = split(',', $ipmi->{list})); $n++) {
$str = $u . $package . $n . "." . $tf->{when} . ".$imgfmt_lc";
push(@IMG, $str);
unlink("$IMG_DIR" . $str);
if(lc($config->{enable_zoom}) eq "y") {
$str = $u . $package . $n . "z." . $tf->{when} . ".$imgfmt_lc";
push(@IMGz, $str);
unlink("$IMG_DIR" . $str);
}
}
@riglim = @{setup_riglim($rigid[0], $limit[0])};
$n = 0;
while($n < scalar(my @sl = split(',', $ipmi->{list}))) {
if($title) {
if($n == 0) {
main::graph_header($title, $ipmi->{graphs_per_row});
}
print(" <tr>\n");
}
for($n2 = 0; $n2 < $ipmi->{graphs_per_row}; $n2++) {
last unless $n < scalar(my @sl = split(',', $ipmi->{list}));
if($title) {
print(" <td bgcolor='" . $colors->{title_bg_color} . "'>\n");
}
undef(@tmp);
undef(@tmpz);
undef(@CDEF);
my $e = 0;
my $unit = $ipmi->{units}->{$n};
foreach my $i (split(',', $ipmi->{desc}->{$n})) {
$i = trim($i);
$str = $ipmi->{map}->{$i} || $i;
$str = sprintf("%-40s", substr($str, 0, 40));
push(@tmp, "LINE2:s" . ($e + 1) . $LC[$e] . ":$str");
push(@tmp, "GPRINT:s" . ($e + 1) . ":LAST: Current\\:%7.1lf\\n");
push(@tmpz, "LINE2:s" . ($e + 1) . $LC[$e] . ":$str");
$e++;
}
while($e < 9) {
push(@tmp, "COMMENT: \\n");
$e++;
}
if(lc($config->{show_gaps}) eq "y") {
push(@tmp, "AREA:wrongdata#$colors->{gap}:");
push(@tmpz, "AREA:wrongdata#$colors->{gap}:");
push(@CDEF, "CDEF:wrongdata=allvalues,UN,INF,UNKN,IF");
}
($width, $height) = split('x', $config->{graph_size}->{medium});
$str = substr(trim($sl[$n]), 0, 25);
$pic = $rrd{$version}->("$IMG_DIR" . "$IMG[$n]",
"--title=$str ($tf->{nwhen}$tf->{twhen})",
"--start=-$tf->{nwhen}$tf->{twhen}",
"--imgformat=$imgfmt_uc",
"--vertical-label=$unit",
"--width=$width",
"--height=$height",
@riglim,
$zoom,
@{$cgi->{version12}},
@{$cgi->{version12_small}},
@{$colors->{graph_colors}},
"DEF:s1=$rrd:ipmi" . $n . "_s1:AVERAGE",
"DEF:s2=$rrd:ipmi" . $n . "_s2:AVERAGE",
"DEF:s3=$rrd:ipmi" . $n . "_s3:AVERAGE",
"DEF:s4=$rrd:ipmi" . $n . "_s4:AVERAGE",
"DEF:s5=$rrd:ipmi" . $n . "_s5:AVERAGE",
"DEF:s6=$rrd:ipmi" . $n . "_s6:AVERAGE",
"DEF:s7=$rrd:ipmi" . $n . "_s7:AVERAGE",
"DEF:s8=$rrd:ipmi" . $n . "_s8:AVERAGE",
"DEF:s9=$rrd:ipmi" . $n . "_s9:AVERAGE",
"CDEF:allvalues=s1,s2,s3,s4,s5,s6,s7,s8,s9,+,+,+,+,+,+,+,+",
@CDEF,
@tmp);
$err = RRDs::error;
print("ERROR: while graphing $IMG_DIR" . "$IMG[$n]: $err\n") if $err;
if(lc($config->{enable_zoom}) eq "y") {
($width, $height) = split('x', $config->{graph_size}->{zoom});
$picz = $rrd{$version}->("$IMG_DIR" . "$IMGz[$n]",
"--title=$str ($tf->{nwhen}$tf->{twhen})",
"--start=-$tf->{nwhen}$tf->{twhen}",
"--imgformat=$imgfmt_uc",
"--vertical-label=$unit",
"--width=$width",
"--height=$height",
@riglim,
$zoom,
@{$cgi->{version12}},
@{$cgi->{version12_small}},
@{$colors->{graph_colors}},
"DEF:s1=$rrd:ipmi" . $n . "_s1:AVERAGE",
"DEF:s2=$rrd:ipmi" . $n . "_s2:AVERAGE",
"DEF:s3=$rrd:ipmi" . $n . "_s3:AVERAGE",
"DEF:s4=$rrd:ipmi" . $n . "_s4:AVERAGE",
"DEF:s5=$rrd:ipmi" . $n . "_s5:AVERAGE",
"DEF:s6=$rrd:ipmi" . $n . "_s6:AVERAGE",
"DEF:s7=$rrd:ipmi" . $n . "_s7:AVERAGE",
"DEF:s8=$rrd:ipmi" . $n . "_s8:AVERAGE",
"DEF:s9=$rrd:ipmi" . $n . "_s9:AVERAGE",
"CDEF:allvalues=s1,s2,s3,s4,s5,s6,s7,s8,s9,+,+,+,+,+,+,+,+",
@CDEF,
@tmpz);
$err = RRDs::error;
print("ERROR: while graphing $IMG_DIR" . "$IMGz[$n]: $err\n") if $err;
}
if($title || ($silent =~ /imagetag/ && $graph =~ /ipmi$n/)) {
if(lc($config->{enable_zoom}) eq "y") {
if(lc($config->{disable_javascript_void}) eq "y") {
print(" <a href=\"" . $config->{url} . "/" . $config->{imgs_dir} . $IMGz[$n] . "\"><img src='" . $config->{url} . "/" . $config->{imgs_dir} . $IMG[$n] . "' border='0'></a>\n");
} else {
if($version eq "new") {
$picz_width = $picz->{image_width} * $config->{global_zoom};
$picz_height = $picz->{image_height} * $config->{global_zoom};
} else {
$picz_width = $width + 115;
$picz_height = $height + 100;
}
print(" <a href=\"javascript:void(window.open('" . $config->{url} . "/" . $config->{imgs_dir} . $IMGz[$n] . "','','width=" . $picz_width . ",height=" . $picz_height . ",scrollbars=0,resizable=0'))\"><img src='" . $config->{url} . "/" . $config->{imgs_dir} . $IMG[$n] . "' border='0'></a>\n");
}
} else {
print(" <img src='" . $config->{url} . "/" . $config->{imgs_dir} . $IMG[$n] . "'>\n");
}
}
if($title) {
print(" </td>\n");
}
$n++;
}
if($title) {
print(" </tr>\n");
}
}
if($title) {
main::graph_footer();
}
print(" <br>\n");
return;
}
1;

View File

@ -275,6 +275,14 @@ EOF
}
next;
}
if($g eq "ipmi") {
for($n = 0; $n < scalar(my @ipmi_list = split(',', $config{ipmi}->{list})); $n++) {
my $name = trim($ipmi_list[$n]);
$gname = "_" . $g;
print(OUT " <option value='" . $gname . $n . "'>" . $name . "</option>\n");
}
next;
}
if($g eq "fs" || $g eq "process") {
$n = 0;
foreach my $k (sort keys %{$config{graphs}}) {

View File

@ -78,6 +78,7 @@ secure_log_date_format = %b %e
hptemp = n
lmsens = n
gensens = n
ipmi = n
nvidia = n
disk = n
fs = y
@ -222,6 +223,28 @@ secure_log_date_format = %b %e
</gensens>
# IPMI graph
# -----------------------------------------------------------------------------
<ipmi>
list = Temperatures, Fans, Voltages
<desc>
0 = CPU Temp, System Temp
1 = FAN 1
2 = Vcore, 3.3VCC, 12V, VDIMM, 5VCC, CPU VTT, VBAT, VSB, AVCC
</desc>
<units>
0 = degrees C
1 = RPM
2 = Volts
</units>
<map>
</map>
graphs_per_row = 2
rigid = 0
limit = 100
</ipmi>
# NVIDIA graph
# -----------------------------------------------------------------------------
<nvidia>
@ -812,7 +835,7 @@ logo_bottom = logo_bot.png
remote = 300x100
</graph_size>
graph_name = system, kern, proc, hptemp, lmsens, gensens, nvidia, disk, fs, zfs, du, net, netstat, tc, libvirt, process, serv, mail, port, user, ftp, apache, nginx, lighttpd, mysql, varnish, pagespeed, squid, nfss, nfsc, bind, ntp, chrony, fail2ban, icecast, raspberrypi, phpapc, memcached, apcupsd, nut, wowza, int, verlihub
graph_name = system, kern, proc, hptemp, lmsens, gensens, ipmi, nvidia, disk, fs, zfs, du, net, netstat, tc, libvirt, process, serv, mail, port, user, ftp, apache, nginx, lighttpd, mysql, varnish, pagespeed, squid, nfss, nfsc, bind, ntp, chrony, fail2ban, icecast, raspberrypi, phpapc, memcached, apcupsd, nut, wowza, int, verlihub
<graph_title>
system = System load average and usage
@ -821,6 +844,7 @@ graph_name = system, kern, proc, hptemp, lmsens, gensens, nvidia, disk, fs, zfs,
hptemp = HP ProLiant System Health
lmsens = LM-Sensors and GPU temperatures
gensens = Generic sensor statistics
ipmi = IPMI sensor statistics
nvidia = NVIDIA temperatures and usage
disk = Disk drive temperatures and health
fs = Filesystem usage and I/O activity
@ -880,6 +904,7 @@ graph_name = system, kern, proc, hptemp, lmsens, gensens, nvidia, disk, fs, zfs,
_lmsens5 = GPU temperatures
_gensens1 = Temperatures
_gensens2 = CPU frequency
_ipmi = IPMI sensors
_nvidia1 = NVIDIA temperatures
_nvidia2 = CPU usage
_nvidia3 = Memory usage