#
# Monitorix - A lightweight system monitoring tool.
#
# Copyright (C) 2005-2014 by Jordi Sanfeliu <jordi@fibranet.cat>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#

package nvidia;

use strict;
use warnings;
use Monitorix;
use RRDs;
use Exporter 'import';
our @EXPORT = qw(nvidia_init nvidia_update nvidia_cgi);

sub nvidia_init {
	my $myself = (caller(0))[3];
	my ($package, $config, $debug) = @_;
	my $rrd = $config->{base_lib} . $package . ".rrd";

	my $info;
	my @rra;
	my @tmp;
	my $n;

	my @average;
	my @min;
	my @max;
	my @last;

	# checks if 'nvidia-smi' does exists.
	if(!open(IN, "nvidia-smi |")) {
		logger("$myself: unable to execute 'nvidia-smi'. $!");
		return;
	}
	close(IN);

	if(-e $rrd) {
		$info = RRDs::info($rrd);
		for my $key (keys %$info) {
			if(index($key, 'rra[') == 0) {
				if(index($key, '.rows') != -1) {
					push(@rra, substr($key, 4, index($key, ']') - 4));
				}
			}
		}
		if(scalar(@rra) < 12 + (4 * $config->{max_historic_years})) {
			logger("$myself: Detected size mismatch between 'max_historic_years' (" . $config->{max_historic_years} . ") and $rrd (" . ((scalar(@rra) -12) / 4) . "). Resizing it accordingly. All historical data will be lost. Backup file created.");
			rename($rrd, "$rrd.bak");
		}
	}

	if(!(-e $rrd)) {
		logger("Creating '$rrd' file.");
		for($n = 1; $n <= $config->{max_historic_years}; $n++) {
			push(@average, "RRA:AVERAGE:0.5:1440:" . (365 * $n));
			push(@min, "RRA:MIN:0.5:1440:" . (365 * $n));
			push(@max, "RRA:MAX:0.5:1440:" . (365 * $n));
			push(@last, "RRA:LAST:0.5:1440:" . (365 * $n));
		}
		eval {
			RRDs::create($rrd,
				"--step=60",
				"DS:nvidia_temp0:GAUGE:120:0:U",
				"DS:nvidia_temp1:GAUGE:120:0:U",
				"DS:nvidia_temp2:GAUGE:120:0:U",
				"DS:nvidia_temp3:GAUGE:120:0:U",
				"DS:nvidia_temp4:GAUGE:120:0:U",
				"DS:nvidia_temp5:GAUGE:120:0:U",
				"DS:nvidia_temp6:GAUGE:120:0:U",
				"DS:nvidia_temp7:GAUGE:120:0:U",
				"DS:nvidia_temp8:GAUGE:120:0:U",
				"DS:nvidia_gpu0:GAUGE:120:0:100",
				"DS:nvidia_gpu1:GAUGE:120:0:100",
				"DS:nvidia_gpu2:GAUGE:120:0:100",
				"DS:nvidia_gpu3:GAUGE:120:0:100",
				"DS:nvidia_gpu4:GAUGE:120:0:100",
				"DS:nvidia_gpu5:GAUGE:120:0:100",
				"DS:nvidia_gpu6:GAUGE:120:0:100",
				"DS:nvidia_gpu7:GAUGE:120:0:100",
				"DS:nvidia_gpu8:GAUGE:120:0:100",
				"DS:nvidia_mem0:GAUGE:120:0:100",
				"DS:nvidia_mem1:GAUGE:120:0:100",
				"DS:nvidia_mem2:GAUGE:120:0:100",
				"DS:nvidia_mem3:GAUGE:120:0:100",
				"DS:nvidia_mem4:GAUGE:120:0:100",
				"DS:nvidia_mem5:GAUGE:120:0:100",
				"DS:nvidia_mem6:GAUGE:120:0:100",
				"DS:nvidia_mem7:GAUGE:120:0:100",
				"DS:nvidia_mem8:GAUGE:120:0:100",
				"RRA:AVERAGE:0.5:1:1440",
				"RRA:AVERAGE:0.5:30:336",
				"RRA:AVERAGE:0.5:60:744",
				@average,
				"RRA:MIN:0.5:1:1440",
				"RRA:MIN:0.5:30:336",
				"RRA:MIN:0.5:60:744",
				@min,
				"RRA:MAX:0.5:1:1440",
				"RRA:MAX:0.5:30:336",
				"RRA:MAX:0.5:60:744",
				@max,
				"RRA:LAST:0.5:1:1440",
				"RRA:LAST:0.5:30:336",
				"RRA:LAST:0.5:60:744",
				@last,
			);
		};
		my $err = RRDs::error;
		if($@ || $err) {
			logger("$@") unless !$@;
			if($err) {
				logger("ERROR: while creating $rrd: $err");
				if($err eq "RRDs::error") {
					logger("... is the RRDtool Perl package installed?");
				}
			}
			return;
		}
	}

	push(@{$config->{func_update}}, $package);
	logger("$myself: Ok") if $debug;
}

sub nvidia_update {
	my $myself = (caller(0))[3];
	my ($package, $config, $debug) = @_;
	my $rrd = $config->{base_lib} . $package . ".rrd";
	my $nvidia = $config->{nvidia};

	my @temp;
	my @gpu;
	my @mem;
	my @data;
	my $utilization;

	my $l;
	my $n;
	my $rrdata = "N";

	for($n = 0; $n < 9; $n++) {
		$temp[$n] = 0;
		$gpu[$n] = 0;
		$mem[$n] = 0;
		if($n < $nvidia->{max}) {
			($mem[$n], $gpu[$n], $temp[$n]) = split(' ', get_nvidia_data($n));
			if(!$temp[$n] && !$gpu[$n] && !$mem[$n]) {
				# attempt to get data using the old driver version
				$utilization = 0;
	  			open(IN, "nvidia-smi -g $n |");
				@data = <IN>;
				close(IN);
				for($l = 0; $l < scalar(@data); $l++) {
					if($data[$l] =~ /Temperature/) {
						my (undef, $tmp) = split(':', $data[$l]);
						if($tmp eq "\n") {
							$l++;
							$tmp = $data[$l];
						}
						my ($value, undef) = split(' ', $tmp);
						$value =~ s/[-]/./;
						$value =~ s/[^0-9.]//g;
						if(int($value) > 0) {
							$temp[$n] = int($value);
						}
					}
					if($data[$l] =~ /Utilization/) {
						$utilization = 1;
					}
					if($utilization == 1) {
						if($data[$l] =~ /GPU/) {
							my (undef, $tmp) = split(':', $data[$l]);
							if($tmp eq "\n") {
								$l++;
								$tmp = $data[$l];
							}
							my ($value, undef) = split(' ', $tmp);
							$value =~ s/[-]/./;
							$value =~ s/[^0-9.]//g;
							if(int($value) > 0) {
								$gpu[$n] = int($value);
							}
						}
						if($data[$l] =~ /Memory/) {
							my (undef, $tmp) = split(':', $data[$l]);
							if($tmp eq "\n") {
								$l++;
								$tmp = $data[$l];
							}
							my ($value, undef) = split(' ', $tmp);
							$value =~ s/[-]/./;
							$value =~ s/[^0-9.]//g;
							if(int($value) > 0) {
								$mem[$n] = int($value);
							}
						}
					}
				}
			}
		}
	}

	for($n = 0; $n < scalar(@temp); $n++) {
		$rrdata .= ":$temp[$n]";
	}
	for($n = 0; $n < scalar(@gpu); $n++) {
		$rrdata .= ":$gpu[$n]";
	}
	for($n = 0; $n < scalar(@mem); $n++) {
		$rrdata .= ":$mem[$n]";
	}

	RRDs::update($rrd, $rrdata);
	logger("$myself: $rrdata") if $debug;
	my $err = RRDs::error;
	logger("ERROR: while updating $rrd: $err") if $err;
}

sub nvidia_cgi {
	my ($package, $config, $cgi) = @_;

	my $nvidia = $config->{nvidia};
	my @rigid = split(',', ($nvidia->{rigid} || ""));
	my @limit = split(',', ($nvidia->{limit} || ""));
	my $tf = $cgi->{tf};
	my $colors = $cgi->{colors};
	my $graph = $cgi->{graph};
	my $silent = $cgi->{silent};
	my $zoom = "--zoom=" . $config->{global_zoom};

	my $u = "";
	my $width;
	my $height;
	my @riglim;
	my $temp_scale = "Celsius";
	my @tmp;
	my @tmpz;
	my @CDEF,
	my $n;
	my $err;
	my @LC = (
		"#FFA500",
		"#44EEEE",
		"#44EE44",
		"#4444EE",
		"#448844",
		"#EE4444",
		"#EE44EE",
		"#EEEE44",
		"#963C74",
	);

	my $rrd = $config->{base_lib} . $package . ".rrd";
	my $title = $config->{graph_title}->{$package};
	my $PNG_DIR = $config->{base_dir} . "/" . $config->{imgs_dir};

	$title = !$silent ? $title : "";

	if(lc($config->{temperature_scale}) eq "f") {
		$temp_scale = "Fahrenheit";
	}


	# text mode
	#
	if(lc($config->{iface_mode}) eq "text") {
		if($title) {
			main::graph_header($title, 2);
			print("    <tr>\n");
			print("    <td bgcolor='$colors->{title_bg_color}'>\n");
		}
		my (undef, undef, undef, $data) = RRDs::fetch("$rrd",
			"--start=-$tf->{nwhen}$tf->{twhen}",
			"AVERAGE",
			"-r $tf->{res}");
		$err = RRDs::error;
		print("ERROR: while fetching $rrd: $err\n") if $err;
		my $line1;
		my $line2;
		my $line3;
		print("    <pre style='font-size: 12px; color: $colors->{fg_color}';>\n");
		print("    ");
		for($n = 0; $n < $nvidia->{max}; $n++) {
			print("    NVIDIA card $n");
		}
		print("\n");
		for($n = 0; $n < $nvidia->{max}; $n++) {
			$line2 .= "   Temp  GPU  Mem";
			$line3 .= "-----------------";
		}
		print("Time$line2\n");
		print("----$line3 \n");
		my $line;
		my @row;
		my $time;
		my $n2;
		for($n = 0, $time = $tf->{tb}; $n < ($tf->{tb} * $tf->{ts}); $n++) {
			$line = @$data[$n];
			$time = $time - (1 / $tf->{ts});
			printf(" %2d$tf->{tc} ", $time);
			undef($line1);
			undef(@row);
			for($n2 = 0; $n2 < $nvidia->{max}; $n2++) {
				push(@row, celsius_to($config, @$line[$n2]));
				push(@row, celsius_to($config, @$line[$n2 + 9]));
				push(@row, celsius_to($config, @$line[$n2 + 18]));
				$line1 .= "   %3d %3d%% %3d%%";
			}
			print(sprintf($line1, @row));
			print("\n");
		}
		print("    </pre>\n");
		if($title) {
			print("    </td>\n");
			print("    </tr>\n");
			main::graph_footer();
		}
		print("  <br>\n");
		return;
	}


	# graph mode
	#
	if($silent eq "yes" || $silent eq "imagetag") {
		$colors->{fg_color} = "#000000";  # visible color for text mode
		$u = "_";
	}
	if($silent eq "imagetagbig") {
		$colors->{fg_color} = "#000000";  # visible color for text mode
		$u = "";
	}

	my $PNG1 = $u . $package . "1." . $tf->{when} . ".png";
	my $PNG2 = $u . $package . "2." . $tf->{when} . ".png";
	my $PNG3 = $u . $package . "3." . $tf->{when} . ".png";
	my $PNG1z = $u . $package . "1z." . $tf->{when} . ".png";
	my $PNG2z = $u . $package . "2z." . $tf->{when} . ".png";
	my $PNG3z = $u . $package . "3z." . $tf->{when} . ".png";

	unlink ("$PNG_DIR" . "$PNG1",
		"$PNG_DIR" . "$PNG2",
		"$PNG_DIR" . "$PNG3");
	if(lc($config->{enable_zoom}) eq "y") {
		unlink ("$PNG_DIR" . "$PNG1z",
			"$PNG_DIR" . "$PNG2z",
			"$PNG_DIR" . "$PNG3z");
	}

	if($title) {
		main::graph_header($title, 2);
	}

	@riglim = @{setup_riglim($rigid[0], $limit[0])};
	for($n = 0; $n < 9; $n++) {
		if($n < $nvidia->{max}) {
			push(@tmp, "LINE2:temp_" . $n . $LC[$n] . ":Card $n");
			push(@tmpz, "LINE2:temp_" . $n . $LC[$n] . ":Card $n");
			push(@tmp, "GPRINT:temp_" . $n . ":LAST:             Current\\: %2.0lf");
			push(@tmp, "GPRINT:temp_" . $n . ":AVERAGE:   Average\\: %2.0lf");
			push(@tmp, "GPRINT:temp_" . $n . ":MIN:   Min\\: %2.0lf");
			push(@tmp, "GPRINT:temp_" . $n . ":MAX:   Max\\: %2.0lf\\n");
		} else {
			push(@tmp, "COMMENT: \\n");
		}
	}

	if($title) {
		print("    <tr>\n");
		print("    <td bgcolor='$colors->{title_bg_color}'>\n");
	}
	if(lc($config->{temperature_scale}) eq "f") {
		push(@CDEF, "CDEF:temp_0=9,5,/,temp0,*,32,+");
		push(@CDEF, "CDEF:temp_1=9,5,/,temp1,*,32,+");
		push(@CDEF, "CDEF:temp_2=9,5,/,temp2,*,32,+");
		push(@CDEF, "CDEF:temp_3=9,5,/,temp3,*,32,+");
		push(@CDEF, "CDEF:temp_4=9,5,/,temp4,*,32,+");
		push(@CDEF, "CDEF:temp_5=9,5,/,temp5,*,32,+");
		push(@CDEF, "CDEF:temp_6=9,5,/,temp6,*,32,+");
		push(@CDEF, "CDEF:temp_7=9,5,/,temp7,*,32,+");
		push(@CDEF, "CDEF:temp_8=9,5,/,temp8,*,32,+");
	} else {
		push(@CDEF, "CDEF:temp_0=temp0");
		push(@CDEF, "CDEF:temp_1=temp1");
		push(@CDEF, "CDEF:temp_2=temp2");
		push(@CDEF, "CDEF:temp_3=temp3");
		push(@CDEF, "CDEF:temp_4=temp4");
		push(@CDEF, "CDEF:temp_5=temp5");
		push(@CDEF, "CDEF:temp_6=temp6");
		push(@CDEF, "CDEF:temp_7=temp7");
		push(@CDEF, "CDEF:temp_8=temp8");
	}
	if(lc($config->{show_gaps}) eq "y") {
		push(@tmp, "AREA:wrongdata#$colors->{gap}:");
		push(@tmpz, "AREA:wrongdata#$colors->{gap}:");
		push(@CDEF, "CDEF:wrongdata=allvalues,UN,INF,UNKN,IF");
	}
	($width, $height) = split('x', $config->{graph_size}->{main});
	if($silent =~ /imagetag/) {
		($width, $height) = split('x', $config->{graph_size}->{remote}) if $silent eq "imagetag";
		($width, $height) = split('x', $config->{graph_size}->{main}) if $silent eq "imagetagbig";
		@tmp = @tmpz;
		push(@tmp, "COMMENT: \\n");
	}
	RRDs::graph("$PNG_DIR" . "$PNG1",
		"--title=$config->{graphs}->{_nvidia1}  ($tf->{nwhen}$tf->{twhen})",
		"--start=-$tf->{nwhen}$tf->{twhen}",
		"--imgformat=PNG",
		"--vertical-label=$temp_scale",
		"--width=$width",
		"--height=$height",
		@riglim,
		$zoom,
		@{$cgi->{version12}},
		@{$colors->{graph_colors}},
		"DEF:temp0=$rrd:nvidia_temp0:AVERAGE",
		"DEF:temp1=$rrd:nvidia_temp1:AVERAGE",
		"DEF:temp2=$rrd:nvidia_temp2:AVERAGE",
		"DEF:temp3=$rrd:nvidia_temp3:AVERAGE",
		"DEF:temp4=$rrd:nvidia_temp4:AVERAGE",
		"DEF:temp5=$rrd:nvidia_temp5:AVERAGE",
		"DEF:temp6=$rrd:nvidia_temp6:AVERAGE",
		"DEF:temp7=$rrd:nvidia_temp7:AVERAGE",
		"DEF:temp8=$rrd:nvidia_temp8:AVERAGE",
		"CDEF:allvalues=temp0,temp1,temp2,temp3,temp4,temp5,temp6,temp7,temp8,+,+,+,+,+,+,+,+",
		@CDEF,
		@tmp,
		"COMMENT: \\n",
		"COMMENT: \\n");
	$err = RRDs::error;
	print("ERROR: while graphing $PNG_DIR" . "$PNG1: $err\n") if $err;
	if(lc($config->{enable_zoom}) eq "y") {
		($width, $height) = split('x', $config->{graph_size}->{zoom});
		RRDs::graph("$PNG_DIR" . "$PNG1z",
			"--title=$config->{graphs}->{_nvidia1}  ($tf->{nwhen}$tf->{twhen})",
			"--start=-$tf->{nwhen}$tf->{twhen}",
			"--imgformat=PNG",
			"--vertical-label=$temp_scale",
			"--width=$width",
			"--height=$height",
			@riglim,
			@{$cgi->{version12}},
			@{$colors->{graph_colors}},
			"DEF:temp0=$rrd:nvidia_temp0:AVERAGE",
			"DEF:temp1=$rrd:nvidia_temp1:AVERAGE",
			"DEF:temp2=$rrd:nvidia_temp2:AVERAGE",
			"DEF:temp3=$rrd:nvidia_temp3:AVERAGE",
			"DEF:temp4=$rrd:nvidia_temp4:AVERAGE",
			"DEF:temp5=$rrd:nvidia_temp5:AVERAGE",
			"DEF:temp6=$rrd:nvidia_temp6:AVERAGE",
			"DEF:temp7=$rrd:nvidia_temp7:AVERAGE",
			"DEF:temp8=$rrd:nvidia_temp8:AVERAGE",
			"CDEF:allvalues=temp0,temp1,temp2,temp3,temp4,temp5,temp6,temp7,temp8,+,+,+,+,+,+,+,+",
			@CDEF,
			@tmpz);
		$err = RRDs::error;
		print("ERROR: while graphing $PNG_DIR" . "$PNG1z: $err\n") if $err;
	}
	if($title || ($silent =~ /imagetag/ && $graph =~ /nvidia1/)) {
		if(lc($config->{enable_zoom}) eq "y") {
			if(lc($config->{disable_javascript_void}) eq "y") {
				print("      <a href=\"" . $config->{url} . "/" . $config->{imgs_dir} . $PNG1z . "\"><img src='" . $config->{url} . "/" . $config->{imgs_dir} . $PNG1 . "' border='0'></a>\n");
			}
			else {
				print("      <a href=\"javascript:void(window.open('" . $config->{url} . "/" . $config->{imgs_dir} . $PNG1z . "','','width=" . ($width + 115) . ",height=" . ($height + 100) . ",scrollbars=0,resizable=0'))\"><img src='" . $config->{url} . "/" . $config->{imgs_dir} . $PNG1 . "' border='0'></a>\n");
			}
		} else {
			print("      <img src='" . $config->{url} . "/" . $config->{imgs_dir} . $PNG1 . "'>\n");
		}
	}

	if($title) {
		print("    </td>\n");
		print("    <td valign='top' bgcolor='" . $colors->{title_bg_color} . "'>\n");
	}
	@riglim = @{setup_riglim($rigid[1], $limit[1])};
	undef(@tmp);
	undef(@tmpz);
	undef(@CDEF);
	push(@tmp, "LINE2:gpu0#FFA500:Card 0\\g");
	push(@tmp, "GPRINT:gpu0:LAST:\\:%3.0lf%%");
	push(@tmp, "LINE2:gpu3#4444EE:Card 3\\g");
	push(@tmp, "GPRINT:gpu3:LAST:\\:%3.0lf%%");
	push(@tmp, "LINE2:gpu6#EE44EE:Card 6\\g");
	push(@tmp, "GPRINT:gpu6:LAST:\\:%3.0lf%%\\n");
	push(@tmp, "LINE2:gpu1#44EEEE:Card 1\\g");
	push(@tmp, "GPRINT:gpu1:LAST:\\:%3.0lf%%");
	push(@tmp, "LINE2:gpu4#448844:Card 4\\g");
	push(@tmp, "GPRINT:gpu4:LAST:\\:%3.0lf%%");
	push(@tmp, "LINE2:gpu7#EEEE44:Card 7\\g");
	push(@tmp, "GPRINT:gpu7:LAST:\\:%3.0lf%%\\n");
	push(@tmp, "LINE2:gpu2#44EE44:Card 2\\g");
	push(@tmp, "GPRINT:gpu2:LAST:\\:%3.0lf%%");
	push(@tmp, "LINE2:gpu5#EE4444:Card 5\\g");
	push(@tmp, "GPRINT:gpu5:LAST:\\:%3.0lf%%");
	push(@tmp, "LINE2:gpu8#963C74:Card 8\\g");
	push(@tmp, "GPRINT:gpu8:LAST:\\:%3.0lf%%\\n");
	push(@tmpz, "LINE2:gpu0#FFA500:Card 0");
	push(@tmpz, "LINE2:gpu3#4444EE:Card 3");
	push(@tmpz, "LINE2:gpu6#EE44EE:Card 6");
	push(@tmpz, "LINE2:gpu1#44EEEE:Card 1");
	push(@tmpz, "LINE2:gpu4#448844:Card 4");
	push(@tmpz, "LINE2:gpu7#EEEE44:Card 7");
	push(@tmpz, "LINE2:gpu2#44EE44:Card 2");
	push(@tmpz, "LINE2:gpu5#EE4444:Card 5");
	push(@tmpz, "LINE2:gpu8#963C74:Card 8");
	if(lc($config->{show_gaps}) eq "y") {
		push(@tmp, "AREA:wrongdata#$colors->{gap}:");
		push(@tmpz, "AREA:wrongdata#$colors->{gap}:");
		push(@CDEF, "CDEF:wrongdata=allvalues,UN,INF,UNKN,IF");
	}
	($width, $height) = split('x', $config->{graph_size}->{small});
	if($silent =~ /imagetag/) {
		($width, $height) = split('x', $config->{graph_size}->{remote}) if $silent eq "imagetag";
		($width, $height) = split('x', $config->{graph_size}->{main}) if $silent eq "imagetagbig";
		@tmp = @tmpz;
		push(@tmp, "COMMENT: \\n");
	}
	RRDs::graph("$PNG_DIR" . "$PNG2",
		"--title=$config->{graphs}->{_nvidia2}  ($tf->{nwhen}$tf->{twhen})",
		"--start=-$tf->{nwhen}$tf->{twhen}",
		"--imgformat=PNG",
		"--vertical-label=Percent",
		"--width=$width",
		"--height=$height",
		@riglim,
		$zoom,
		@{$cgi->{version12}},
		@{$cgi->{version12_small}},
		@{$colors->{graph_colors}},
		"DEF:gpu0=$rrd:nvidia_gpu0:AVERAGE",
		"DEF:gpu1=$rrd:nvidia_gpu1:AVERAGE",
		"DEF:gpu2=$rrd:nvidia_gpu2:AVERAGE",
		"DEF:gpu3=$rrd:nvidia_gpu3:AVERAGE",
		"DEF:gpu4=$rrd:nvidia_gpu4:AVERAGE",
		"DEF:gpu5=$rrd:nvidia_gpu5:AVERAGE",
		"DEF:gpu6=$rrd:nvidia_gpu6:AVERAGE",
		"DEF:gpu7=$rrd:nvidia_gpu7:AVERAGE",
		"DEF:gpu8=$rrd:nvidia_gpu8:AVERAGE",
		"CDEF:allvalues=gpu0,gpu1,gpu2,gpu3,gpu4,gpu5,gpu6,gpu7,gpu8,+,+,+,+,+,+,+,+",
		@CDEF,
		"COMMENT: \\n",
		@tmp);
	$err = RRDs::error;
	print("ERROR: while graphing $PNG_DIR" . "$PNG2: $err\n") if $err;
	if(lc($config->{enable_zoom}) eq "y") {
		($width, $height) = split('x', $config->{graph_size}->{zoom});
		RRDs::graph("$PNG_DIR" . "$PNG2z",
			"--title=$config->{graphs}->{_nvidia2}  ($tf->{nwhen}$tf->{twhen})",
			"--start=-$tf->{nwhen}$tf->{twhen}",
			"--imgformat=PNG",
			"--vertical-label=Percent",
			"--width=$width",
			"--height=$height",
			@riglim,
			@{$cgi->{version12}},
			@{$cgi->{version12_small}},
			@{$colors->{graph_colors}},
			"DEF:gpu0=$rrd:nvidia_gpu0:AVERAGE",
			"DEF:gpu1=$rrd:nvidia_gpu1:AVERAGE",
			"DEF:gpu2=$rrd:nvidia_gpu2:AVERAGE",
			"DEF:gpu3=$rrd:nvidia_gpu3:AVERAGE",
			"DEF:gpu4=$rrd:nvidia_gpu4:AVERAGE",
			"DEF:gpu5=$rrd:nvidia_gpu5:AVERAGE",
			"DEF:gpu6=$rrd:nvidia_gpu6:AVERAGE",
			"DEF:gpu7=$rrd:nvidia_gpu7:AVERAGE",
			"DEF:gpu8=$rrd:nvidia_gpu8:AVERAGE",
			"CDEF:allvalues=gpu0,gpu1,gpu2,gpu3,gpu4,gpu5,gpu6,gpu7,gpu8,+,+,+,+,+,+,+,+",
			@CDEF,
			@tmpz);
		$err = RRDs::error;
		print("ERROR: while graphing $PNG_DIR" . "$PNG2z: $err\n") if $err;
	}
	if($title || ($silent =~ /imagetag/ && $graph =~ /nvidia2/)) {
		if(lc($config->{enable_zoom}) eq "y") {
			if(lc($config->{disable_javascript_void}) eq "y") {
				print("      <a href=\"" . $config->{url} . "/" . $config->{imgs_dir} . $PNG2z . "\"><img src='" . $config->{url} . "/" . $config->{imgs_dir} . $PNG2 . "' border='0'></a>\n");
			}
			else {
				print("      <a href=\"javascript:void(window.open('" . $config->{url} . "/" . $config->{imgs_dir} . $PNG2z . "','','width=" . ($width + 115) . ",height=" . ($height + 100) . ",scrollbars=0,resizable=0'))\"><img src='" . $config->{url} . "/" . $config->{imgs_dir} . $PNG2 . "' border='0'></a>\n");
			}
		} else {
			print("      <img src='" . $config->{url} . "/" . $config->{imgs_dir} . $PNG2 . "'>\n");
		}
	}

	@riglim = @{setup_riglim($rigid[2], $limit[2])};
	undef(@tmp);
	undef(@tmpz);
	undef(@CDEF);
	push(@tmp, "LINE2:mem0#FFA500:Card 0\\g");
	push(@tmp, "GPRINT:mem0:LAST:\\:%3.0lf%%");
	push(@tmp, "LINE2:mem3#4444EE:Card 3\\g");
	push(@tmp, "GPRINT:mem3:LAST:\\:%3.0lf%%");
	push(@tmp, "LINE2:mem6#EE44EE:Card 6\\g");
	push(@tmp, "GPRINT:mem6:LAST:\\:%3.0lf%%\\n");
	push(@tmp, "LINE2:mem1#44EEEE:Card 1\\g");
	push(@tmp, "GPRINT:mem1:LAST:\\:%3.0lf%%");
	push(@tmp, "LINE2:mem4#448844:Card 4\\g");
	push(@tmp, "GPRINT:mem4:LAST:\\:%3.0lf%%");
	push(@tmp, "LINE2:mem7#EEEE44:Card 7\\g");
	push(@tmp, "GPRINT:mem7:LAST:\\:%3.0lf%%\\n");
	push(@tmp, "LINE2:mem2#44EE44:Card 2\\g");
	push(@tmp, "GPRINT:mem2:LAST:\\:%3.0lf%%");
	push(@tmp, "LINE2:mem5#EE4444:Card 5\\g");
	push(@tmp, "GPRINT:mem5:LAST:\\:%3.0lf%%");
	push(@tmp, "LINE2:mem8#963C74:Card 8\\g");
	push(@tmp, "GPRINT:mem8:LAST:\\:%3.0lf%%\\n");
	push(@tmpz, "LINE2:mem0#FFA500:Card 0");
	push(@tmpz, "LINE2:mem3#4444EE:Card 3");
	push(@tmpz, "LINE2:mem6#EE44EE:Card 6");
	push(@tmpz, "LINE2:mem1#44EEEE:Card 1");
	push(@tmpz, "LINE2:mem4#448844:Card 4");
	push(@tmpz, "LINE2:mem7#EEEE44:Card 7");
	push(@tmpz, "LINE2:mem2#44EE44:Card 2");
	push(@tmpz, "LINE2:mem5#EE4444:Card 5");
	push(@tmpz, "LINE2:mem8#963C74:Card 8");
	if(lc($config->{show_gaps}) eq "y") {
		push(@tmp, "AREA:wrongdata#$colors->{gap}:");
		push(@tmpz, "AREA:wrongdata#$colors->{gap}:");
		push(@CDEF, "CDEF:wrongdata=allvalues,UN,INF,UNKN,IF");
	}
	($width, $height) = split('x', $config->{graph_size}->{small});
	if($silent =~ /imagetag/) {
		($width, $height) = split('x', $config->{graph_size}->{remote}) if $silent eq "imagetag";
		($width, $height) = split('x', $config->{graph_size}->{main}) if $silent eq "imagetagbig";
		@tmp = @tmpz;
		push(@tmp, "COMMENT: \\n");
	}
	RRDs::graph("$PNG_DIR" . "$PNG3",
		"--title=$config->{graphs}->{_nvidia3}  ($tf->{nwhen}$tf->{twhen})",
		"--start=-$tf->{nwhen}$tf->{twhen}",
		"--imgformat=PNG",
		"--vertical-label=Percent",
		"--width=$width",
		"--height=$height",
		@riglim,
		@{$cgi->{version12}},
		$zoom,
		@{$cgi->{version12_small}},
		@{$colors->{graph_colors}},
		"DEF:mem0=$rrd:nvidia_mem0:AVERAGE",
		"DEF:mem1=$rrd:nvidia_mem1:AVERAGE",
		"DEF:mem2=$rrd:nvidia_mem2:AVERAGE",
		"DEF:mem3=$rrd:nvidia_mem3:AVERAGE",
		"DEF:mem4=$rrd:nvidia_mem4:AVERAGE",
		"DEF:mem5=$rrd:nvidia_mem5:AVERAGE",
		"DEF:mem6=$rrd:nvidia_mem6:AVERAGE",
		"DEF:mem7=$rrd:nvidia_mem7:AVERAGE",
		"DEF:mem8=$rrd:nvidia_mem8:AVERAGE",
		"CDEF:allvalues=mem0,mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,+,+,+,+,+,+,+,+",
		@CDEF,
		"COMMENT: \\n",
		@tmp);
	$err = RRDs::error;
	print("ERROR: while graphing $PNG_DIR" . "$PNG3: $err\n") if $err;
	if(lc($config->{enable_zoom}) eq "y") {
		($width, $height) = split('x', $config->{graph_size}->{zoom});
		RRDs::graph("$PNG_DIR" . "$PNG3z",
			"--title=$config->{graphs}->{_nvidia3}  ($tf->{nwhen}$tf->{twhen})",
			"--start=-$tf->{nwhen}$tf->{twhen}",
			"--imgformat=PNG",
			"--vertical-label=Percent",
			"--width=$width",
			"--height=$height",
			@riglim,
			@{$cgi->{version12}},
			@{$cgi->{version12_small}},
			@{$colors->{graph_colors}},
			"DEF:mem0=$rrd:nvidia_mem0:AVERAGE",
			"DEF:mem1=$rrd:nvidia_mem1:AVERAGE",
			"DEF:mem2=$rrd:nvidia_mem2:AVERAGE",
			"DEF:mem3=$rrd:nvidia_mem3:AVERAGE",
			"DEF:mem4=$rrd:nvidia_mem4:AVERAGE",
			"DEF:mem5=$rrd:nvidia_mem5:AVERAGE",
			"DEF:mem6=$rrd:nvidia_mem6:AVERAGE",
			"DEF:mem7=$rrd:nvidia_mem7:AVERAGE",
			"DEF:mem8=$rrd:nvidia_mem8:AVERAGE",
			"CDEF:allvalues=mem0,mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,+,+,+,+,+,+,+,+",
			@CDEF,
			@tmpz);
		$err = RRDs::error;
		print("ERROR: while graphing $PNG_DIR" . "$PNG3z: $err\n") if $err;
	}
	if($title || ($silent =~ /imagetag/ && $graph =~ /nvidia3/)) {
		if(lc($config->{enable_zoom}) eq "y") {
			if(lc($config->{disable_javascript_void}) eq "y") {
				print("      <a href=\"" . $config->{url} . "/" . $config->{imgs_dir} . $PNG3z . "\"><img src='" . $config->{url} . "/" . $config->{imgs_dir} . $PNG3 . "' border='0'></a>\n");
			}
			else {
				print("      <a href=\"javascript:void(window.open('" . $config->{url} . "/" . $config->{imgs_dir} . $PNG3z . "','','width=" . ($width + 115) . ",height=" . ($height + 100) . ",scrollbars=0,resizable=0'))\"><img src='" . $config->{url} . "/" . $config->{imgs_dir} . $PNG3 . "' border='0'></a>\n");
			}
		} else {
			print("      <img src='" . $config->{url} . "/" . $config->{imgs_dir} . $PNG3 . "'>\n");
		}
	}

	if($title) {
		print("    </td>\n");
		print("    </tr>\n");
		main::graph_footer();
	}
	print("  <br>\n");
	return;
}

1;