feature: Added Nvidia GPU application support (#6024)

2024-10-07 16:52:45 +00:00 · 2017-03-03 20:41:20 +00:00
parent dc41c0eae1
commit 671654d155
21 changed files with 332 additions and 1 deletions
@@ -29,12 +29,14 @@ Different applications support a variety of ways collect data: by direct connect
 1. [Munin](#munin) - Agent
 1. [PHP-FPM](#php-fpm) - SNMP extend
 1. [Fail2ban](#fail2ban) - SNMP extend
+1. [Nvidia GPU](#nvidia-gpu) - SNMP extend
 1. [Squid](#squid) - SNMP proxy
 1. [FreeBSD NFS Server](#freebsd-nfs-server) - SNMP extend
 1. [FreeBSD NFS Client](#freebsd-nfs-client) - SNMP extend
 1. [Postgres](#postgres) - SNMP extend
 1. [Postfix](#postfix) - SNMP extend

+
 ### Apache
 Either use SNMP extend or use the agent.
 ##### SNMP Extend
@@ -524,6 +526,31 @@ In regards to the totals graphed there are two variables banned and firewalled.

 If you have more than a few jails configured, you may need to use caching as each jail needs to be polled and fail2ban-client can't do so in a timely manner for than a few. This can result in failure of other SNMP information being polled.

+### Nvidia GPU
+
+##### SNMP Extend
+
+1: Copy the shell script, nvidia, to the desired host (the host must be added to LibreNMS devices) (wget https://github.com/librenms/librenms-agent/raw/master/snmp/nvidia -O /etc/snmp/nvidia)
+
+2: Make the script executable (chmod +x /etc/snmp/nvidia)
+
+3: Edit your snmpd.conf file and add:
+```
+extend nvidia /etc/snmp/nvidia
+```
+
+5: Restart snmpd on your host.
+
+6: Verify you have nvidia-smi installed, which it generally should be if you have the driver from Nvida installed.
+
+7: On the device page in Librenms, edit your host and check `Nvidia` under the Applications tab.
+
+The GPU numbering on the graphs will correspond to how the nvidia-smi sees them as being.
+
+For questions about what the various values are/mean, please see the nvidia-smi man file under the section covering dmon.
+
+Please be aware that if you have more than 35 GPUs, you will need to add more colors to the config entry $config['graph_colours']['manycolours'].
+=======
 #### Squid

 ##### SNMP Proxy
@@ -633,3 +660,4 @@ extend postfixdetailed /etc/snmp/postfixdetailed
 7: On the device page in Librenms, edit your host and check `Postfix` under the Applications tab. Before doing this, run /etc/snmp/postfixdetailed to create the initial cache file so you don't end up with some crazy initial starting value.

 Please note that each time /etc/snmp/postfixdetailed is ran, the cache file is updated, so if this happens in between LibreNMS doing it then the values will be thrown off for that polling period.
+
@@ -0,0 +1,33 @@
+<?php
+$name = 'nvidia';
+$app_id = $app['app_id'];
+$scale_min     = 0;
+$colours       = 'manycolours';
+$dostack       = 0;
+$printtotal    = 0;
+$addarea       = 1;
+$transparency  = 15;
+$name = 'nvidia';
+$app_id = $app['app_id'];
+$scale_min     = 0;
+
+$int=0;
+$rrd_list=array();
+$rrd_filename=rrd_name($device['hostname'], array('app', $app['app_type'], $app['app_id'], $int));
+
+if (!rrdtool_check_rrd_exists($rrd_filename)) {
+    echo "file missing: $rrd_filename";
+}
+
+while (rrdtool_check_rrd_exists($rrd_filename)) {
+    $rrd_list[]=array(
+        'filename' => $rrd_filename,
+        'descr'    => 'GPU '.$int,
+        'ds'       => $rrdVar,
+    );
+
+    $int++;
+    $rrd_filename=rrd_name($device['hostname'], array('app', $app['app_type'], $app['app_id'], $int));
+}
+
+require 'includes/graphs/generic_multi_line_exact_numbers.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'Bar1 MB';
+$unitlen       = 8;
+$bigdescrlen   = 8;
+$smalldescrlen = 8;
+
+$rrdVar='bar1';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'Double Bit ECC';
+$unitlen       = 15;
+$bigdescrlen   = 15;
+$smalldescrlen = 15;
+
+$rrdVar='dbecc';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'dec use %';
+$unitlen       = 10;
+$bigdescrlen   = 5;
+$smalldescrlen = 5;
+
+$rrdVar='dec';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'enc use %';
+$unitlen       = 10;
+$bigdescrlen   = 5;
+$smalldescrlen = 5;
+
+$rrdVar='enc';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'FB MB';
+$unitlen       = 6;
+$bigdescrlen   = 6;
+$smalldescrlen = 6;
+
+$rrdVar='fb';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'RAM MHz';
+$unitlen       = 9;
+$bigdescrlen   = 9;
+$smalldescrlen = 9;
+
+$rrdVar='mclk';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'mem use %';
+$unitlen       = 10;
+$bigdescrlen   = 5;
+$smalldescrlen = 5;
+
+$rrdVar='mem';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'GPU MHz';
+$unitlen       = 9;
+$bigdescrlen   = 9;
+$smalldescrlen = 9;
+
+$rrdVar='pclk';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'Therm Voil. %';
+$unitlen       = 15;
+$bigdescrlen   = 15;
+$smalldescrlen = 15;
+
+$rrdVar='pviol';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'Watts';
+$unitlen       = 5;
+$bigdescrlen   = 5;
+$smalldescrlen = 5;
+
+$rrdVar='pwr';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'RX MB/s';
+$unitlen       = 8;
+$bigdescrlen   = 8;
+$smalldescrlen = 8;
+
+$rrdVar='rxpci';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'Single Bit ECC';
+$unitlen       = 15;
+$bigdescrlen   = 15;
+$smalldescrlen = 15;
+
+$rrdVar='sbecc';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'GPU use %';
+$unitlen       = 10;
+$bigdescrlen   = 7;
+$smalldescrlen = 7;
+
+$rrdVar='sm';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'C';
+$unitlen       = 5;
+$bigdescrlen   = 5;
+$smalldescrlen = 5;
+
+$rrdVar='temp';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'Therm Voil';
+$unitlen       = 15;
+$bigdescrlen   = 15;
+$smalldescrlen = 15;
+
+$rrdVar='tviol';
+
+require 'nvidia-common.inc.php';
@@ -0,0 +1,9 @@
+<?php
+$unit_text     = 'TX MB/s';
+$unitlen       = 8;
+$bigdescrlen   = 8;
+$smalldescrlen = 8;
+
+$rrdVar='txpci';
+
+require 'nvidia-common.inc.php';
@@ -158,6 +158,25 @@ $graphs['php-fpm'] = array(
    'stats'
 );

+$graphs['nvidia'] = array(
+    'sm',
+    'mem',
+    'enc',
+    'dec',
+    'rxpci',
+    'txpci',
+    'fb',
+    'bar1',
+    'mclk',
+    'pclk',
+    'pwr',
+    'temp',
+    'pviol',
+    'tviol',
+    'sbecc',
+    'dbecc',
+);
+
 $graphs['squid'] = array(
    'memory',
    'clients',
@@ -172,7 +191,7 @@ $graphs['squid'] = array(
    'bytehit',
    'sysnumread',
    'pagefaults',
-    'cputime'
+    'cputime',
 );

 $graphs['fbsd-nfs-server'] = array(
@@ -0,0 +1,42 @@
+<?php
+
+global $config;
+
+$graphs = array(
+    'nvidia_sm' => 'GPU Utilization',
+    'nvidia_mem' => 'Memory Utilization',
+    'nvidia_enc' => 'Encoder Utilization',
+    'nvidia_dec' => 'Decoder Utilization',
+    'nvidia_fb' => 'Frame Buffer Memory Usage',
+    'nvidia_bar1' => 'Bar1 Memory Usage',
+    'nvidia_rxpci' => 'PCIe RX',
+    'nvidia_txpci' => 'PCIe TX',
+    'nvidia_pwr' => 'Power Usage',
+    'nvidia_temp' => 'Temperature',
+    'nvidia_mclk' => 'Memory Clock',
+    'nvidia_pclk' => 'GPU Clock',
+    'nvidia_pviol' => 'Thermal Violation Percentage',
+    'nvidia_tviol' => 'Thermal Violation Boolean',
+    'nvidia_sbecc' => 'Single Bit ECC Errors',
+    'nvidia_dbecc' => 'Double Bit ECC Errors',
+);
+
+foreach ($graphs as $key => $text) {
+    $graph_type            = $key;
+    $graph_array['height'] = '100';
+    $graph_array['width']  = '215';
+    $graph_array['to']     = $config['time']['now'];
+    $graph_array['id']     = $app['app_id'];
+    $graph_array['type']   = 'application_'.$key;
+
+    echo '<div class="panel panel-default">
+    <div class="panel-heading">
+        <h3 class="panel-title">'.$text.'</h3>
+    </div>
+    <div class="panel-body">
+    <div class="row">';
+    include 'includes/print-graphrow.inc.php';
+    echo '</div>';
+    echo '</div>';
+    echo '</div>';
+}
@@ -0,0 +1,65 @@
+<?php
+
+use LibreNMS\RRD\RrdDefinition;
+
+$name = 'nvidia';
+$app_id = $app['app_id'];
+
+$options      = '-O qv';
+$mib          = 'NET-SNMP-EXTEND-MIB';
+$oid          = 'nsExtendOutputFull.6.110.118.105.100.105.97';
+$gpus = snmp_walk($device, $oid, $options, $mib);
+
+$gpuArray = explode("\n", $gpus);
+
+$rrd_def = RrdDefinition::make()
+    ->addDataset('pwr', 'GAUGE', 0)
+    ->addDataset('temp', 'GAUGE', 0)
+    ->addDataset('sm', 'GAUGE', 0)
+    ->addDataset('mem', 'GAUGE', 0)
+    ->addDataset('enc', 'GAUGE', 0)
+    ->addDataset('dec', 'GAUGE', 0)
+    ->addDataset('mclk', 'GAUGE', 0)
+    ->addDataset('pclk', 'GAUGE', 0)
+    ->addDataset('pviol', 'GAUGE', 0)
+    ->addDataset('tviol', 'GAUGE', 0)
+    ->addDataset('fb', 'GAUGE', 0)
+    ->addDataset('bar1', 'GAUGE', 0)
+    ->addDataset('sbecc', 'GAUGE', 0)
+    ->addDataset('dbecc', 'GAUGE', 0)
+    ->addDataset('pci', 'GAUGE', 0)
+    ->addDataset('rxpci', 'GAUGE', 0)
+    ->addDataset('txpci', 'GAUGE', 0);
+
+$int=0;
+while (isset($gpuArray[$int])) {
+    list($gpu, $pwr, $temp, $sm, $mem, $enc, $dec, $mclk, $pclk, $pviol, $tviol,
+        $fb, $bar1, $sbecc, $dbecc, $pci, $rxpci, $txpci)=explode(",", $gpuArray[$int]);
+
+        $rrd_name = array('app', $name, $app_id, $int);
+
+        $fields = array(
+            'pwr' => $pwr,
+            'temp' => $temp,
+            'sm' => $sm,
+            'mem' => $mem,
+            'enc' => $enc,
+            'dec' => $dec,
+            'mclk' => $mclk,
+            'pclk' => $pclk,
+            'pviol' => $pviol,
+            'tviol' => $tviol,
+            'fb' => $fb,
+            'bar1' => $bar1,
+            'sbecc' => $sbecc,
+            'dbecc' => $dbecc,
+            'pci' => $pci,
+            'rxpci' => $rxpci,
+            'txpci' => $txpci
+        );
+
+        $tags = array('name' => $name, 'app_id' => $app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name);
+        data_update($device, 'app', $tags, $fields);
+
+    $int++;
+}