diff --git a/doc/Extensions/Applications.md b/doc/Extensions/Applications.md index a69e68bd66..67d5894092 100644 --- a/doc/Extensions/Applications.md +++ b/doc/Extensions/Applications.md @@ -2905,32 +2905,22 @@ b.) "public_key_to_arbitrary_name" - A dictionary to convert between the publick ### SNMP Extend -`zfs-linux` requires python3 >=python3.5. - -The installation steps are: - -1. Copy the polling script to the desired host (the host must be added - to LibreNMS devices) -2. Make the script executable -3. Edit snmpd.conf to include ZFS stats - -#### FreeBSD +1: Install the depends. ``` -wget https://github.com/librenms/librenms-agent/raw/master/snmp/zfs-freebsd -O /etc/snmp/zfs-freebsd -chmod +x /etc/snmp/zfs-freebsd -echo "extend zfs /etc/snmp/zfs-freebsd" >> /etc/snmp/snmpd.conf +### FreeBSD +pkg install p5-JSON p5-MIME-Base64 p5-Gzip-Faster +### Debian +apt-get install -y cpanminus zlib1g-dev +cpanm Mime::Base64 JSON Gzip::Faster ``` -#### Linux +2: Fetch the script in question and make it executable. ``` -wget https://github.com/librenms/librenms-agent/raw/master/snmp/zfs-linux -O /etc/snmp/zfs-linux -chmod +x /etc/snmp/zfs-linux -echo "extend zfs /usr/bin/sudo /etc/snmp/zfs-linux" >> /etc/snmp/snmpd.conf +wget https://github.com/librenms/librenms-agent/raw/master/snmp/zfs -O /etc/snmp/zfs +chmod +x /etc/snmp/zfs ``` -Edit your sudo users (usually `visudo`) and add at the bottom: +3: Add the following to snmpd.conf and restart snmpd. ``` -snmp ALL=(ALL) NOPASSWD: /etc/snmp/zfs-linux +extend zfs /etc/snmp/zfs ``` - -Now restart snmpd and you're all set. diff --git a/includes/html/graphs/application/zfs_l2_abort_lowmem.inc.php b/includes/html/graphs/application/zfs_l2_abort_lowmem.inc.php new file mode 100644 index 0000000000..c978255539 --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_abort_lowmem.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_access_total.inc.php b/includes/html/graphs/application/zfs_l2_access_total.inc.php new file mode 100644 index 0000000000..1a7be22805 --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_access_total.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_asize.inc.php b/includes/html/graphs/application/zfs_l2_asize.inc.php new file mode 100644 index 0000000000..30794b5dab --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_asize.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_bufc_d_asize.inc.php b/includes/html/graphs/application/zfs_l2_bufc_d_asize.inc.php new file mode 100644 index 0000000000..7db171b436 --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_bufc_d_asize.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_bufc_m_asize.inc.php b/includes/html/graphs/application/zfs_l2_bufc_m_asize.inc.php new file mode 100644 index 0000000000..b8f8773f1b --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_bufc_m_asize.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_d_to_m_ratio.inc.php b/includes/html/graphs/application/zfs_l2_d_to_m_ratio.inc.php new file mode 100644 index 0000000000..93064f5982 --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_d_to_m_ratio.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_errors.inc.php b/includes/html/graphs/application/zfs_l2_errors.inc.php new file mode 100644 index 0000000000..3af89da93a --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_errors.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_hdr_size.inc.php b/includes/html/graphs/application/zfs_l2_hdr_size.inc.php new file mode 100644 index 0000000000..838501ada0 --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_hdr_size.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_log_blk_asize.inc.php b/includes/html/graphs/application/zfs_l2_log_blk_asize.inc.php new file mode 100644 index 0000000000..867a6552f3 --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_log_blk_asize.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_mfu_asize.inc.php b/includes/html/graphs/application/zfs_l2_mfu_asize.inc.php new file mode 100644 index 0000000000..fc503c5fca --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_mfu_asize.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_mru_asize.inc.php b/includes/html/graphs/application/zfs_l2_mru_asize.inc.php new file mode 100644 index 0000000000..ec483565a7 --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_mru_asize.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_prefetch_asize.inc.php b/includes/html/graphs/application/zfs_l2_prefetch_asize.inc.php new file mode 100644 index 0000000000..8c173c393e --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_prefetch_asize.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_rb_asize.inc.php b/includes/html/graphs/application/zfs_l2_rb_asize.inc.php new file mode 100644 index 0000000000..a5d23b5c0f --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_rb_asize.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_rw_bytes.inc.php b/includes/html/graphs/application/zfs_l2_rw_bytes.inc.php new file mode 100644 index 0000000000..ba3eca0736 --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_rw_bytes.inc.php @@ -0,0 +1,29 @@ +app_id, '_____group2']); + +$rrd_list = []; +if (Rrd::checkRrdExists($rrd_filename)) { + $rrd_list[] = [ + 'filename' => $rrd_filename, + 'descr' => 'L2 Write', + 'ds' => 'l2_write_bytes', + ]; + $rrd_list[] = [ + 'filename' => $rrd_filename, + 'descr' => 'L2 Read', + 'ds' => 'l2_read_bytes', + ]; +} else { + d_echo('RRD "' . $rrd_filename . '" not found'); +} + +require 'includes/html/graphs/generic_multi_line.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_size.inc.php b/includes/html/graphs/application/zfs_l2_size.inc.php new file mode 100644 index 0000000000..46f89867ab --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_size.inc.php @@ -0,0 +1,15 @@ +app_id, '_____group2']); + +if (! Rrd::checkRrdExists($filename)) { + d_echo('RRD "' . $filename . '" not found'); +} + +require 'includes/html/graphs/generic_stats.inc.php'; diff --git a/includes/html/graphs/application/zfs_l2_sizes.inc.php b/includes/html/graphs/application/zfs_l2_sizes.inc.php new file mode 100644 index 0000000000..e90084f92a --- /dev/null +++ b/includes/html/graphs/application/zfs_l2_sizes.inc.php @@ -0,0 +1,69 @@ +app_id, '_____group2']); + +$rrd_list = []; +if (Rrd::checkRrdExists($rrd_filename)) { + $rrd_list[] = [ + 'filename' => $rrd_filename, + 'descr' => 'A Size', + 'ds' => 'l2_asize', + ]; + $rrd_list[] = [ + 'filename' => $rrd_filename, + 'descr' => 'BufC D Asize', + 'ds' => 'l2_bufc_d_asize', + ]; + $rrd_list[] = [ + 'filename' => $rrd_filename, + 'descr' => 'BufC M Asize', + 'ds' => 'l2_bufc_m_asize', + ]; + $rrd_list[] = [ + 'filename' => $rrd_filename, + 'descr' => 'Log Blk Asize', + 'ds' => 'l2_log_blk_asize', + ]; + $rrd_list[] = [ + 'filename' => $rrd_filename, + 'descr' => 'MFU Asize', + 'ds' => 'l2_mfu_asize', + ]; + $rrd_list[] = [ + 'filename' => $rrd_filename, + 'descr' => 'MRU Asize', + 'ds' => 'l2_mru_asize', + ]; + $rrd_list[] = [ + 'filename' => $rrd_filename, + 'descr' => 'Prefetch Asize', + 'ds' => 'l2_prefetch_asize', + ]; + $rrd_list[] = [ + 'filename' => $rrd_filename, + 'descr' => 'Rebuild Asize', + 'ds' => 'l2_rb_asize', + ]; + $rrd_list[] = [ + 'filename' => $rrd_filename, + 'descr' => 'Log Blk Avg Asize', + 'ds' => 'l2_log_blk_avg_as', + ]; + $rrd_list[] = [ + 'filename' => $rrd_filename, + 'descr' => 'HDR Size', + 'ds' => 'l2_hdr_size', + ]; +} else { + d_echo('RRD "' . $rrd_filename . '" not found'); +} + +require 'includes/html/graphs/generic_multi_line.inc.php'; diff --git a/includes/html/pages/device/apps/zfs.inc.php b/includes/html/pages/device/apps/zfs.inc.php index 8bd03edfc4..c4e5ba8d89 100644 --- a/includes/html/pages/device/apps/zfs.inc.php +++ b/includes/html/pages/device/apps/zfs.inc.php @@ -10,7 +10,8 @@ $link_array = [ print_optionbar_start(); echo generate_link('ARC', $link_array); -echo '| Pools:'; +echo ' | ' . generate_link('L2', $link_array, ['zfs_page'=>'l2']); +echo ' | Pools: '; $pools = $app->data['pools'] ?? []; sort($pools); @@ -28,7 +29,33 @@ foreach ($pools as $index => $pool) { print_optionbar_end(); -if (! isset($vars['pool'])) { +if (isset($vars['pool'])) { + $graphs = [ + 'zfs_pool_space'=>'Pool Space', + 'zfs_pool_cap'=>'Pool Capacity', + 'zfs_pool_frag'=>'Pool Fragmentation', + ]; +} elseif (isset($vars['zfs_page']) && $vars['zfs_page'] == 'l2') { + $graphs = [ + 'zfs_l2_size' => 'L2 size in bytes', + 'zfs_l2_rw_bytes' => 'L2 Read And Writes Bytes Per Second', + 'zfs_l2_d_to_m_ratio' => 'L2 Data To Meta Ratio', + 'zfs_l2_access_total' => 'L2 Total Hits And Misses Per Second', + 'zfs_l2_errors' => 'L2 Errors Per Second', + 'zfs_l2_errors' => 'L2 Error Types Per Second', + 'zfs_l2_sizes' => 'L2 Sizes', + 'zfs_l2_asize' => 'L2 Asize', + 'zfs_l2_bufc_d_asize' => 'L2 BufC Data Asize', + 'zfs_l2_bufc_m_asize' => 'L2 BufC Metadata Asize', + 'zfs_l2_hdr_size' => 'L2 HDR Size', + 'zfs_l2_log_blk_asize' => 'L2 Log Blk Asize', + 'zfs_l2_mfu_asize' => 'L2 MFU Asize', + 'zfs_l2_mru_asize' => 'L2 MRU Asize', + 'zfs_l2_prefetch_asize' => 'L2 Prefetch Asize', + 'zfs_l2_rb_asize' => 'L2 Rebuild Asize', + 'zfs_l2_abort_lowmem' => 'L2 Abort Lowmem Per Second', + ]; +} else { $graphs = [ 'zfs_arc_misc' => 'ARC misc', 'zfs_arc_size' => 'ARC size in bytes', @@ -41,12 +68,6 @@ if (! isset($vars['pool'])) { 'zfs_arc_cache_hits' => 'ARC cache hits', 'zfs_arc_cache_miss' => 'ARC cache misses', ]; -} else { - $graphs = [ - 'zfs_pool_space'=>'Pool Space', - 'zfs_pool_cap'=>'Pool Capacity', - 'zfs_pool_frag'=>'Pool Fragmentation', - ]; } foreach ($graphs as $key => $text) { diff --git a/includes/polling/applications/zfs.inc.php b/includes/polling/applications/zfs.inc.php index ab1d585e7f..5e05eaa6b4 100644 --- a/includes/polling/applications/zfs.inc.php +++ b/includes/polling/applications/zfs.inc.php @@ -135,6 +135,102 @@ $fields = [ $tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name]; data_update($device, 'app', $tags, $fields); +// name choosen based on this is the second group of variables +$rrd_name = ['app', $name, $app->app_id, '_____group2']; +$rrd_def = RrdDefinition::make() + ->addDataset('l2_abort_lowmem', 'DERIVE', 0) + ->addDataset('l2_access_total', 'DERIVE', 0) + ->addDataset('l2_asize', 'GAUGE', 0) + ->addDataset('l2_bufc_d_asize', 'GAUGE', 0) + ->addDataset('l2_bufc_m_asize', 'GAUGE', 0) + ->addDataset('l2_cksum_bad', 'DERIVE', 0) + ->addDataset('l2_d_to_m_ratio', 'GAUGE', 0) + ->addDataset('l2_errors', 'DERIVE', 0) + ->addDataset('l2_evict_l1cached', 'DERIVE', 0) + ->addDataset('l2_evict_l_retry', 'DERIVE', 0) + ->addDataset('l2_evict_reading', 'DERIVE', 0) + ->addDataset('l2_feeds', 'DERIVE', 0) + ->addDataset('l2_free_on_write', 'DERIVE', 0) + ->addDataset('l2_hdr_size', 'GAUGE', 0) + ->addDataset('l2_hits', 'DERIVE', 0) + ->addDataset('l2_io_error', 'DERIVE', 0) + ->addDataset('l2_log_blk_asize', 'GAUGE', 0) + ->addDataset('l2_log_blk_avg_as', 'DERIVE', 0) + ->addDataset('l2_log_blk_count', 'DERIVE', 0) + ->addDataset('l2_log_blk_writes', 'DERIVE', 0) + ->addDataset('l2_mfu_asize', 'GAUGE', 0) + ->addDataset('l2_misses', 'DERIVE', 0) + ->addDataset('l2_mru_asize', 'GAUGE', 0) + ->addDataset('l2_prefetch_asize', 'GAUGE', 0) + ->addDataset('l2_read_bytes', 'DERIVE', 0) + ->addDataset('l2_rb_asize', 'GAUGE', 0) + ->addDataset('l2_rb_bufs', 'DERIVE', 0) + ->addDataset('l2_rb_bufs_prec', 'DERIVE', 0) + ->addDataset('l2_rb_csum_lb_err', 'DERIVE', 0) + ->addDataset('l2_rb_dh_err', 'DERIVE', 0) + ->addDataset('l2_rb_io_errors', 'DERIVE', 0) + ->addDataset('l2_rb_log_blks', 'DERIVE', 0) + ->addDataset('l2_rb_lowmem', 'DERIVE', 0) + ->addDataset('l2_rb_size', 'GAUGE', 0) + ->addDataset('l2_rb_success', 'DERIVE', 0) + ->addDataset('l2_rb_unsup', 'DERIVE', 0) + ->addDataset('l2_rw_clash', 'DERIVE', 0) + ->addDataset('l2_size', 'GAUGE', 0) + ->addDataset('l2_write_bytes', 'DERIVE', 0) + ->addDataset('l2_writes_done', 'DERIVE', 0) + ->addDataset('l2_writes_error', 'DERIVE', 0) + ->addDataset('l2_writes_l_retry', 'DERIVE', 0) + ->addDataset('l2_writes_sent', 'DERIVE', 0); + +$fields = [ + 'l2_abort_lowmem' => $zfs['l2_abort_lowmem'], + 'l2_access_total' => $zfs['l2_access_total'], + 'l2_asize' => $zfs['l2_asize'], + 'l2_bufc_d_asize' => $zfs['l2_bufc_data_asize'], + 'l2_bufc_m_asize' => $zfs['l2_bufc_metadata_asize'], + 'l2_cksum_bad' => $zfs['l2_cksum_bad'], + 'l2_d_to_m_ratio' => $zfs['l2_data_to_meta_ratio'], + 'l2_errors' => $zfs['l2_errors'], + 'l2_evict_l1cached' => $zfs['l2_evict_l1cached'], + 'l2_evict_l_retry' => $zfs['l2_evict_lock_retry'], + 'l2_evict_reading' => $zfs['l2_evict_reading'], + 'l2_feeds' => $zfs['l2_feeds'], + 'l2_free_on_write' => $zfs['l2_free_on_write'], + 'l2_hdr_size' => $zfs['l2_hdr_size'], + 'l2_hits' => $zfs['l2_hits'], + 'l2_io_error' => $zfs['l2_io_error'], + 'l2_log_blk_asize' => $zfs['l2_log_blk_asize'], + 'l2_log_blk_avg_as' => $zfs['l2_log_blk_avg_asize'], + 'l2_log_blk_count' => $zfs['l2_log_blk_count'], + 'l2_log_blk_writes' => $zfs['l2_log_blk_writes'], + 'l2_mfu_asize' => $zfs['l2_mfu_asize'], + 'l2_misses' => $zfs['l2_misses'], + 'l2_mru_asize' => $zfs['l2_mru_asize'], + 'l2_prefetch_asize' => $zfs['l2_prefetch_asize'], + 'l2_read_bytes' => $zfs['l2_read_bytes'], + 'l2_rb_asize' => $zfs['l2_rebuild_asize'], + 'l2_rb_bufs' => $zfs['l2_rebuild_bufs'], + 'l2_rb_bufs_prec' => $zfs['l2_rebuild_bufs_precached'], + 'l2_rb_csum_lb_err' => $zfs['l2_rebuild_cksum_lb_errors'], + 'l2_rb_dh_err' => $zfs['l2_rebuild_dh_errors'], + 'l2_rb_io_errors' => $zfs['l2_rebuild_io_errors'], + 'l2_rb_log_blks' => $zfs['l2_rebuild_log_blks'], + 'l2_rb_lowmem' => $zfs['l2_rebuild_lowmem'], + 'l2_rb_size' => $zfs['l2_rebuild_size'], + 'l2_rb_success' => $zfs['l2_rebuild_success'], + 'l2_rb_unsup' => $zfs['l2_rebuild_unsupported'], + 'l2_rw_clash' => $zfs['l2_rw_clash'], + 'l2_size' => $zfs['l2_size'], + 'l2_write_bytes' => $zfs['l2_write_bytes'], + 'l2_writes_done' => $zfs['l2_writes_done'], + 'l2_writes_error' => $zfs['l2_writes_error'], + 'l2_writes_l_retry' => $zfs['l2_writes_lock_retry'], + 'l2_writes_sent' => $zfs['l2_writes_sent'], +]; + +$tags = ['name' => $name, 'app_id' => $app->app_id, 'rrd_def' => $rrd_def, 'rrd_name' => $rrd_name]; +data_update($device, 'app', $tags, $fields); + // // process additional info returned // @@ -174,6 +270,29 @@ foreach ($zfs['pools'] as $pool) { } } +// gets the pool health status +$old_health = $app->data['health'] ?? 1; +if (isset($zfs['health'])) { + $health = $zfs['health']; + if ($old_health != $zfs['health']) { + if ($zfs['health'] == 1) { + log_event('ZFS pool(s) now healthy', $device, 'application', 1); + } else { + log_event('ZFS pool(s) DEGRADED, FAULTED, UNAVAIL, REMOVED, or unknown', $device, 'application', 5); + } + } +} else { + $health = 1; +} + +// gets the l2 error status +$old_l2_errors = $app->data['l2_errors'] ?? 0; +if (isset($zfs['l2_errors'])) { + if ($old_l2_errors != $zfs['l2_errors']) { + log_event('ZFS L2 cache has experienced errors', $device, 'application', 5); + } +} + // check for added or removed pools $old_pools = $app->data['pools'] ?? []; $added_pools = array_diff($pools, $old_pools); @@ -181,11 +300,11 @@ $removed_pools = array_diff($old_pools, $pools); // if we have any source pools, save and log if (count($added_pools) > 0 || count($removed_pools) > 0) { - $app->data = ['pools' => $pools]; $log_message = 'ZFS Pool Change:'; $log_message .= count($added_pools) > 0 ? ' Added ' . implode(',', $added_pools) : ''; $log_message .= count($removed_pools) > 0 ? ' Removed ' . implode(',', $added_pools) : ''; log_event($log_message, $device, 'application'); } +$app->data = ['pools' => $pools, 'health' => $health, 'l2_errors' => $zfs['l2_errors']]; update_application($app, 'OK', $metrics); diff --git a/misc/alert_rules.json b/misc/alert_rules.json index 8cec166086..93b6ad21f8 100644 --- a/misc/alert_rules.json +++ b/misc/alert_rules.json @@ -593,6 +593,16 @@ "name": "Systemd Services Failed > 0", "severity": "warning" }, + { + "rule": "applications.app_type = \"zfs\" && application_metrics.metric = \"l2_errors\" && application_metrics.value >= \"1\"", + "name": "ZFS L2 errors present", + "severity": "critical" + }, + { + "rule": "applications.app_type = \"zfs\" && application_metrics.metric = \"health\" && application_metrics.value = \"0\"", + "name": "ZFS has one or more pool DEGRADED, FAULTED, UNAVAIL, REMOVED, or other", + "severity": "critical" + }, { "rule": "applications.app_type = \"linux_config_files\" && application_metrics.metric = \"number_of_confs\" && application_metrics.value > \"0\"", "name": "linux_config_files Configuration Files Out-Of-Sync > 0", diff --git a/tests/data/linux_zfs-legacy.json b/tests/data/linux_zfs-legacy.json index 18a835b0f2..45752b7e51 100644 --- a/tests/data/linux_zfs-legacy.json +++ b/tests/data/linux_zfs-legacy.json @@ -22,7 +22,7 @@ "app_state_prev": "UNKNOWN", "app_status": "", "app_instance": "", - "data": "{\"pools\":[\"arc\"]}" + "data": "{\"pools\":[\"arc\"],\"health\":1,\"l2_errors\":null}" } ], "application_metrics": [ diff --git a/tests/data/linux_zfs-v1-compressed.json b/tests/data/linux_zfs-v1-compressed.json index f6a8b78ac3..5051459d56 100644 --- a/tests/data/linux_zfs-v1-compressed.json +++ b/tests/data/linux_zfs-v1-compressed.json @@ -22,7 +22,7 @@ "app_state_prev": "UNKNOWN", "app_status": "", "app_instance": "", - "data": "{\"pools\":[\"arc\"]}" + "data": "{\"pools\":[\"arc\"],\"health\":1,\"l2_errors\":null}" } ], "application_metrics": [ diff --git a/tests/data/linux_zfs-v1.json b/tests/data/linux_zfs-v1.json index 18a835b0f2..45752b7e51 100644 --- a/tests/data/linux_zfs-v1.json +++ b/tests/data/linux_zfs-v1.json @@ -22,7 +22,7 @@ "app_state_prev": "UNKNOWN", "app_status": "", "app_instance": "", - "data": "{\"pools\":[\"arc\"]}" + "data": "{\"pools\":[\"arc\"],\"health\":1,\"l2_errors\":null}" } ], "application_metrics": [ diff --git a/tests/data/linux_zfs-v3.json b/tests/data/linux_zfs-v3.json new file mode 100644 index 0000000000..c01709f721 --- /dev/null +++ b/tests/data/linux_zfs-v3.json @@ -0,0 +1,734 @@ +{ + "applications": { + "discovery": { + "applications": [ + { + "app_type": "zfs", + "app_state": "UNKNOWN", + "discovered": 1, + "app_state_prev": null, + "app_status": "", + "app_instance": "", + "data": null + } + ] + }, + "poller": { + "applications": [ + { + "app_type": "zfs", + "app_state": "OK", + "discovered": 1, + "app_state_prev": "UNKNOWN", + "app_status": "", + "app_instance": "", + "data": "{\"pools\":[\"arc\"],\"health\":1,\"l2_errors\":0}" + } + ], + "application_metrics": [ + { + "metric": "actual_hit_per", + "value": 96.09917208021, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "anon_hits", + "value": -2439949, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "anon_hits_per", + "value": -0.21443943717893, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "arc_accesses_total", + "value": 1183187927, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "arc_hits", + "value": 1137826620, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "arc_misses", + "value": 45361307, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "arc_size", + "value": 1048270624, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "arc_size_per", + "value": 3.1581346931151, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "cache_hits_per", + "value": 96.166179018154, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "cache_miss_per", + "value": 3.8338209818465, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "data_demand_per", + "value": 95.461167284163, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "data_pre_per", + "value": 8.2718806970289, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "degraded", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "deleted", + "value": 42382148, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_data_hits", + "value": 1015682, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_data_misses", + "value": 48292, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_data_total", + "value": 1063974, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_hits_per", + "value": 0.089265093833013, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_meta_hits", + "value": 1133980912, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_meta_misses", + "value": 31377034, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "demand_misses_per", + "value": 0.10646077724348, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "evict_l2_cached", + "value": 227948678144, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "evict_l2_eligible", + "value": 128654282240, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "evict_l2_eligible_mfu", + "value": 28666240512, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "evict_l2_eligible_mru", + "value": 99988041728, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "evict_l2_ineligible", + "value": 94654349824, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "evict_l2_skip", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "evict_skip", + "value": 133213142, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "faulted", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "freq_used_per", + "value": 32.977470290171, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "health", + "value": 1, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_abort_lowmem", + "value": 2264908, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_access_total", + "value": 45361287, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_asize", + "value": 59218449920, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_bufc_data_asize", + "value": 33123006976, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_bufc_metadata_asize", + "value": 26095442944, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_cksum_bad", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_data_to_meta_ratio", + "value": 374, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_errors", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_evict_l1cached", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_evict_lock_retry", + "value": 1, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_evict_reading", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_feeds", + "value": 27311, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_free_on_write", + "value": 112, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_hdr_size", + "value": 808407552, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_hits", + "value": 30337921, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_io_error", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_log_blk_asize", + "value": 120487424, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_log_blk_avg_asize", + "value": 11287, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_log_blk_count", + "value": 8863, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_log_blk_writes", + "value": 586, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_mfu_asize", + "value": 29688845824, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_misses", + "value": 15023366, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_mru_asize", + "value": 29442406400, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_prefetch_asize", + "value": 87197696, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_read_bytes", + "value": 26255805440, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_rebuild_asize", + "value": 61671941120, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_rebuild_bufs", + "value": 8478512, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_rebuild_bufs_precached", + "value": 1510, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_rebuild_cksum_lb_errors", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_rebuild_dh_errors", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_rebuild_io_errors", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_rebuild_log_blks", + "value": 8296, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_rebuild_lowmem", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_rebuild_size", + "value": 162050568192, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_rebuild_success", + "value": 1, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_rebuild_unsupported", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_rw_clash", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_size", + "value": 159099220992, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_write_bytes", + "value": 2453267968, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_writes_done", + "value": 3537, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_writes_error", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_writes_lock_retry", + "value": 5, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "l2_writes_sent", + "value": 3537, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "meta_hits_per", + "value": 99.662012829336, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "meta_misses_per", + "value": 69.171362280192, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mfu_ghost_hits", + "value": 2064593, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mfu_ghost_per", + "value": 0.18145057987833, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mfu_hits", + "value": 859725964, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mfu_per", + "value": 75.558608744802, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mfu_size", + "value": 353131574, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "min_size_per", + "value": 3.2260897504717, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mru_ghost_hits", + "value": 1168174, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mru_ghost_per", + "value": 0.10266713570122, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mru_hits", + "value": 277307838, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mru_per", + "value": 24.371712976798, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "mutex_skip", + "value": 4047053, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "offline", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "online", + "value": 1, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "p", + "value": 717695178, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_alloc", + "value": 14344697704448, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_cap", + "value": 89, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_dedup", + "value": 1, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_expandsz", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_frag", + "value": 55, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_free", + "value": 1598220898304, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pool_arc_size", + "value": 15942918602752, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_data_hits", + "value": 10073, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_data_misses", + "value": 111701, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_data_total", + "value": 121774, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_hits_per", + "value": 0.00088528426237734, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_meta_hits", + "value": 2819953, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_meta_hits_per", + "value": 0.24783679256863, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_meta_misses", + "value": 13824280, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_meta_misses_per", + "value": 30.475929628747, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "pre_misses_per", + "value": 0.24624731381748, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "real_hits", + "value": 1137033802, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "rec_used_per", + "value": 67.022529709829, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "recycle_miss", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "removed", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "target_size", + "value": 1070826752, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "target_size_arat", + "value": 0.032260897504717, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "target_size_max", + "value": 33192714240, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "target_size_min", + "value": 1070826752, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "target_size_per", + "value": 3.2260897504717, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "unavail", + "value": 0, + "value_prev": null, + "app_type": "zfs" + }, + { + "metric": "unknown", + "value": 0, + "value_prev": null, + "app_type": "zfs" + } + ] + } + } +} diff --git a/tests/snmpsim/linux_zfs-v3.snmprec b/tests/snmpsim/linux_zfs-v3.snmprec new file mode 100644 index 0000000000..f688e40f1d --- /dev/null +++ b/tests/snmpsim/linux_zfs-v3.snmprec @@ -0,0 +1,10 @@ +1.3.6.1.2.1.1.1.0|4|Linux server 3.10.0-693.5.2.el7.x86_64 #1 SMP Fri Oct 20 20:32:50 UTC 2017 x86_64 +1.3.6.1.2.1.1.2.0|6|1.3.6.1.4.1.8072.3.2.10 +1.3.6.1.2.1.1.3.0|67|77550514 +1.3.6.1.2.1.1.4.0|4| +1.3.6.1.2.1.1.5.0|4| +1.3.6.1.2.1.1.6.0|4| +1.3.6.1.2.1.25.1.1.0|67|77552962 +1.3.6.1.4.1.8072.1.3.2.2.1.21.6.100.105.115.116.114.111|2|1 +1.3.6.1.4.1.8072.1.3.2.2.1.21.3.122.102.115|2|1 +1.3.6.1.4.1.8072.1.3.2.3.1.2.3.122.102.115|4x|7b226572726f72223a302c2264617461223a7b226c325f73697a65223a22313539303939323230393932222c226d66755f67686f73745f68697473223a2232303634353933222c226c325f68697473223a223330333337393231222c2264656d616e645f6d6574615f6d6973736573223a223331333737303334222c227072655f646174615f6d6973736573223a22313131373031222c22706f6f6c73223a5b7b226173796e63715f77726974655f70223a2230222c226865616c7468223a2230222c2273796e63715f776169745f72223a2231353433393733222c2273796e63715f726561645f70223a223732313930303937222c226f7065726174696f6e735f77223a2236222c2266726167223a223535222c226465647570223a22312e3030222c22616c74726f6f74223a222d222c2262616e6477696474685f72223a2232222c22746f74616c5f776169745f72223a22313038303232222c226173796e63715f726561645f61223a2230222c227363727562715f726561645f70223a2230222c227472696d5f77616974223a223137353339373732222c2266726565223a2231353938323230383938333034222c226469736b5f776169745f72223a223138323432373233222c2273697a65223a223135393432393138363032373532222c2273796e63715f77726974655f61223a2230222c227472696d715f77726974655f70223a2230222c226173796e63715f726561645f70223a2230222c226f7065726174696f6e735f72223a2231353938323230383938333034222c227363727562715f726561645f61223a2230222c2273637275625f77616974223a223130383135383639222c22657870616e64737a223a2230222c2273796e63715f776169745f77223a2231333134303130222c22636170223a223839222c22746f74616c5f776169745f77223a223131353438313237222c2273796e63715f77726974655f70223a2230222c227472696d715f77726974655f61223a2230222c2262616e6477696474685f77223a223338353831222c2273796e63715f726561645f61223a302c226173796e63715f776169745f77223a223137323831222c226469736b5f776169745f77223a2239383438363738222c226e616d65223a22617263222c226173796e63715f77726974655f61223a2230222c22636b706f696e74223a2230222c22616c6c6f63223a223134333434363937373034343438227d5d2c226c325f636b73756d5f626164223a2230222c2263616368655f6d6973735f706572223a332e38333338323039383138343634352c226c325f696f5f6572726f72223a2230222c226c325f6665656473223a223237333131222c22646174615f7072655f706572223a382e32373138383036393730323839322c2264656d616e645f646174615f68697473223a2231303135363832222c226c325f6d72755f6173697a65223a223239343432343036343030222c226c325f61626f72745f6c6f776d656d223a2232323634393038222c226d696e5f73697a655f706572223a332e323236303839373530343731372c226c325f72656275696c645f627566735f707265636163686564223a2231353130222c226c325f6c6f675f626c6b5f6173697a65223a22313230343837343234222c227072655f646174615f746f74616c223a3132313737342c226c325f72656275696c645f64685f6572726f7273223a2230222c22756e617661696c223a302c226c325f72775f636c617368223a2230222c2265766963745f6c325f736b6970223a2230222c226172635f68697473223a2231313337383236363230222c226c325f6864725f73697a65223a22383038343037353532222c2272656379636c655f6d697373223a302c226c325f726561645f6279746573223a223236323535383035343430222c226d72755f68697473223a22323737333037383338222c226172635f61636365737365735f746f74616c223a313138333138373932372c226172635f73697a655f706572223a332e31353831333436393331313531312c226c325f6c6f675f626c6b5f6176675f6173697a65223a223131323837222c226d66755f706572223a37352e353538363038373434383031362c227461726765745f73697a655f6d696e223a2231303730383236373532222c226f6e6c696e65223a312c226c325f6c6f675f626c6b5f636f756e74223a2238383633222c227072655f6d69737365735f706572223a302e3234363234373331333831373437392c2263616368655f686974735f706572223a39362e313636313739303138313533362c2264656d616e645f6d6574615f68697473223a2231313333393830393132222c226c325f7772697465735f646f6e65223a2233353337222c2270223a22373137363935313738222c2264656d616e645f6d69737365735f706572223a302e3130363436303737373234333437382c226d72755f67686f73745f706572223a302e31303236363731333537303132322c226172635f73697a65223a2231303438323730363234222c22667265715f757365645f706572223a33322e393737343730323930313730372c2264656d616e645f646174615f746f74616c223a313036333937342c2264656d616e645f646174615f6d6973736573223a223438323932222c226c325f72656275696c645f6c6f776d656d223a2230222c226c325f7772697465735f73656e74223a2233353337222c2265766963745f6c325f656c696769626c655f6d7275223a223939393838303431373238222c227461726765745f73697a655f61726174223a302e3033323236303839373530343731372c2265766963745f6c325f696e656c696769626c65223a223934363534333439383234222c226d6574615f6d69737365735f706572223a36392e313731333632323830313931382c226c325f72656275696c645f636b73756d5f6c625f6572726f7273223a2230222c226c325f6163636573735f746f74616c223a34353336313238372c227072655f6d6574615f6d6973736573223a223133383234323830222c22756e6b6e6f776e223a302c226465677261646564223a302c227265616c5f68697473223a313133373033333830322c226c325f646174615f746f5f6d6574615f726174696f223a22333734222c226c325f65766963745f6c31636163686564223a2230222c226d66755f68697473223a22383539373235393634222c22616e6f6e5f686974735f706572223a2d302e3231343433393433373137383932362c226d72755f706572223a32342e333731373132393736373937382c2265766963745f6c325f656c696769626c65223a22313238363534323832323430222c227072655f6d6574615f6d69737365735f706572223a33302e343735393239363238373437322c2265766963745f6c325f636163686564223a22323237393438363738313434222c22646174615f64656d616e645f706572223a39352e3436313136373238343136332c227461726765745f73697a655f706572223a332e323236303839373530343731372c226c325f6d66755f6173697a65223a223239363838383435383234222c226865616c7468223a312c226c325f72656275696c645f6173697a65223a223631363731393431313230222c226c325f627566635f6d657461646174615f6173697a65223a223236303935343432393434222c226c325f72656275696c645f696f5f6572726f7273223a2230222c226c325f667265655f6f6e5f7772697465223a22313132222c227072655f646174615f68697473223a223130303733222c226d72755f67686f73745f68697473223a2231313638313734222c227072655f6d6574615f686974735f706572223a302e3234373833363739323536383632352c2272656d6f766564223a302c226f66666c696e65223a302c226d6574615f686974735f706572223a39392e3636323031323832393333362c226c325f77726974655f6279746573223a2232343533323637393638222c227461726765745f73697a655f6d6178223a223333313932373134323430222c226c325f72656275696c645f6c6f675f626c6b73223a2238323936222c227072655f6d6574615f68697473223a2232383139393533222c226c325f6572726f7273223a302c227461726765745f73697a65223a2231303730383236373532222c226c325f72656275696c645f73697a65223a22313632303530353638313932222c226c325f6c6f675f626c6b5f777269746573223a22353836222c227072655f686974735f706572223a302e3030303838353238343236323337373333382c2265766963745f6c325f656c696769626c655f6d6675223a223238363636323430353132222c226d757465785f736b6970223a2234303437303533222c226c325f70726566657463685f6173697a65223a223837313937363936222c226c325f6d6973736573223a223135303233333636222c226661756c746564223a302c226c325f65766963745f6c6f636b5f7265747279223a2231222c226c325f65766963745f72656164696e67223a2230222c2264656d616e645f686974735f706572223a302e3038393236353039333833333031332c226c325f627566635f646174615f6173697a65223a223333313233303036393736222c22616e6f6e5f68697473223a2d323433393934392c226c325f72656275696c645f62756673223a2238343738353132222c226c325f6173697a65223a223539323138343439393230222c227265635f757365645f706572223a36372e303232353239373039383239332c226d66755f67686f73745f706572223a302e31383134353035373938373833332c226c325f72656275696c645f756e737570706f72746564223a2230222c226c325f7772697465735f6c6f636b5f7265747279223a2235222c2261637475616c5f6869745f706572223a39362e303939313732303830323130312c226d66755f73697a65223a3335333133313537342c226c325f72656275696c645f73756363657373223a2231222c2265766963745f736b6970223a22313333323133313432222c226c325f7772697465735f6572726f72223a2230222c2264656c65746564223a223432333832313438222c226172635f6d6973736573223a223435333631333037227d2c2276657273696f6e223a332c226572726f72537472696e67223a22227d0a