'sysUpTime', 'sysUpTime' => $poll_device['sysUpTime'] ]; // Find MIB-specific SNMP data via OID fetch: sysDescr, sysLocation, sysContact, sysName, sysUpTime $system_metatypes = [ 'sysDescr', 'sysLocation', 'sysContact', 'sysName', 'sysUpTime', 'reboot' ]; // 'snmpEngineID' ]; poll_device_mib_metatypes($device, $system_metatypes, $poll_device); // Store original sysName for devices who store hardware in this Oid, $poll_device['sysName_original'] = $poll_device['sysName']; $poll_device['sysName'] = strtolower($poll_device['sysName']); print_debug_vars($poll_device); // If polled time not set by MIB include, set to unixtime if (!isset($polled)) { $polled = time(); } // Uptime data and reboot triggers // NOTES. http://net-snmp.sourceforge.net/docs/FAQ.html#The_system_uptime__sysUpTime__returned_is_wrong_ // According to it, sysUptime reports time since last snmpd restart, while // hrSystemUptime reports time since last system reboot. // And prefer snmpEngineTime over hrSystemUptime and sysUptime, since they limited with 497 days // 5. As last point used sysUptime (see above) if (isset($agent_data['uptime'])) { list($agent_data['uptime']) = explode(' ', $agent_data['uptime']); $uptimes['unix-agent'] = round((float)$agent_data['uptime']); } elseif (isset($wmi['uptime'])) { $uptimes['wmi'] = round((float)$wmi['uptime']); } if (is_numeric($agent_data['uptime']) && $agent_data['uptime'] > 0) { // 1. Unix-agent uptime is highest priority, since mostly accurate $uptimes['use'] = 'unix-agent'; $uptimes['message'] = 'Using UNIX Agent Uptime'; } elseif (is_numeric($wmi['uptime']) && $wmi['uptime'] > 0) { // 1. WMI uptime is highest priority, since mostly accurate (for Windows) $uptimes['use'] = 'wmi'; $uptimes['message'] = 'Using WMI Uptime'; } elseif (isset($poll_device['device_uptime']) && is_numeric($poll_device['device_uptime']) && $poll_device['device_uptime'] > 0) { // 2. Uptime from os specific OID, see in includes/polling/system MIB specific // Get uptime by some custom way in device os poller, see example in wowza-engine os poller $uptimes['device_uptime'] = round((float)$poll_device['device_uptime']); $uptimes['use'] = 'device_uptime'; $uptimes['message'] = $uptimes['message'] ?: 'Using device MIB poller Uptime'; } else { // 3. Uptime from hrSystemUptime (only in snmp v2c/v3) // NOTE, about windows uptime, // sysUpTime resets when SNMP service restarted, but hrSystemUptime resets at 49.7 days (always), // Now we use LanMgr-Mib-II-MIB::comStatStart.0 as reboot time instead if ($device['os'] !== 'windows' && $device['snmp_version'] !== 'v1' && is_device_mib($device, 'HOST-RESOURCES-MIB')) { // HOST-RESOURCES-MIB::hrSystemUptime.0 = Wrong Type (should be Timeticks): 1632295600 // HOST-RESOURCES-MIB::hrSystemUptime.0 = Timeticks: (63050465) 7 days, 7:08:24.65 $hrSystemUptime = snmp_get_oid($device, 'hrSystemUptime.0', 'HOST-RESOURCES-MIB'); $uptimes['hrSystemUptime'] = timeticks_to_sec($hrSystemUptime); if (is_numeric($uptimes['hrSystemUptime']) && $uptimes['hrSystemUptime'] > 0) { // hrSystemUptime always prefer if not zero $uptimes['use'] = 'hrSystemUptime'; } } // 4. Uptime from snmpEngineTime (only in snmp v2c/v3) if ($device['snmp_version'] !== 'v1' && is_device_mib($device, 'SNMP-FRAMEWORK-MIB')) { $snmpEngineTime = snmp_get_oid($device, 'snmpEngineTime.0', 'SNMP-FRAMEWORK-MIB'); if (is_numeric($snmpEngineTime) && $snmpEngineTime > 0) { if ($device['os'] === 'aos' && strlen($snmpEngineTime) > 8) { // Some Alcatel have bug with snmpEngineTime // See: http://jira.observium.org/browse/OBSERVIUM-763 $snmpEngineTime = 0; } elseif ($device['os'] === 'ironware') { // Check if version correct like "07.4.00fT7f3" $ironware_version = explode('.', $device['version']); if (count($ironware_version) > 2 && $ironware_version[0] > 0 && version_compare($device['version'], '5.1.0') === -1) { // IronWare before Release 05.1.00b have bug (firmware returning snmpEngineTime * 10) // See: http://jira.observium.org/browse/OBSERVIUM-1199 $snmpEngineTime /= 10; } } $uptimes['snmpEngineTime'] = (int)$snmpEngineTime; if ($uptimes['use'] === 'hrSystemUptime') { // Prefer snmpEngineTime only if more than hrSystemUptime if ($uptimes['snmpEngineTime'] > $uptimes['hrSystemUptime']) { $uptimes['use'] = 'snmpEngineTime'; } } elseif ($uptimes['snmpEngineTime'] >= $uptimes['sysUpTime']) { // in other cases prefer if more than sysUpTime $uptimes['use'] = 'snmpEngineTime'; } } } } $uptimes['uptime'] = $uptimes[$uptimes['use']]; // Get actual uptime based on use flag $uptimes['formatted'] = format_uptime($uptimes['uptime']); // Human readable uptime if (!isset($uptimes['message'])) { $uptimes['message'] = 'Using SNMP Agent '.$uptimes['use']; } $uptime = $uptimes['uptime']; print_debug($uptimes['message']." ($uptime sec. => ".$uptimes['formatted'].')'); if (is_numeric($uptime) && $uptime > 0) { // it really is very impossible case for $uptime equals to zero $uptimes['previous'] = $device['uptime']; // Uptime from previous device poll $uptimes['diff'] = $uptimes['previous'] - $uptime; // Difference between previous and current uptimes // Calculate current last rebooted time $last_rebooted = $polled - $uptime; // Previous reboot unixtime $uptimes['last_rebooted'] = $device['last_rebooted']; if (empty($uptimes['last_rebooted']) || // 0 or '' abs($device['last_rebooted'] - $last_rebooted) > 1200) { // Fix when uptime updated by other Oid // Set last_rebooted for all devices who not have it already $uptimes['last_rebooted'] = $last_rebooted; $update_array['last_rebooted'] = $last_rebooted; } // Notify only if current uptime less than one week (eg if changed from sysUpTime to snmpEngineTime) $rebooted = 0; if ($uptime < 604800) { if ($uptimes['diff'] > 60) { // If difference betwen old uptime ($device['uptime']) and new ($uptime) // greater than 60 sec, than device truly rebooted $rebooted = 1; } elseif ($uptimes['previous'] < 300 && $uptimes['diff'] !== 0 && abs($uptimes['diff']) < 280) { // This is rare, boundary case, when device rebooted multiple times between polling runs // diff 0 mean we monitor snmp dump $rebooted = 1; } // Fix reboot flag with some borderline states (roll over counters) if ($rebooted) { switch($uptimes['use']) { case 'hrSystemUptime': case 'sysUpTime': $uptimes_max = array(42949673); // 497 days 2 hours 27 minutes 53 seconds, counter 2^32 (4294967296) divided by 100 break; case 'snmpEngineTime': $uptimes_max = array(2147483647); // Average 68.05 years, counter is 2^32 (4294967296) divided by 2 break; default: // By default uptime limited only by PHP max values // Usually int(2147483647) in 32 bit systems and int(9223372036854775807) in 64 bit systems $uptimes_max = array(PHP_INT_MAX); } if (isset($config['os'][$device['os']]['uptime_max'][$uptimes['use']])) { // Add rollover counter time from definitions $uptimes_max = array_merge($uptimes_max, (array)$config['os'][$device['os']]['uptime_max'][$uptimes['use']]); } // Exclude uptime counter rollover /** * Examples with APC PDU (rollover max sysUpTime is 49 days, 17h 2m 47s): * 1. rebooted uptime previous: 49 days, 16h 52m 18s * less than max: 10m 29s * 2. rebooted uptime previous: 49 days, 16h 54m 50s * less than max: 7m 57s */ foreach ($uptimes_max as $max) { // Exclude 660 sec (11 min) from maximal if ($uptimes['previous'] > ($max - 660) && $uptimes['previous'] <= $max) { $uptimes['max'] = $max; $rebooted = 0; break; } } } if ($rebooted) { $update_array['last_rebooted'] = $polled - $uptime; // Update last reboot unixtime //$reboot_diff = $update_array['last_rebooted'] - $uptimes['last_rebooted']; // Calculate time between 2 reboots log_event('Device rebooted: after '.format_uptime($polled - $uptimes['last_rebooted']) . ' (Uptime: '.$uptimes['formatted'].', previous: '.format_uptime($uptimes['previous']).', used: '.$uptimes['use'] . ')', $device, 'device', $device['device_id'], 4); $uptimes['last_rebooted'] = $update_array['last_rebooted']; // store new reboot unixtime } } $uptimes['rebooted'] = $rebooted; rrdtool_update_ng($device, 'uptime', array('uptime' => $uptime)); $graphs['uptime'] = TRUE; print_cli_data('Uptime', $uptimes['formatted']); print_cli_data('Last reboot', format_unixtime($uptimes['last_rebooted'])); $update_array['uptime'] = $uptime; $cache['devices']['uptime'][$device['device_id']]['uptime'] = $uptime; $cache['devices']['uptime'][$device['device_id']]['sysUpTime'] = $uptimes['sysUpTime']; // Required for ports (ifLastChange) $cache['devices']['uptime'][$device['device_id']]['polled'] = $polled; } else { print_warning('Device does not have any uptime counter or uptime equals zero.'); } print_debug_vars($uptimes, 1); // Load Average foreach (get_device_mibs_permitted($device) as $mib) // Check every MIB supported by the device { if (isset($config['mibs'][$mib]['la'])) { $la_def = $config['mibs'][$mib]['la']; // See Cisco CISCO-PROCESS-MIB if (discovery_check_requires($device, $la_def, $device, 'device')) { continue; } $la_oids = array(); if (isset($la_def['type']) && $la_def['type'] === 'table') { // First element from table walk, currently only for Cisco IOS-XE $la_oids = ['1min' => $la_def['oid_1min'], '5min' => $la_def['oid_5min'], '15min' => $la_def['oid_15min']]; $la = snmpwalk_cache_oid($device, $la_def['oid_1min'], array(), $mib); $la = snmpwalk_cache_oid($device, $la_def['oid_5min'], $la, $mib); $la = snmpwalk_cache_oid($device, $la_def['oid_15min'], $la, $mib); $la = array_shift($la); } else { // Single oid foreach (array('1min', '5min', '15min') as $min) { if (isset($la_def['oid_'.$min.'_num'])) { $la_oids[$min] = $la_def['oid_'.$min.'_num']; } elseif (isset($la_def['oid_'.$min])) { $la_oids[$min] = snmp_translate($la_def['oid_'.$min], $mib); } } $la = snmp_get_multi_oid($device, $la_oids, array(), $mib, NULL, OBS_SNMP_ALL_NUMERIC); } print_debug_vars($la_oids); print_debug_vars($la); if (snmp_status() && is_numeric($la[$la_oids['5min']])) { $scale = isset($la_def['scale']) ? $la_def['scale'] : 1; $scale_graph = $scale * 100; // Since want to keep compatibility with old UCD-SNMP-MIB LA, graph stored as la * 100 foreach ($la_oids as $min => $oid) { $device_state['la'][$min] = $la[$oid] * $scale; // Now, graph specific scale if not equals 1 $la[$min] = $la[$oid] * $scale_graph; } $device_state['ucd_load'] = $device_state['la']['5min']; // Compatibility with old UCD-SNMP-MIB code rrdtool_update_ng($device, 'la', array('1min' => $la['1min'], '5min' => $la['5min'], '15min' => $la['15min'])); $graphs['la'] = TRUE; print_cli_data('Load average', $device_state['la']['1min'].', '.$device_state['la']['5min'].', '.$device_state['la']['15min']); break; // Stop walking other LAs } } } // END LA // Check if many oids empty (seems as device down after check) $poll_empty_count = 0; // Rewrite sysLocation if there is a mapping array or DB override //$poll_device['sysLocation'] = snmp_fix_string($poll_device['sysLocation']); $poll_device['sysLocation'] = rewrite_location($poll_device['sysLocation']); if ($device['location'] != $poll_device['sysLocation']) { $update_array['location'] = $poll_device['sysLocation']; log_event("sysLocation changed: '".$device['location']."' -> '".$poll_device['sysLocation']."'", $device, 'device', $device['device_id']); if (strlen($poll_device['sysLocation']) === 0) { $poll_empty_count++; } else { $poll_empty_count--; } } $poll_device['sysContact'] = str_replace(array('\"', '"') , '', $poll_device['sysContact']); if (!is_valid_param($poll_device['sysContact'], 'sysContact')) { // Common wrong contacts: Uninitialized, not set, , (none), SNMPv2, Unknown, ?, $poll_device['sysContact'] = ''; } print_debug_vars($poll_device); // Check if snmpEngineID changed $force_discovery = FALSE; if ($poll_device['snmpEngineID'] != $device['snmpEngineID']) { $update_array['snmpEngineID'] = (string) $poll_device['snmpEngineID']; if (!safe_empty($device['snmpEngineID']) && !safe_empty($poll_device['snmpEngineID'])) { // snmpEngineID changed, force full device rediscovery log_event('snmpEngineID changed: '.$device['snmpEngineID'].' -> '.$poll_device['snmpEngineID'].' (probably the device was replaced). The device will be rediscovered.', $device, 'device', $device['device_id'], 4); $force_discovery = TRUE; force_discovery($device); } else { log_event('snmpEngineID -> '.$poll_device['snmpEngineID'], $device, 'device', $device['device_id']); } if (safe_empty($poll_device['snmpEngineID'])) { $poll_empty_count++; } else { $poll_empty_count--; } } $oids = [ 'sysObjectID', 'sysContact', 'sysName', 'sysDescr' ]; foreach ($oids as $oid) { $poll_device[$oid] = snmp_fix_string($poll_device[$oid]); //print_vars($poll_device[$oid]); if ($poll_device[$oid] != $device[$oid]) { $update_array[$oid] = $poll_device[$oid] ?: [ 'NULL' ]; log_event("$oid -> '".$poll_device[$oid]."'", $device, 'device', $device['device_id']); if (safe_empty($poll_device[$oid])) { $poll_empty_count++; } else { $poll_empty_count--; } // Update $device array for cases when model specific options required if ($oid === 'sysObjectID' && $poll_device['sysObjectID']) { $device['sysObjectID'] = $poll_device['sysObjectID']; } } } // Re-check device status if ($poll_empty_count >= 2) { $device_status = device_status_array($device); if ($device['status'] != $device_status['status'] && $device_status['status'] == 0) { $status = $device_status['status']; $status_type = $device_status['status_type']; $device['status'] = $device_status['status']; $device['status_type'] = $device_status['status_type']; // device going down by some reasons, prevent other polling // reset update array $update_array = []; return; } } // Check if both (sysObjectID and sysDescr) changed, force full device rediscovery if (!$force_discovery && // Check if not already forced isset($update_array['sysObjectID'], $update_array['sysDescr']) && strlen($poll_device['sysObjectID']) && strlen($device['sysObjectID']) && strlen($poll_device['sysDescr']) && strlen($device['sysDescr'])) { log_event('sysObjectID and sysDescr changed: seems the device was replaced. The device will be rediscovered.', $device, 'device', $device['device_id'], 4); force_discovery($device); } if (!empty($device['snmpable'])) { print_cli_data('SNMPable OID', $device['snmpable'], 2); } print_cli_data('sysObjectID', $poll_device['sysObjectID'], 2); print_cli_data('snmpEngineID', $poll_device['snmpEngineID'], 2); print_cli_data('sysDescr', $poll_device['sysDescr'], 2); print_cli_data('sysName', $poll_device['sysName_original'], 2); print_cli_data('Location', $poll_device['sysLocation'], 2); // Restore original (not lower case) sysName if (isset($poll_device['sysName_SNMPv2'])) { $poll_device['sysName'] = $poll_device['sysName_SNMPv2']; unset($poll_device['sysName_SNMPv2']); } else { $poll_device['sysName'] = $poll_device['sysName_original']; } unset($poll_device['sysName_original']); // Geolocation detect $geo_detect = FALSE; if ($config['geocoding']['enable']) { $db_version = get_db_version(); // Need for detect old geo DB schema $geo_db = dbFetchRow('SELECT * FROM `devices_locations` WHERE `device_id` = ?', array($device['device_id'])); $geo_db['hostname'] = $device['hostname']; // Hostname required for detect by DNS print_debug_vars($geo_db); $geo_updated = $config['time']['now'] - strtotime($geo_db['location_updated']); // Seconds since previous GEO update $geo_frequency = 86400; // Minimum seconds for next GEO api request (default is 1 day) // Device coordinates still empty, re-detect no more than 1 time per 1 day ($geo_frequency param) if (!(is_numeric($geo_db['location_lat']) && is_numeric($geo_db['location_lon']))) { // Re-detect geolocation if coordinates still empty, no more frequently than once a day $geo_detect = $geo_detect || ($geo_updated > $geo_frequency); } // sysLocation changed (and not empty!), redetect now $geo_detect = $geo_detect || ($poll_device['sysLocation'] && $device['location'] != $poll_device['sysLocation']); // Geo API changed, force re-detect $geo_detect = $geo_detect || ($geo_db['location_geoapi'] != strtolower($config['geocoding']['api'])); // This seems to cause endless geolocation every poll. Disabled. //$geo_detect = $geo_detect || ($geo_db['location_manual'] && (!$geo_db['location_country'] || $geo_db['location_country'] == 'Unknown')); // Manual coordinates passed // Detect location by DNS LOC record for hostname, no more than 1 time per 1 day ($geo_frequency param) $dns_only = !$geo_detect && ($config['geocoding']['dns'] && ($geo_updated > $geo_frequency)); $geo_detect = $geo_detect || $dns_only; if ($geo_detect) { $update_geo = get_geolocation($poll_device['sysLocation'], $geo_db, $dns_only); if ($update_geo) { print_debug_vars($update_geo, 1); if (is_numeric($update_geo['location_lat']) && is_numeric($update_geo['location_lon']) && $update_geo['location_country'] != 'Unknown') { $geo_msg = 'Geolocation ('.strtoupper($update_geo['location_geoapi']).') -> '; $geo_msg .= '['.sprintf('%f', $update_geo['location_lat']) .', ' .sprintf('%f', $update_geo['location_lon']) .'] '; $geo_msg .= $update_geo['location_country'].' (Country), '.$update_geo['location_state'].' (State), '; $geo_msg .= $update_geo['location_county'] .' (County), ' .$update_geo['location_city'] .' (City)'; } else { $geo_msg = FALSE; } if (is_numeric($geo_db['location_id'])) { foreach ($update_geo as $k => $value) { if ($geo_db[$k] == $value) { unset($update_geo[$k]); } } if ($update_geo) { dbUpdate($update_geo, 'devices_locations', '`location_id` = ?', array($geo_db['location_id'])); if ($geo_msg) { log_event($geo_msg, $device, 'device', $device['device_id']); } } // else not changed } else { $update_geo['device_id'] = $device['device_id']; dbInsert($update_geo, 'devices_locations'); if ($geo_msg) { log_event($geo_msg, $device, 'device', $device['device_id']); } } } elseif (is_numeric($geo_db['location_id'])) { $update_geo = array('location_updated' => format_unixtime($config['time']['now'], 'Y-m-d G:i:s')); // Increase updated time dbUpdate($update_geo, 'devices_locations', '`location_id` = ?', array($geo_db['location_id'])); } # end if $update_geo } } unset($geo_detect, $geo_db, $update_geo); // EOF