initial commit; version 22.5.12042

includes/housekeeping/alertlog.inc.php

@@ -0,0 +1,58 @@
+<?php
+
+/**
+ * Observium
+ *
+ *   This file is part of Observium.
+ *
+ * @package    observium
+ * @subpackage housekeeping
+ * @author     Adam Armstrong <adama@observium.org>
+ * @copyright  (C) 2006-2013 Adam Armstrong, (C) 2013-2019 Observium Limited
+ *
+ */
+
+// Minimum allowed age for delete alert_log entries times is 24h
+$cutoff = age_to_unixtime($config['housekeeping']['alertlog']['age'], age_to_seconds('24h'));
+
+if ($cutoff)
+{
+  $where = "`timestamp` < FROM_UNIXTIME($cutoff)";
+  $count = dbFetchCell("SELECT COUNT(*) FROM `alert_log` WHERE $where");
+  if ($count)
+  {
+    if ($prompt)
+    {
+      $answer = print_prompt("$count alert log entries older than " . format_unixtime($cutoff) . " will be deleted");
+    }
+
+    if ($answer)
+    {
+      $rows = dbDelete('alert_log', "$where");
+      if ($rows === FALSE)
+      {
+        // Use LIMIT with big tables
+        print_debug("Alert log table is too big, using LIMIT for delete entries");
+        $rows = 0;
+        $i    = 1000;
+        while ($i && $rows < $count)
+        {
+          $iter = dbDelete('alert_log', $where.' LIMIT 1000000');
+          if ($iter === FALSE) { break; }
+          $rows += $iter;
+          $i--;
+        }
+      }
+      print_debug("Alert log housekeeping: deleted $rows entries");
+      logfile("housekeeping.log", "Alert log: deleted $rows entries older than " . format_unixtime($cutoff));
+    }
+  }
+  else if ($prompt)
+  {
+    print_message("No alert log entries found older than " . format_unixtime($cutoff));
+  }
+} else {
+  print_message("Alert log housekeeping disabled in configuration or incorrectly configured to less than 24h.");
+}
+
+// EOF

includes/housekeeping/authlog.inc.php

@@ -0,0 +1,58 @@
+<?php
+
+/**
+ * Observium
+ *
+ *   This file is part of Observium.
+ *
+ * @package    observium
+ * @subpackage housekeeping
+ * @author     Adam Armstrong <adama@observium.org>
+ * @copyright  (C) 2006-2013 Adam Armstrong, (C) 2013-2019 Observium Limited
+ *
+ */
+
+// Minimum allowed age for delete authlog entries times is 24h
+$cutoff = age_to_unixtime($config['housekeeping']['authlog']['age'], age_to_seconds('24h'));
+
+if ($cutoff)
+{
+  $where = "`datetime` < FROM_UNIXTIME($cutoff)";
+  $count = dbFetchCell("SELECT COUNT(*) FROM `authlog` WHERE $where");
+  if ($count)
+  {
+    if ($prompt)
+    {
+      $answer = print_prompt("$count authlog entries older than " . format_unixtime($cutoff) . " will be deleted");
+    }
+
+    if ($answer)
+    {
+      $rows = dbDelete('authlog', "$where");
+      if ($rows === FALSE)
+      {
+        // Use LIMIT with big tables
+        print_debug("Authlog table is too big, using LIMIT for delete entries");
+        $rows = 0;
+        $i    = 1000;
+        while ($i && $rows < $count)
+        {
+          $iter = dbDelete('authlog', $where.' LIMIT 1000000');
+          if ($iter === FALSE) { break; }
+          $rows += $iter;
+          $i--;
+        }
+      }
+      print_debug("Authlog housekeeping: deleted $rows entries");
+      logfile("housekeeping.log", "Authlog: deleted $rows entries older than " . format_unixtime($cutoff));
+    }
+  }
+  else if ($prompt)
+  {
+    print_message("No authlog entries found older than " . format_unixtime($cutoff));
+  }
+} else {
+  print_message("Authlog housekeeping disabled in configuration or incorrectly configured to less than 24h.");
+}
+
+// EOF

includes/housekeeping/eventlog.inc.php

@@ -0,0 +1,58 @@
+<?php
+
+/**
+ * Observium
+ *
+ *   This file is part of Observium.
+ *
+ * @package    observium
+ * @subpackage housekeeping
+ * @author     Adam Armstrong <adama@observium.org>
+ * @copyright  (C) 2006-2013 Adam Armstrong, (C) 2013-2019 Observium Limited
+ *
+ */
+
+// Minimum allowed age for delete syslog entries times is 24h
+$cutoff = age_to_unixtime($config['housekeeping']['eventlog']['age'], age_to_seconds('24h'));
+
+if ($cutoff)
+{
+  $where = "`timestamp` < FROM_UNIXTIME($cutoff)";
+  $count = dbFetchCell("SELECT COUNT(*) FROM `eventlog` WHERE $where");
+  if ($count)
+  {
+    if ($prompt)
+    {
+      $answer = print_prompt("$count eventlog entries older than " . format_unixtime($cutoff) . " will be deleted");
+    }
+
+    if ($answer)
+    {
+      $rows = dbDelete('eventlog', $where);
+      if ($rows === FALSE)
+      {
+        // Use LIMIT with big tables
+        print_debug("Event log table is too big, using LIMIT to delete entries");
+        $rows = 0;
+        $i    = 1000;
+        while ($i && $rows < $count)
+        {
+          $iter = dbDelete('eventlog', $where.' LIMIT 1000000');
+          if ($iter === FALSE) { break; }
+          $rows += $iter;
+          $i--;
+        }
+      }
+      print_debug("Event log housekeeping: deleted $rows entries");
+      logfile("housekeeping.log", "Event log: deleted $rows entries older than " . format_unixtime($cutoff));
+    }
+  }
+  else if ($prompt)
+  {
+    print_message("No event log entries found older than " . format_unixtime($cutoff));
+  }
+} else {
+  print_message("Event log housekeeping is disabled in configuration or incorrectly configured to less than 24h.");
+}
+
+// EOF

includes/housekeeping/inventory.inc.php

@@ -0,0 +1,58 @@
+<?php
+
+/**
+ * Observium
+ *
+ *   This file is part of Observium.
+ *
+ * @package    observium
+ * @subpackage housekeeping
+ * @author     Adam Armstrong <adama@observium.org>
+ * @copyright  (C) 2006-2013 Adam Armstrong, (C) 2013-2019 Observium Limited
+ *
+ */
+
+// Minimum allowed age for deleted inventory entries times is 24h
+$cutoff = age_to_unixtime($config['housekeeping']['inventory']['age'], age_to_seconds('24h'));
+
+if ($cutoff)
+{
+  $where = "`deleted` < FROM_UNIXTIME($cutoff)";
+  $count = dbFetchCell("SELECT COUNT(*) FROM `entPhysical` WHERE $where");
+  if ($count)
+  {
+    if ($prompt)
+    {
+      $answer = print_prompt("$count deleted inventory entries older than " . format_unixtime($cutoff) . " will be deleted");
+    }
+
+    if ($answer)
+    {
+      $rows = dbDelete('entPhysical', "$where");
+      if ($rows === FALSE)
+      {
+        // Use LIMIT with big tables
+        print_debug("entPhysical table is too big, using LIMIT for delete entries");
+        $rows = 0;
+        $i    = 1000;
+        while ($i && $rows < $count)
+        {
+          $iter = dbDelete('entPhysical', $where.' LIMIT 1000000');
+          if ($iter === FALSE) { break; }
+          $rows += $iter;
+          $i--;
+        }
+      }
+      print_debug("Inventory housekeeping: deleted $rows entries");
+      logfile("housekeeping.log", "Inventory: deleted $rows entries older than " . format_unixtime($cutoff));
+    }
+  }
+  else if ($prompt)
+  {
+    print_message("No inventory entries found older than " . format_unixtime($cutoff));
+  }
+} else {
+  print_message("Inventory housekeeping disabled in configuration or incorrectly configured to less than 24h.");
+}
+
+// EOF

includes/housekeeping/ports.inc.php

@@ -0,0 +1,43 @@
+<?php
+/**
+ * Observium
+ *
+ *   This file is part of Observium.
+ *
+ * @package    observium
+ * @subpackage housekeeping
+ * @copyright  (C) 2006-2013 Adam Armstrong, (C) 2013-2021 Observium Limited
+ *
+ */
+
+$cutoff = age_to_unixtime($config['housekeeping']['deleted_ports']['age']);
+
+if ($cutoff) {
+  $where = "`deleted` = 1 AND `ifLastChange` < FROM_UNIXTIME($cutoff)";
+
+  // Prevent delete billed ports
+  if ($bill_ports = dbFetchColumn('SELECT `entity_id` FROM `bill_entities` WHERE `entity_type` = ?', [ 'port' ])) {
+    $where .= generate_query_values($bill_ports, 'port_id', '!=');
+  }
+
+  $ports  = dbFetchRows("SELECT `port_id` FROM `ports` WHERE $where");
+  $count  = count($ports);
+  if ($count) {
+    if ($prompt) {
+      $answer = print_prompt("$count ports marked as deleted before " . format_unixtime($cutoff) . " will be deleted");
+    }
+    if ($answer) {
+      foreach ($ports as $entry) {
+        delete_port($entry['port_id']);
+      }
+      print_debug("Deleted ports housekeeping: deleted $count entries");
+      logfile("housekeeping.log", "Deleted ports: deleted $count entries older than " . format_unixtime($cutoff));
+    }
+  } elseif ($prompt) {
+    print_message("No ports found marked as deleted before " . format_unixtime($cutoff));
+  }
+} else {
+  print_message("Deleted ports housekeeping disabled in configuration.");
+}
+
+// EOF

includes/housekeeping/rrd.inc.php

@@ -0,0 +1,203 @@
+<?php
+/**
+ * Observium
+ *
+ *   This file is part of Observium.
+ *
+ * @package    observium
+ * @subpackage housekeeping
+ * @copyright  (C) 2006-2013 Adam Armstrong, (C) 2013-2021 Observium Limited
+ *
+ */
+
+// Minimum allowed age to delete RRDs is 24h
+$cutoff = age_to_unixtime($config['housekeeping']['rrd']['age'], age_to_seconds('24h'));
+
+// Validate rrdtool version before check invalid
+if ($config['housekeeping']['rrd']['invalid']) {
+  list(,$rrdtool_version) = explode(' ', external_exec($GLOBALS['config']['rrdtool'] . ' --version | head -n1'));
+  if (!$rrdtool_version || !preg_match('/^\d+(\.\d+)+/', $rrdtool_version)) {
+    // Not valid rrdtool found, disable check invalid files
+    $config['housekeeping']['rrd']['invalid'] = FALSE;
+  }
+}
+
+if ($cutoff ||
+    $config['housekeeping']['rrd']['invalid'] ||
+    $config['housekeeping']['rrd']['deleted'] ||
+    $config['housekeeping']['rrd']['disabled']) {
+  if ($prompt) {
+    $msg = "RRD files:" . PHP_EOL;
+    if ($config['housekeeping']['rrd']['invalid']) {
+      $msg .= " - not valid RRD" . PHP_EOL;
+    }
+    if ($config['housekeeping']['rrd']['deleted']) {
+      $msg .= " - RRD dirs for deleted hosts" . PHP_EOL;
+    }
+    if ($config['housekeeping']['rrd']['disabled'] && $cutoff) {
+      $msg .= " - RRD dirs for disabled devices since " . format_unixtime($cutoff) . PHP_EOL;
+    }
+    if ($config['housekeeping']['rrd']['notmodified'] && $cutoff) {
+      $msg .= " - not modified since " . format_unixtime($cutoff) . PHP_EOL;
+    }
+    $answer = print_prompt($msg . "will be deleted");
+  }
+} else {
+  print_message("RRD housekeeping disabled in configuration or less than 24h.");
+  $answer = FALSE;
+}
+
+if ($answer) {
+  $rrd_stat = [ 'notmodified' => 0, 'invalid' => 0, 'deleted' => 0, 'disabled' => 0 ];
+
+  $table_rows = [];
+  $rrd_dir_len = strlen($config['rrd_dir']);
+  foreach (glob($config['rrd_dir'] . '/*', GLOB_ONLYDIR) as $rrd_dir) {
+
+    $rrd_host = ltrim(substr($rrd_dir, $rrd_dir_len), '/');
+    $rrd_host_valid = is_valid_hostname($rrd_host) || get_ip_version($rrd_host);
+
+    $rrd_files = [];
+    $file_count = 0;
+    foreach (get_recursive_directory_iterator($rrd_dir) as $file => $info) {
+      if ($info->getExtension() === 'rrd') {
+        print_debug("Found file ending in '.rrd': " . $file);
+        $rrd_files[] = $file;
+      }
+      $file_count++;
+    }
+    $rrd_count = count($rrd_files);
+
+    if (!$rrd_count && !$rrd_host_valid) {
+      print_debug("Incorrect hostname detected [$rrd_host] in directory [$rrd_dir]. Seems as not RRD dir.");
+      $table_rows['incorrect'][] = [ $rrd_host, $rrd_dir, 0, '%yIncorrect host%n' ];
+      continue;
+    }
+
+    if (!$rrd_count && $file_count) {
+      print_debug("Not empty directory [$rrd_dir] not have RRD files. Skipped..");
+      $table_rows['empty'][] = [ $rrd_host, $rrd_dir, $rrd_count, '%yNot empty dir.%n' ];
+
+      // Skip other checks in dir
+      continue;
+    }
+
+    if ($rrd_host_valid) {
+      $rrd_device = device_by_name($rrd_host);
+
+      if (!$rrd_device) {
+        print_debug("Found stale RRD directory [$rrd_dir] for nonexistent host [$rrd_host]");
+        if ($config['housekeeping']['rrd']['deleted']) {
+          $table_rows['deleted'][] = [ $rrd_host, $rrd_dir, $rrd_count, '%rDeleted host%n' ];
+          $rrd_stat['deleted']++;
+          if (!$test) {
+            if (delete_dir($rrd_dir)) {
+              logfile("housekeeping.log", "Stale RRD directory [$rrd_dir] for nonexistent host [$rrd_host] - deleted.");
+            } else {
+              logfile("housekeeping.log", "Stale RRD directory [$rrd_dir] for nonexistent host [$rrd_host] - NOT DELETED, because no access to dir.");
+              print_debug("Stale RRD directory [$rrd_dir] for nonexistent host [$rrd_host] - NOT DELETED, because no access to dir.");
+            }
+          }
+        } else {
+          print_warning("Found stale RRD directory [$rrd_dir] for nonexistent host [$rrd_host]. To remove it, set \$config['housekeeping']['rrd']['deleted'] = TRUE;");
+          //log_event("Housekeeping: Found stale RRD directories for nonexistent devices. To remove, it set \$config['housekeeping']['rrd']['deleted'] = TRUE;", $rrd_device, 'device', $rrd_device, 7);
+        }
+        // Skip other checks in dir
+        continue;
+      }
+
+      if ($rrd_device['disabled']) {
+        $last_polled = $rrd_device['last_polled'] ? strtotime($rrd_device['last_polled']) : 0;
+        if ($config['housekeeping']['rrd']['disabled'] && $last_polled < $cutoff) {
+          //print_cli($rrd_device['last_polled'] . ": $last_polled < $cutoff\n");
+          print_debug("Found old RRD directory [$rrd_dir] for disabled device [$rrd_host] - deleting");
+          $table_rows['disabled'][] = [ $rrd_host, $rrd_dir, $rrd_count, '%mDisabled device%n (Last polled: ' . format_uptime(get_time() - $last_polled, 'short-2') . ' ago)' ];
+          $rrd_stat['disabled']++;
+          if (!$test) {
+            if (delete_dir($rrd_dir)) {
+              logfile("housekeeping.log", "Old RRD directory [$rrd_dir] for disabled device [$rrd_host] - deleted.");
+              //log_event("Housekeeping: Removed RRD directory [$rrd_dir] for disabled device older than ".format_uptime(get_time() - $last_polled, 'short-2'), $rrd_device, 'device', $rrd_device, 7);
+            } else {
+              logfile("housekeeping.log", "Old RRD directory [$rrd_dir] for disabled device [$rrd_host] - NOT DELETED, because no access to dir.");
+              print_debug("Old RRD directory [$rrd_dir] for disabled device [$rrd_host] - NOT DELETED, because no access to dir.");
+            }
+          }
+        }
+        // Skip other checks in dir
+        continue;
+      }
+    }
+
+    // Now checks for every rrd
+    foreach ($rrd_files as $file) {
+      if ($cutoff && $config['housekeeping']['rrd']['notmodified']) {
+        $file_data = stat($file);
+
+        if ($file_data['mtime'] < $cutoff) {
+          print_debug("File modification time is " . format_unixtime($file_data['mtime']) . " - deleting");
+          $table_rows['notmodified'][] = [ $rrd_host, $file, '%yRRD File not modified%n (Last modified: ' . format_uptime(get_time() - $file_data['mtime'], 'short-2') . ' ago)' ];
+          if (!$test) {
+            if (unlink($file)) {
+              logfile("housekeeping.log", "RRD File $file modification time is " . format_unixtime($file_data['mtime']) . " - deleted.");
+              //log_event("Housekeeping: Removed RRD file [$file] not modified since ".format_unixtime($file_data['mtime']), $rrd_device, 'device', $rrd_device, 7);
+            } else {
+              logfile("housekeeping.log", "RRD File $file modification time is " . format_unixtime($file_data['mtime']) . " - NOT DELETED, because no access to file.");
+              print_debug("RRD File $file modification time is " . format_unixtime($file_data['mtime']) . " - NOT DELETED, because no access to file.");
+            }
+          }
+          $rrd_stat['notmodified']++;
+
+          // Skip other check for invalid
+          continue;
+        }
+      }
+
+      if ($config['housekeeping']['rrd']['invalid'] && !rrdtool_file_valid($file)) {
+        print_debug("Invalid RRD file $file - deleting");
+        $table_rows['invalid'][] = [ $rrd_host, $file, '%rRRD File invalid%n' ];
+        if (!$test) {
+          if (unlink($file)) {
+            logfile("housekeeping.log", "File $file is not valid RRD - deleted.");
+            //log_event("Housekeeping: Removed file [$file] not valid RRD", $rrd_device, 'device', $rrd_device, 7);
+          } else {
+            logfile("housekeeping.log", "File $file is not valid RRD - NOT DELETED, because no access to file.");
+            print_debug("File $file is not valid RRD - NOT DELETED, because no access to file.");
+          }
+        }
+        $rrd_stat['invalid']++;
+
+        return;
+      }
+    }
+  }
+
+  if ($prompt) {
+    foreach ([ 'empty', 'incorrect', 'deleted', 'disabled', 'notmodified', 'invalid' ] as $t) {
+      if (isset($table_rows[$t])) {
+        $all = $t === 'notmodified' || $t === 'invalid' ? 'files' : 'all';
+        $table_rows[$all] = array_merge((array)$table_rows[$all], $table_rows[$t]);
+        unset($table_rows[$t]);
+      }
+    }
+    print_cli_table($table_rows['all'], [ 'Host', 'RRD dir', 'RRD count', 'Reason' ]);
+    print_cli_table($table_rows['files'], [ 'Host', 'RRD file', 'Reason' ]);
+  }
+  unset($table_rows);
+
+  if ($prompt && $cutoff && $config['housekeeping']['rrd']['notmodified']) {
+    if ($rrd_stat['notmodified']) {
+      print_message("Deleted ".$rrd_stat['notmodified']." not modified RRD files older than " . format_unixtime($cutoff));
+    } else {
+      print_message("No RRD files found last modified before " . format_unixtime($cutoff));
+    }
+  }
+  if ($prompt && $config['housekeeping']['rrd']['invalid']) {
+    if ($rrd_stat['invalid']) {
+      print_message("Deleted ".$rrd_stat['invalid']." invalid RRD files");
+    } else {
+      print_message("No invalid RRD files found");
+    }
+  }
+}
+
+// EOF

includes/housekeeping/staledb.inc.php

@@ -0,0 +1,154 @@
+<?php
+/**
+ * Observium
+ *
+ *   This file is part of Observium.
+ *
+ * @package    observium
+ * @subpackage housekeeping
+ * @copyright  (C) 2006-2013 Adam Armstrong, (C) 2013-2020 Observium Limited
+ *
+ */
+
+// Store highest device ID so we don't delete data from devices that were added during our run.
+// Counting on mysql auto_increment to never give out a lower ID, obviously.
+//$max_id = -1;
+$max_id = dbShowNextID('devices');
+
+// Fetch all existing device IDs and update the maximum ID.
+$devices = [];
+foreach (dbFetchRows("SELECT `device_id` FROM `devices` ORDER BY `device_id` ASC") as $device) {
+  $devices[] = $device['device_id'];
+
+  // Not sure in which cases this can happen, but I keep this for compatibility
+  if ($device['device_id'] > $max_id) {
+    $max_id = $device['device_id'];
+  }
+}
+
+
+
+// Cleanup tables with links to devices that on longer exist
+foreach ($config['device_tables'] as $table) {
+  $where = '`device_id` NOT IN (' . implode(',', $devices) . ') AND `device_id` < ?';
+  //$where = '`device_id` NOT IN (' . implode(',', $devices) . ') OR `device_id` > ?';
+  if ($table === 'observium_processes') {
+    $where .= ' AND `device_id` != 0';
+  } elseif ($table === 'eventlog') {
+    // Global events with device_id = 0
+    $where .= " AND `entity_type` NOT IN ('global', 'info')";
+  }
+
+  $rows  = dbFetchRows("SELECT 1 FROM `$table` WHERE $where", [ $max_id ], $test);
+  $count = safe_count($rows);
+  if ($count) {
+    if ($prompt) {
+      $answer = print_prompt("$count rows in table '$table' for non-existing devices will be deleted");
+    }
+    if ($answer) {
+      // IP addresses need additionally check if associated network is empty
+      if ($table === 'ipv4_addresses') {
+        $network_table = 'ipv4_networks';
+        $network_where = '`ipv4_network_id` = ?';
+        $network_ids = dbFetchColumn("SELECT DISTINCT `ipv4_network_id` FROM `$table` WHERE $where", array($max_id));
+        print_debug_vars($network_ids);
+      } elseif ($table === 'ipv6_addresses') {
+        $network_table = 'ipv6_networks';
+        $network_where = '`ipv6_network_id` = ?';
+        $network_ids = dbFetchColumn("SELECT DISTINCT `ipv6_network_id` FROM `$table` WHERE $where", array($max_id));
+        print_debug_vars($network_ids);
+      }
+
+      // Remove stale entries
+      print_debug("Database cleanup for table '$table': deleted $count entries");
+      if (!$test) {
+        $table_status = dbDelete($table, $where, [ $max_id ]);
+        logfile("housekeeping.log", "Database cleanup for table '$table': deleted $count entries");
+      }
+
+      // Clean network ids after delete ip addresses if network unused
+      if ($table_status &&
+          ($table === 'ipv4_addresses' || $table === 'ipv6_addresses')) {
+        foreach ($network_ids as $network_id) {
+          if (!dbExist($table, $network_where, [ $network_id ], $test) && !$test) {
+            $network_status = dbDelete($network_table, $network_where, array($network_id));
+          }
+        }
+      }
+    }
+  } elseif ($prompt) {
+    print_message("No orphaned rows found in table '$table'.");
+  }
+}
+
+// Cleanup autodiscovery entries
+if (dbExist('autodiscovery', '`remote_device_id` IS NOT NULL AND `remote_device_id` NOT IN (SELECT `device_id` FROM `devices`)', NULL, $test)) {
+  if (!$test) {
+    $table_status = dbDelete('autodiscovery', '`remote_device_id` IS NOT NULL AND `remote_device_id` NOT IN (SELECT `device_id` FROM `devices`)');
+    logfile("housekeeping.log", "Deleted stale autodiscovery entries.");
+  }
+  print_debug("Deleted stale autodiscovery entries.");
+}
+
+// Cleanup common entity tables with links to devices that no longer exist
+// Loop for found stale entity entries
+/*
+ * Probably not the best idea to remove user-generated configuration like this, this very occasionally seems to delete billing ports
+ *
+$where = " WHERE " . generate_query_values([ 'bill', 'group' ], 'entity_type', '!=', FALSE); // Exclude pseudo-entities from permissions checks
+foreach ($config['entity_tables'] as $table) {
+  $query = "SELECT DISTINCT `entity_type` FROM `$table` $where";
+  foreach (dbFetchColumn($query, NULL, $test) as $entity_type) {
+    $translate = entity_type_translate_array($entity_type);
+    if (safe_empty($translate['device_id_field'])) { continue; }
+    $id_field = $translate['id_field'];
+    $query = 'SELECT `entity_id` FROM `'.$table.'` WHERE `entity_type` = ? AND `entity_id` NOT IN (SELECT `' . $id_field . '` FROM `' . $translate['table'] . '`)';
+    $ids = dbFetchColumn($query, [ $entity_type ], $test);
+    $count = safe_count($ids);
+    if ($count) {
+      if ($prompt) {
+        $answer = print_prompt("$count rows in table '$table' for non-existing '$entity_type' entities will be deleted");
+      }
+
+      // Remove stale entries
+      print_debug("Database cleanup for table '$table': deleted $count '$entity_type' entities");
+      if (!$test) {
+        $table_status = dbDelete($table, "`entity_type` = ? ".generate_query_values($ids, 'entity_id'), [ $entity_type ]);
+        logfile("housekeeping.log", "Database cleanup for table '$table': deleted $count '$entity_type' entities");
+      }
+    } elseif ($prompt) {
+      print_message("No orphaned rows found in table '$table' for '$entity_type' entities.");
+    }
+  }
+}
+*/
+
+// Cleanup duplicate entries in the device_graphs table
+$graphs = [];
+foreach (dbFetchRows("SELECT * FROM `device_graphs`", NULL, $test) as $entry) {
+  $graphs[$entry['device_id']][$entry['graph']][] = $entry['device_graph_id'];
+}
+print_debug_vars($graphs);
+
+foreach ($graphs as $device_id => $device_graph) {
+  foreach ($device_graph as $graph => $data) {
+    if (safe_count($data) > 1) {
+      // More than one entry for a single graph type for this device, let's clean up.
+      // Leave the first entry intact, chop it off the array
+      $device_graph_ids = array_slice($data, 1);
+      $device_graph_count = safe_count($device_graph_ids);
+      if ($prompt) {
+        $answer = print_prompt($device_graph_count . " duplicate graph rows of type $graph for device $device_id will be deleted");
+      }
+      if ($answer) {
+        if (!$test) {
+          $table_status = dbDelete('device_graphs', "`device_graph_id` IN (?)", [ $device_graph_ids ]);
+          logfile("housekeeping.log", "Deleted " . $device_graph_count . " duplicate graph rows of type $graph for device $device_id");
+        }
+        print_debug("Deleted " . $device_graph_count . " duplicate graph rows of type $graph for device $device_id");
+      }
+    }
+  }
+}
+
+// EOF

includes/housekeeping/syslog.inc.php

@@ -0,0 +1,58 @@
+<?php
+
+/**
+ * Observium
+ *
+ *   This file is part of Observium.
+ *
+ * @package    observium
+ * @subpackage housekeeping
+ * @author     Adam Armstrong <adama@observium.org>
+ * @copyright  (C) 2006-2013 Adam Armstrong, (C) 2013-2019 Observium Limited
+ *
+ */
+
+// Minimum allowed age for delete syslog entries times is 24h
+$cutoff = age_to_unixtime($config['housekeeping']['syslog']['age'], age_to_seconds('24h'));
+
+if ($cutoff)
+{
+  $where = "`timestamp` < FROM_UNIXTIME($cutoff)";
+  $count = dbFetchCell("SELECT COUNT(*) FROM `syslog` WHERE $where");
+  if ($count)
+  {
+    if ($prompt)
+    {
+      $answer = print_prompt("$count syslog entries older than " . format_unixtime($cutoff) . " will be deleted");
+    }
+
+    if ($answer)
+    {
+      $rows = dbDelete('syslog', "$where");
+      if ($rows === FALSE)
+      {
+        // Use LIMIT with big tables
+        print_debug("Syslog table is too big, using LIMIT for delete entries");
+        $rows = 0;
+        $i    = 1000;
+        while ($i && $rows < $count)
+        {
+          $iter = dbDelete('syslog', $where.' LIMIT 1000000');
+          if ($iter === FALSE) { break; }
+          $rows += $iter;
+          $i--;
+        }
+      }
+      print_debug("Syslog housekeeping: deleted $rows entries");
+      logfile("housekeeping.log", "Syslog: deleted $rows entries older than " . format_unixtime($cutoff));
+    }
+  }
+  else if ($prompt)
+  {
+    print_message("No syslog entries found older than " . format_unixtime($cutoff));
+  }
+} else {
+  print_message("Syslog housekeeping disabled in configuration or incorrectly configured to less than 24h.");
+}
+
+// EOF