deferred-data-backfill

#!/usr/bin/php
<?php
/**
	Program to upload data to SevOne using the Deferred Data API
	Based on DK's script, located at https://gist.github.com/3004669

	@author MC (mc@sevone.com)
	@version 0.2
	@since 27-June-2012
*/

function usage() { // {{{
	echo 'Usage:', PHP_EOL;
	echo '   deferred-data-import sevone-hostname-or-ip sevone-username sevone-password sevone-full-device-name', PHP_EOL;
	echo PHP_EOL;
/* }}} */ }

//! The Appliance IP
$soapIp = '127.0.0.1';

//! The username and password
$username = '<username>';
$password = '<password>';

//! The Device to add the objects to
$deviceName = 'myrouter';

if($argc != 5) {
	usage();
	exit;
}

$soapIp = $argv[1];
$username = $argv[2];
$password = $argv[3];
$deviceName = $argv[4];

//! This is the WSDL url.
$soapUrl = 'http://' . $soapIp . '/soap3/api.wsdl';

//! Connect to the SOAP server.
$client = new SoapClient($soapUrl, array('cache_wsdl' => WSDL_CACHE_NONE));
if( !$client ) {
	echo '!!! Could not connect to SOAP server at "', $soapUrl, '".', PHP_EOL;
	exit( 1 );
}
$client->__setLocation('http://' . $soapIp . '/soap3/api.php');

echo '== Information ==', PHP_EOL;
echo 'IP: ', $soapIp, PHP_EOL;
echo 'URL: ', $soapUrl, PHP_EOL;
echo 'Username: ', $username, PHP_EOL;
echo 'Password: ', $password, PHP_EOL;
echo 'Device name: ', $deviceName, PHP_EOL;

echo '== Authentication ==', PHP_EOL;
try {
	//! This authenticates a user for the duration of this script.
	$result = $client->authenticate($username, $password);
	if( !$result ) {
		echo "!!! Could not authenticate with the server.\n";
		exit( 1 );
	} else {
		echo 'Authentication successful: ', $result, PHP_EOL;
	}
} catch( Exception $e ) {
	echo "Exception:\n";
	print_r( $e );
	exit( 1 );
}

//! Get the device
try {
	if(!$device = $client->core_getDeviceByName($deviceName)) {
		echo '!!! Could not find host device.', PHP_EOL;
		exit( 1 );
	} else {
		echo 'Device found: ', $device->id, PHP_EOL;
	}
} catch(Exception $e) {
	echo '!!! Could not find host device: Exception:', PHP_EOL;
	print_r($e);
	exit(1);
}

define('SO_PARSE_MODE_NORM', 0);
define('SO_PARSE_MODE_INQUOTE_SINGLE', 1);
define('SO_PARSE_MODE_INQUOTE_DOUBLE', 2);
// We are expecting:
//    "timestamp" "object name" "object type" "object description" ["indicator name 1" "indicator format 1" "indicator units 1" "indicator value 1" ["indicator name 2" "indicator format 2" "indicator units 2" "indicator value 2" [... etc]]]
// Note that the units can be just a single unit for both measurement and display, or it can be of the format "measurement units//display units"
function split_data_line($line) { // {{{
	//$parts = preg_split('/(\s+|\'|")/', $line, -1, PREG_SPLIT_DELIM_CAPTURE);
	$parts = preg_split('/(\s+)/', $line, -1, PREG_SPLIT_DELIM_CAPTURE);
	$items = array();
	$mode = SO_PARSE_MODE_NORM;
	$currentItem = '';
	foreach($parts as $part) {
		switch($mode) {
			case SO_PARSE_MODE_NORM:
				if(preg_match('/^\s+$/', $part)) {
					if(substr($currentItem, -1) == "\\" && substr($currentItem, -2, 1) != "\\") {
						$currentItem = substr($currentItem, 0, -1);
					} else {
						$items[] = $currentItem;
						$currentItem = '';
						continue;
					}
				} else {
					$startChar = substr($part, 0, 1);
					if($startChar == '"') {
						$part = substr($part, 1);
						$endChar1 = substr($part, -2, 1);
						$endChar2 = substr($part, -1);
						if($endChar2 == '"' && $endChar1 != "\\") {
							$part = substr($part, 0, -1);
						} else {
							$mode = SO_PARSE_MODE_INQUOTE_DOUBLE;
						}
					} elseif($startChar == '\'') {
						$part = substr($part, 1);
						$endChar1 = substr($part, -2, 1);
						$endChar2 = substr($part, -1);
						if($endChar2 == '\'' && $endChar1 != "\\") {
							$part = substr($part, 0, -1);
						} else {
							$mode = SO_PARSE_MODE_INQUOTE_SINGLE;
						}
					}
				}
				$currentItem .= $part;
				break;
			case SO_PARSE_MODE_INQUOTE_SINGLE:
				$endChar1 = substr($part, -2, 1);
				$endChar2 = substr($part, -1);
				if($endChar2 == '\'' && $endChar1 != "\\") {
					$part = substr($part, 0, -1);
					$mode = SO_PARSE_MODE_NORM;
				}
				$currentItem .= $part;
				break;
			case SO_PARSE_MODE_INQUOTE_DOUBLE:
				$endChar1 = substr($part, -2, 1);
				$endChar2 = substr($part, -1);
				if($endChar2 == '"' && $endChar1 != "\\") {
					$part = substr($part, 0, -1);
					$mode = SO_PARSE_MODE_NORM;
				}
				$currentItem .= $part;
				break;
		}
	}
	return $items;
/* }}} */ }

function parse_data_line($line) { // {{{
	$items = split_data_line($line);
	$original_time = $items[0];

	$output = array(
		'time' => is_numeric($original_time) ? $original_time : @strtotime($original_time),
		'object_name' => $items[1],
		'object_type' => $items[2],
		'object_description' => $items[3],
		'indicators' => array()
	);

	if($output['time'] === false) {
		echo '!!! Data row for object "', $output['object_name'], '" had invalid time "', $original_time, '"', PHP_EOL;
		return false;
	} elseif(count($items) % 4 != 0) {
		echo '!!! Data row for object "', $output['object_name'], '" at time "', $original_time, '" had an invalid number of arguments (', count($items), ')', PHP_EOL;
		return false;
	}

	for($i = 4; $i + 2 < count($items); $i += 4) {
		$output['indicators'][] = array(
			'name' => $items[$i],
			'type' => $items[$i + 1],
			'units' => $items[$i + 2],
			'value' => $items[$i + 3]
		);
	}

	foreach($output['indicators'] as $i => &$indicator) {
		if(!in_array($indicator['type'], array('GAUGE', 'COUNTER32', 'COUNTER64'))) {
			echo '!!! Indicator number ', $i, ' for object "', $output['object_name'], '" at time "', $original_time, '" had invalid format "', $indicator['type'], '"', PHP_EOL;
			return false;
		}
		$matches = null;
		if(preg_match(':(.+)//(.+):', $indicator['units'], $matches)) {
			$indicator['data_units'] = $matches[1];
			$indicator['display_units'] = $matches[2];
			unset($indicator['units']);
		} else {
			$indicator['data_units'] = $indicator['units'];
			$indicator['display_units'] = $indicator['units'];
			unset($indicator['units']);
		}
		if(preg_match(':(.+)/(.+):', $indicator['value'], $matches)) {
			$indicator['value'] = $matches[1];
			$indicator['maxValue'] = $matches[2];
		}
	}

	return $output;
/* }}} */ }

/**
	Report Statistics to the Server

	This function will lookup and create the object type and indicator types necessary to submit data to the server
	It will also handle flagging a device for discovery if a new object needs to be created.
	This assumes the $client variable exists and it a valid SevOne API client

	@param string[$device] The name of the device to associate this data with
	@param int[$time] The timestamp to associate with the data
	@param string[$objectType] The object type
	@param string[$objectName] The name of the object
	@param string[$objectDescription] The name of the object
	@param array[$indicators] An array of indicators. having this structure
	array(
		"name" => "<indicator name>",
		"value" => "<indicator value>",
		// The following are optional
		"type" => "<indicator type>",
		"maxValue" => "<indicator Max Value>",
		"data_units" => "<Units in which this indicator is measured>"
		"display_units" => "<Units in which this indicator is displayed>"
	)

	@return None
*/

function insert_from_queue($deviceId, $objectId, &$queue) {
	global $client;
	$client->plugin_deferred_insertDataRows($deviceId, $queue[$objectId]->indicatorIds, $queue[$objectId]->time, $queue[$objectId]->data);
	unset($queue[$objectId]);
}

$dataQueue = array();
function report_stats($deviceId, $time, $objectType, $objectName, $objectDescription, $indicators) /* {{{ */ {
	global $client;
	global $dataQueue;
	static $ot_by_name = array();
	static $itypes_by_otId = array();
	static $obj_by_devIdAndName = array();
	static $inds_by_devIdAndObjId = array();

	//! Find the object type (create if necessary)
	if(array_key_exists($objectType, $ot_by_name)) {
		$ot = $ot_by_name[$objectType];
	} else {
		echo '+++ ot_by_name cache miss ("', $objectType, '")', PHP_EOL;

		try {
			$ot = $client->plugin_deferred_getObjectTypeByOsIdAndName(0, $objectType);
		} catch(SoapFault $e) {
			$ots = $client->plugin_deferred_getObjectTypes();
			$ot = null;
			foreach($ots as $_ot) {
				if($_ot->name == $objectType) {
					$ot = $_ot;
					break;
				}
			}
		}

		if(!$ot) {
			$otid = $client->plugin_deferred_createObjectType(0, $objectType);
			sleep(5);
			$ot = $client->plugin_deferred_getObjectTypeById($otid);
		}
		if(!$ot) {
			throw new Exception("Error creating new object type\n");
		} elseif(isset($otid)) {
			echo '--- Created new Object Type, called "', $objectType, '", with ID ', $otid, PHP_EOL;
		}
		$ot_by_name[$objectType] = $ot;
	}

	//! Find the Indicator Types (create if necessary)
	unset($indicatorTypes);
	if(array_key_exists($ot->id, $itypes_by_otId)) {
		$indicatorTypes = &$itypes_by_otId[$ot->id];
	} else {
		echo '+++ itypes_by_otId cache miss ("', $ot->id, '")', PHP_EOL;
		$indicatorTypes = array();
		$itypes = $client->plugin_deferred_getIndicatorTypesByObjectTypeId($ot->id);
		foreach($itypes as $indicator) {
			$indicatorTypes[$indicator->name] = $indicator;
		}
		$itypes_by_otId[$ot->id] = &$indicatorTypes;
	}
	foreach($indicators as $indicator) {
		if(!array_key_exists($indicator["name"], $indicatorTypes)){
			$id = $client->plugin_deferred_createIndicatorType($ot->id, $indicator['name']);
			echo '--- Created new Indicator Type, called "', $indicator['name'], '", with ID ', $id, ', on Object Type ', $ot->id, PHP_EOL;
			if(array_key_exists("type", $indicator)) {
				$client->plugin_deferred_setIndicatorTypeFormat($id, $indicator['type']);
			}
			if(array_key_exists('data_units', $indicator) && array_key_exists('display_units', $indicator)) {
				$client->plugin_deferred_setIndicatorTypeUnits($id, $indicator['data_units'], $indicator['display_units']);
			} elseif(array_key_exists('data_units', $indicator)) {
				$client->plugin_deferred_setIndicatorTypeUnits($id, $indicator['data_units']);
			}
			if(array_key_exists("maxValue",$indicator)) {
				$client->plugin_deferred_setIndicatorTypeHasMaxValue($id, 1);
			}
			$indicatorTypes[$indicator["name"]] = $client->plugin_deferred_getIndicatorTypeByObjectTypeIdAndName($ot->id, $indicator["name"]);
		}
	}

	//! Get the object
	$objectKey = $deviceId . ':' . $objectName;
	if(array_key_exists($objectKey, $obj_by_devIdAndName)) {
		$obj = $obj_by_devIdAndName[$objectKey];
	} else {
		echo '+++ $obj_by_devIdAndName cache miss ("', $objectKey, '")', PHP_EOL;
		$obj = $client->plugin_deferred_getObject($deviceId,$objectName);
		if(!(array)$obj) {
			$objId = $client->plugin_deferred_createObject($deviceId,$ot->id,$objectName);
			echo '--- Created new Object, called "', $objectName, '", with ID ', $objId, ' and Object Type ', $ot->id, PHP_EOL;
			$client->plugin_deferred_setObjectOverrideDescription($deviceId, $objId, 1, $objectDescription);
			$client->plugin_deferred_setObjectIsEnabled($deviceId, $objId, 1);
			$client->plugin_deferred_setObjectIsHidden($deviceId, $objId, 0);
			//! If this object doesn't exist we need discovery to make it
			$client->core_rediscoverDevice($deviceId);
			return;
		}
		$client->plugin_deferred_setObjectOverrideDescription($deviceId, $obj->id, 1, $objectDescription);
		$obj_by_devIdAndName[$objectKey] = $obj;
	}

	//! Find the indicators (Create if necessary)
	$indicatorKey = $deviceId . ':' . $obj->id;
	if(array_key_exists($indicatorKey, $inds_by_devIdAndObjId)) {
		$inds = $inds_by_devIdAndObjId[$indicatorKey];
	} else {
		echo '+++ $inds_by_devIdAndObjId cache miss ("', $indicatorKey, '")', PHP_EOL;
		$inds = array();
		$itypes = $client->plugin_deferred_getIndicatorsByObject($deviceId, $obj->name);
		foreach($itypes as $indicator) {
			$inds[$indicator->indicatorType] = $indicator;
		}
		$inds_by_devIdAndObjId[$indicatorKey] = $inds;
	}

	//! Submit the indicators
	$data = array();
	foreach($indicators as $indicator) {
		if(array_key_exists($indicator["name"], $inds)){
			$data[$inds[$indicator["name"]]->id] = $indicator["value"];
			if(array_key_exists("maxValue", $indicator)) {
				$client->plugin_deferred_setIndicatorOverrideMaxValue($deviceId,$inds[$indicator["name"]]->id,1,$indicator["maxValue"]);
			}
		}
	}

	if(count($data) > 0) {
		foreach($data as &$value) {
			if(!is_numeric($value)) {
				$value = null;
			}
		}

		if(!(array_key_exists($obj->id, $dataQueue) && is_object($dataQueue[$obj->id]))) {
			$dataQueue[$obj->id] = (object)array(
				'indicatorIds' => array_keys($data),
				'time' => array(),
				'data' => array()
			);
		}

		$dataQueue[$obj->id]->time[] = $time;
		$dataQueue[$obj->id]->data[$time] = array_values($data);

		if(count($dataQueue[$obj->id]->time) > 250) {
			echo '--- Inserting ', count($dataQueue[$obj->id]->time), ' data rows for Object "', $objectName, '" (ID# ', $obj->id, ') at ', $time, ' (', @date('Y-m-d H:i:s', $time), ')', PHP_EOL;
			insert_from_queue($deviceId, $obj->id, $dataQueue);
		}
	}
} // }}}

//read in the lines from the script
$in = fopen('php://stdin', 'r');
while($line = fgets($in)) {
	$parseyBits = parse_data_line($line);
	if($parseyBits === false) {
		continue;
	}

	//report the data
	try {
		report_stats($device->id, $parseyBits['time'], $parseyBits['object_type'], $parseyBits['object_name'], $parseyBits['object_description'], $parseyBits['indicators']);
	} catch(Exception $e) {
		echo 'Exception:', PHP_EOL;
		print_r($e);
		exit(1);
	}
}

foreach($dataQueue as $objectId => $queue) {
	if(is_object($queue) && count($queue->time) > 0) {
		echo '--- Inserting ', count($queue->time), ' leftover rows for Object ID# ', $objectId, PHP_EOL;
		insert_from_queue($device->id, $objectId, $dataQueue);
	}
}