From 932cd992ff2a9fa9c2eb7489c67a947d905e2c14 Mon Sep 17 00:00:00 2001 From: Walter Dal Mut Date: Thu, 25 Jun 2015 09:56:58 +0200 Subject: [PATCH 1/2] Optimized list to string conversion The inline protocol converts key values to strings --- README.md | 6 +- .../Benchmarks/InfluxDB/AdapterEvent.php | 2 + .../InfluxDB/MessageToInlineProtocolEvent.php | 28 ++------ composer.json | 3 +- composer.lock | 2 +- src/Adapter/UdpAdapter.php | 68 +++---------------- src/Adapter/helpers.php | 52 ++++++++++++++ src/inline_protocol_functions.php | 15 ++++ tests/integration/Adapter/UdpAdapterTest.php | 2 + tests/unit/Adapter/HelpersTest.php | 23 +++++++ tests/unit/Adapter/UdpAdapterTest.php | 51 +++++++++----- 11 files changed, 148 insertions(+), 104 deletions(-) create mode 100644 src/Adapter/helpers.php create mode 100644 src/inline_protocol_functions.php create mode 100644 tests/unit/Adapter/HelpersTest.php diff --git a/README.md b/README.md index e8f640f1a9..50460f30ef 100644 --- a/README.md +++ b/README.md @@ -284,9 +284,9 @@ The impact of message to inline protocol conversion is: Corley\Benchmarks\InfluxDB\MessageToInlineProtocolEvent Method Name Iterations Average Time Ops/second ---------------------------------------------------- ------------ -------------- ------------- - convertMessageToInlineProtocolWithNoTags : [10,000 ] [0.0000237422466] [42,119.01324] - convertMessageToInlineProtocolWithGlobalTags : [10,000 ] [0.0000306700468] [32,605.10185] - convertMessageToInlineProtocolWithDifferentTagLevels: [10,000 ] [0.0000343942404] [29,074.63543] + convertMessageToInlineProtocolWithNoTags : [10,000 ] [0.0000343696594] [29,095.42942] + convertMessageToInlineProtocolWithGlobalTags : [10,000 ] [0.0000437165260] [22,874.64469] + convertMessageToInlineProtocolWithDifferentTagLevels: [10,000 ] [0.0000493728638] [20,254.04086] ``` ## FAQ diff --git a/benchmarks/Benchmarks/InfluxDB/AdapterEvent.php b/benchmarks/Benchmarks/InfluxDB/AdapterEvent.php index e52604941f..d3d37f6e5e 100644 --- a/benchmarks/Benchmarks/InfluxDB/AdapterEvent.php +++ b/benchmarks/Benchmarks/InfluxDB/AdapterEvent.php @@ -23,6 +23,8 @@ class AdapterEvent extends AthleticEvent $options->setDatabase("tcp.test"); $client = new Client(new GuzzleAdapter(new HttpClient(), $options)); + $client->createDatabase("tcp.test"); + $client->createDatabase("udp.test"); $this->httpClient = $client; diff --git a/benchmarks/Benchmarks/InfluxDB/MessageToInlineProtocolEvent.php b/benchmarks/Benchmarks/InfluxDB/MessageToInlineProtocolEvent.php index 4bfce6ee9c..964fdc34e6 100644 --- a/benchmarks/Benchmarks/InfluxDB/MessageToInlineProtocolEvent.php +++ b/benchmarks/Benchmarks/InfluxDB/MessageToInlineProtocolEvent.php @@ -2,31 +2,15 @@ namespace Corley\Benchmarks\InfluxDB; use Athletic\AthleticEvent; -use InfluxDB\Adapter\UdpAdapter; -use InfluxDB\Options; class MessageToInlineProtocolEvent extends AthleticEvent { - private $method; - private $object; - - public function setUp() - { - $object = new UdpAdapter(new Options()); - $reflection = new \ReflectionClass(get_class($object)); - $method = $reflection->getMethod("serialize"); - $method->setAccessible(true); - - $this->method = $method; - $this->object = $object; - } - /** * @iterations 10000 */ public function convertMessageToInlineProtocolWithNoTags() { - $this->method->invokeArgs($this->object, [ + \InfluxDB\Adapter\message_to_inline_protocol( [ "points" => [ [ @@ -38,7 +22,7 @@ class MessageToInlineProtocolEvent extends AthleticEvent ], ] ] - ]); + ); } /** @@ -46,7 +30,7 @@ class MessageToInlineProtocolEvent extends AthleticEvent */ public function convertMessageToInlineProtocolWithGlobalTags() { - $this->method->invokeArgs($this->object, [ + \InfluxDB\Adapter\message_to_inline_protocol( [ "tags" => [ "dc" => "eu-west-1", @@ -61,7 +45,7 @@ class MessageToInlineProtocolEvent extends AthleticEvent ], ] ] - ]); + ); } /** @@ -69,7 +53,7 @@ class MessageToInlineProtocolEvent extends AthleticEvent */ public function convertMessageToInlineProtocolWithDifferentTagLevels() { - $this->method->invokeArgs($this->object, [ + \InfluxDB\Adapter\message_to_inline_protocol( [ "tags" => [ "dc" => "eu-west-1", @@ -87,6 +71,6 @@ class MessageToInlineProtocolEvent extends AthleticEvent ], ] ] - ]); + ); } } diff --git a/composer.json b/composer.json index a9f2435c8c..dd69664b3f 100644 --- a/composer.json +++ b/composer.json @@ -35,7 +35,8 @@ "psr-4": { "InfluxDB\\": ["./src/"], "Corley\\": ["./benchmarks/"] - } + }, + "files": ["src/Adapter/helpers.php"] }, "autoload-dev": { "psr-4": { diff --git a/composer.lock b/composer.lock index 16711fc9f3..61cce25d3e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "ea197cdcb24cfde2b668ae1750adbfc0", + "hash": "6378134d2378fee11d7741fb4343074d", "packages": [ { "name": "guzzlehttp/guzzle", diff --git a/src/Adapter/UdpAdapter.php b/src/Adapter/UdpAdapter.php index 50f4205121..ac9a08ba64 100644 --- a/src/Adapter/UdpAdapter.php +++ b/src/Adapter/UdpAdapter.php @@ -8,7 +8,12 @@ class UdpAdapter extends AdapterAbstract public function send(array $message) { $message = array_replace_recursive($this->getMessageDefaults(), $message); - $message = $this->serialize($message); + + if (array_key_exists("tags", $message)) { + $message["tags"] = array_replace_recursive($this->getOptions()->getTags(), $message["tags"]); + } + + $message = message_to_inline_protocol($message); $this->write($message); } @@ -17,8 +22,8 @@ class UdpAdapter extends AdapterAbstract { // Create a handler in order to handle the 'Host is down' message set_error_handler(function() { - // Suppress the error, this is the UDP adapter and if we can't send - // it then we shouldn't inturrupt their application. + // Suppress the error, this is the UDP adapter and if we can't send + // it then we shouldn't inturrupt their application. }); $socket = socket_create(AF_INET, SOCK_DGRAM, SOL_UDP); @@ -28,61 +33,4 @@ class UdpAdapter extends AdapterAbstract // Remove our error handler. restore_error_handler(); } - - private function serialize(array $message) - { - $tags = $this->getOptions()->getTags(); - - if (array_key_exists("tags", $message)) { - $tags = array_replace_recursive($tags, $message["tags"]); - } - - $unixepoch = $this->generateTimeInNanoSeconds(); - if (array_key_exists("time", $message)) { - $dt = new DateTime($message["time"]); - $unixepoch = (int)($dt->format("U") * 1e9); - } - - $lines = []; - foreach ($message["points"] as $point) { - if (array_key_exists("tags", $point)) { - $tags = array_replace_recursive($tags, $point["tags"]); - } - - if (!$tags) { - $lines[] = trim( - sprintf( - "%s %s %d", - $point["measurement"], $this->toKeyValue($point["fields"], true), $unixepoch - ) - ); - } else { - $lines[] = trim( - sprintf( - "%s,%s %s %d", - $point["measurement"], $this->toKeyValue($tags), $this->toKeyValue($point["fields"], true), $unixepoch - ) - ); - } - } - return implode("\n", $lines); - } - - protected function generateTimeInNanoSeconds() - { - return (int)(microtime(true) * 1e9); - } - - protected function toKeyValue(array $elems, $escape = false) - { - $list = []; - foreach ($elems as $key => $value) { - if ($escape && is_string($value)) { - $value = "\"{$value}\""; - } - $list[] = sprintf("%s=%s", $key, $value); - } - - return implode(",", $list); - } } diff --git a/src/Adapter/helpers.php b/src/Adapter/helpers.php new file mode 100644 index 0000000000..fcdfe2318f --- /dev/null +++ b/src/Adapter/helpers.php @@ -0,0 +1,52 @@ +format("U") * 1e9); + } + + $lines = []; + foreach ($message["points"] as $point) { + $tags = array_key_exists("tags", $message) ? $message["tags"] : []; + if (array_key_exists("tags", $point)) { + $tags = array_replace_recursive($tags, $point["tags"]); + } + + if (!$tags) { + $lines[] = trim( + sprintf( + "%s %s %d", + $point["measurement"], list_to_string($point["fields"], true), $unixepoch + ) + ); + } else { + $lines[] = trim( + sprintf( + "%s,%s %s %d", + $point["measurement"], list_to_string($tags), list_to_string($point["fields"], true), $unixepoch + ) + ); + } + } + + return implode("\n", $lines); +} + +function list_to_string(array $elements, $escape = false) +{ + array_walk($elements, function(&$value, $key) use ($escape) { + if ($escape && is_string($value)) { + $value = "\"{$value}\""; + } + + $value = "{$key}={$value}"; + }); + + return implode(",", $elements); +} diff --git a/src/inline_protocol_functions.php b/src/inline_protocol_functions.php new file mode 100644 index 0000000000..5fffa2fb2c --- /dev/null +++ b/src/inline_protocol_functions.php @@ -0,0 +1,15 @@ + "mem", "fields" => [ "value" => 1233, + "with_string" => "this is a string", ], ], ], @@ -49,5 +50,6 @@ class UdpAdapterTest extends InfluxDBTestCase $this->assertSerieExists("udp.test", "mem"); $this->assertSerieCount("udp.test", "mem", 1); $this->assertValueExistsInSerie("udp.test", "mem", "value", 1233); + $this->assertValueExistsInSerie("udp.test", "mem", "with_string", "this is a string"); } } diff --git a/tests/unit/Adapter/HelpersTest.php b/tests/unit/Adapter/HelpersTest.php new file mode 100644 index 0000000000..cc3776413f --- /dev/null +++ b/tests/unit/Adapter/HelpersTest.php @@ -0,0 +1,23 @@ +assertEquals($result, list_to_string($message, $escape)); + } + + public function getElements() + { + return [ + [["one" => "two"], "one=two", false], + [["one" => "two"], "one=\"two\"", true], + [["one" => "two", "three" => "four"], "one=two,three=four", false], + [["one" => "two", "three" => "four"], "one=\"two\",three=\"four\"", true], + ]; + } +} diff --git a/tests/unit/Adapter/UdpAdapterTest.php b/tests/unit/Adapter/UdpAdapterTest.php index d9125f1b75..e9e3ce4f64 100644 --- a/tests/unit/Adapter/UdpAdapterTest.php +++ b/tests/unit/Adapter/UdpAdapterTest.php @@ -17,13 +17,15 @@ class UdpAdapterTest extends \PHPUnit_Framework_TestCase public function testRewriteMessages($input, $response) { $object = new UdpAdapter(new Options()); - $reflection = new \ReflectionClass(get_class($object)); - $method = $reflection->getMethod("serialize"); - $method->setAccessible(true); + $object = $this->getMockBuilder("InfluxDB\Adapter\UdpAdapter") + ->setConstructorArgs([new Options()]) + ->setMethods(["write"]) + ->getMock(); + $object->expects($this->once()) + ->method("write") + ->with($response); - $message = $method->invokeArgs($object, [$input]); - - $this->assertEquals($response, $message); + $object->send($input); } public function getMessages() @@ -43,6 +45,21 @@ class UdpAdapterTest extends \PHPUnit_Framework_TestCase ], "cpu value=1 1257894000000000000" ], + [ + [ + "time" => "2009-11-10T23:00:00Z", + "points" => [ + [ + "measurement" => "cpu", + "fields" => [ + "value" => 1, + "string" => "escape", + ], + ], + ], + ], + "cpu value=1,string=\"escape\" 1257894000000000000" + ], [ [ "tags" => [ @@ -115,7 +132,7 @@ EOF $adapter->expects($this->once()) ->method("write") - ->with("udp.test mark=\"element\" 1245"); + ->with($this->matchesRegularExpression("/udp.test mark=\"element\" \d+/i")); $adapter->send([ "points" => [ @@ -146,11 +163,11 @@ EOF $adapter->expects($this->once()) ->method("write") - ->with(<<with($this->matchesRegularExpression(<<send([ "points" => [ @@ -189,10 +206,10 @@ EOF $adapter->expects($this->once()) ->method("write") - ->with(<<with($this->matchesRegularExpression(<<send([ "tags" => [ @@ -228,10 +245,10 @@ EOF $adapter->expects($this->once()) ->method("write") - ->with(<<with($this->matchesRegularExpression(<<send([ "tags" => [ From 4517e5c31c65219eb14df50f5b713704f79cc401 Mon Sep 17 00:00:00 2001 From: Walter Dal Mut Date: Sat, 27 Jun 2015 14:20:46 +0200 Subject: [PATCH 2/2] Cover merge global tags also when are missing --- tests/unit/Adapter/UdpAdapterTest.php | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/unit/Adapter/UdpAdapterTest.php b/tests/unit/Adapter/UdpAdapterTest.php index e9e3ce4f64..68271bac49 100644 --- a/tests/unit/Adapter/UdpAdapterTest.php +++ b/tests/unit/Adapter/UdpAdapterTest.php @@ -187,6 +187,35 @@ EOF ]); } + /** + * @group udp + */ + public function testFillWithGlobalTags() + { + $options = (new Options()) + ->setDatabase("test") + ->setTags(["dc" => "eu-west"]); + $adapter = $this->getMockBuilder("InfluxDB\\Adapter\\UdpAdapter") + ->setConstructorArgs([$options]) + ->setMethods(["write"]) + ->getMock(); + + $adapter->expects($this->once()) + ->method("write") + ->with($this->matchesRegularExpression("/mem,dc=eu-west free=712423 \d+/i")); + + $adapter->send([ + "points" => [ + [ + "measurement" => "mem", + "fields" => [ + "free" => 712423, + ], + ], + ] + ]); + } + /** * @group udp */