From a8bb26d155134a5269307acef23107932e828c75 Mon Sep 17 00:00:00 2001 From: Kenji Okimoto Date: Fri, 6 Jan 2017 16:32:38 +0900 Subject: [PATCH 1/7] Migrate to Fluentd v0.14 API --- lib/fluent/plugin/out_uri_decode.rb | 16 +++++------ test/plugin/test_out_uri_decoder.rb | 44 ++++++++++++++--------------- test/test_helper.rb | 23 ++++----------- 3 files changed, 34 insertions(+), 49 deletions(-) diff --git a/lib/fluent/plugin/out_uri_decode.rb b/lib/fluent/plugin/out_uri_decode.rb index 32af405..89cf186 100644 --- a/lib/fluent/plugin/out_uri_decode.rb +++ b/lib/fluent/plugin/out_uri_decode.rb @@ -1,17 +1,17 @@ -class Fluent::URIDecoder < Fluent::Output +require 'fluent/plugin/output' +require 'uri' + +class Fluent::URIDecoder < Fluent::Plugin::Output Fluent::Plugin.register_output('uri_decode', self) + helpers :event_emitter + config_param :tag, :string, :default => nil config_param :remove_prefix, :string, :default => nil config_param :add_prefix, :string, :default => nil config_param :key_name, :string, :default => nil config_param :key_names, :string, :default => '' - def initialize - super - require 'uri' - end - def configure(conf) super @@ -50,7 +50,7 @@ def tag_mangle(tag) end end - def emit(tag, es, chain) + def process(tag, es) tag = tag_mangle(tag) @_key_names ||= @key_names.split(/,\s*/) @@ -64,7 +64,5 @@ def emit(tag, es, chain) router.emit(tag, time, record) end - - chain.next end end diff --git a/test/plugin/test_out_uri_decoder.rb b/test/plugin/test_out_uri_decoder.rb index efe917b..8060b49 100644 --- a/test/plugin/test_out_uri_decoder.rb +++ b/test/plugin/test_out_uri_decoder.rb @@ -26,8 +26,8 @@ def setup Fluent::Test.setup end - def create_driver(conf=CONFIG0, tag='test') - Fluent::Test::OutputTestDriver.new(Fluent::URIDecoder, tag).configure(conf) + def create_driver(conf=CONFIG0) + Fluent::Test::Driver::Output.new(Fluent::URIDecoder).configure(conf) end def test_configure @@ -55,12 +55,12 @@ def test_tag_mangle def test_emit_with_tag_specification d = create_driver(CONFIG0) - time = Time.now.to_i - d.run do - d.emit({'encoded' => '%23hash', 'value' => 1}, time) + time = event_time + d.run(default_tag: 'test') do + d.feed(time, {'encoded' => '%23hash', 'value' => 1}) end - emits = d.emits + emits = d.events assert_equal 1, emits.size assert_equal 'decoded', emits[0][0] assert_equal time, emits[0][1] @@ -68,13 +68,13 @@ def test_emit_with_tag_specification end def test_emit_with_prefix_specification - d = create_driver(CONFIG1, 'encoded.message') - time = Time.now.to_i - d.run do - d.emit({'encoded' => '%23hash', 'value' => 1}, time) + d = create_driver(CONFIG1) + time = event_time + d.run(default_tag: 'encoded.message') do + d.feed(time, {'encoded' => '%23hash', 'value' => 1}) end - emits = d.emits + emits = d.events assert_equal 1, emits.size assert_equal 'decoded.message', emits[0][0] assert_equal time, emits[0][1] @@ -82,13 +82,13 @@ def test_emit_with_prefix_specification end def test_emit_with_multi_key_names - d = create_driver(CONFIG2, 'encoded.message') - time = Time.now.to_i - d.run do - d.emit({'encoded' => '%23hash', 'another_encoded' => '%23another_hash'}, time) + d = create_driver(CONFIG2) + time = event_time + d.run(default_tag: 'encoded.message') do + d.feed(time, {'encoded' => '%23hash', 'another_encoded' => '%23another_hash'}) end - emits = d.emits + emits = d.events assert_equal 1, emits.size assert_equal 'decoded.message', emits[0][0] assert_equal time, emits[0][1] @@ -97,14 +97,14 @@ def test_emit_with_multi_key_names end def test_multiple_emit - d = create_driver(CONFIG2, 'encoded.message') - time = Time.now.to_i - d.run do - d.emit({'encoded' => '%23hash', 'another_encoded' => '%23another_hash'}, time) + d = create_driver(CONFIG2) + time = event_time + d.run(default_tag: 'encoded.message') do + d.feed(time, {'encoded' => '%23hash', 'another_encoded' => '%23another_hash'}) end - emits = d.emits - emits << d.emits.flatten + emits = d.events + emits << d.events.flatten assert_equal 2, emits.size assert_equal 'decoded.message', emits[0][0] assert_equal time, emits[0][1] diff --git a/test/test_helper.rb b/test/test_helper.rb index 246f822..9654f43 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,21 +1,8 @@ require 'bundler' - -begin - Bundler.setup(:default, :development) -rescue Bundler::BundlerError => e - $stderr.puts e.message - $stderr.puts "Run `bundle install` to install missing gems" - exit e.status_code -end - +require 'bundler/setup' require 'fluent/test' +require 'fluent/test/helpers' +require 'fluent/test/driver/output' +require 'test-unit' -unless ENV.has_key?('VERBOSE') - nulllogger = Object.new - nulllogger.instance_eval {|obj| - def method_missing(method, *args) - # pass - end - } - $log = nulllogger -end +Test::Unit::TestCase.include(Fluent::Test::Helpers) From 2aad578b1616610494612889845d485a11503f38 Mon Sep 17 00:00:00 2001 From: Kenji Okimoto Date: Fri, 6 Jan 2017 16:41:07 +0900 Subject: [PATCH 2/7] Rename plugin class name Because I will add filter version of this in another PR. --- lib/fluent/plugin/out_uri_decode.rb | 2 +- test/plugin/test_out_uri_decoder.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/fluent/plugin/out_uri_decode.rb b/lib/fluent/plugin/out_uri_decode.rb index 89cf186..848b5aa 100644 --- a/lib/fluent/plugin/out_uri_decode.rb +++ b/lib/fluent/plugin/out_uri_decode.rb @@ -1,7 +1,7 @@ require 'fluent/plugin/output' require 'uri' -class Fluent::URIDecoder < Fluent::Plugin::Output +class Fluent::URIDecoderOutput < Fluent::Plugin::Output Fluent::Plugin.register_output('uri_decode', self) helpers :event_emitter diff --git a/test/plugin/test_out_uri_decoder.rb b/test/plugin/test_out_uri_decoder.rb index 8060b49..f790ecd 100644 --- a/test/plugin/test_out_uri_decoder.rb +++ b/test/plugin/test_out_uri_decoder.rb @@ -1,7 +1,7 @@ require 'test_helper' require 'fluent/plugin/out_uri_decode' -class Fluent::URIDecorderTest < Test::Unit::TestCase +class Fluent::URIDecorderOutputTest < Test::Unit::TestCase CONFIG0 = %[ type uri_decode key_name encoded @@ -27,7 +27,7 @@ def setup end def create_driver(conf=CONFIG0) - Fluent::Test::Driver::Output.new(Fluent::URIDecoder).configure(conf) + Fluent::Test::Driver::Output.new(Fluent::URIDecoderOutput).configure(conf) end def test_configure From 5816e507dd8d91d0304be2e58242e60bb043648f Mon Sep 17 00:00:00 2001 From: Kenji Okimoto Date: Fri, 6 Jan 2017 16:49:40 +0900 Subject: [PATCH 3/7] Use array instead of string --- lib/fluent/plugin/out_uri_decode.rb | 10 ++++------ test/plugin/test_out_uri_decoder.rb | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/fluent/plugin/out_uri_decode.rb b/lib/fluent/plugin/out_uri_decode.rb index 848b5aa..04986d8 100644 --- a/lib/fluent/plugin/out_uri_decode.rb +++ b/lib/fluent/plugin/out_uri_decode.rb @@ -10,7 +10,7 @@ class Fluent::URIDecoderOutput < Fluent::Plugin::Output config_param :remove_prefix, :string, :default => nil config_param :add_prefix, :string, :default => nil config_param :key_name, :string, :default => nil - config_param :key_names, :string, :default => '' + config_param :key_names, :array, :default => [] def configure(conf) super @@ -28,6 +28,8 @@ def configure(conf) if @add_prefix @added_prefix_string = @add_prefix + '.' end + @key_names << @key_name if @key_name + @key_names.uniq! end def tag_mangle(tag) @@ -53,12 +55,8 @@ def tag_mangle(tag) def process(tag, es) tag = tag_mangle(tag) - @_key_names ||= @key_names.split(/,\s*/) - @_key_names << @key_name if @key_name - @_key_names.uniq! - es.each do |time, record| - @_key_names.each do |key_name| + @key_names.each do |key_name| record[key_name] = URI.decode_www_form_component(record[key_name] || '') end diff --git a/test/plugin/test_out_uri_decoder.rb b/test/plugin/test_out_uri_decoder.rb index f790ecd..600a491 100644 --- a/test/plugin/test_out_uri_decoder.rb +++ b/test/plugin/test_out_uri_decoder.rb @@ -40,7 +40,7 @@ def test_configure assert_equal 'decoded', d.instance.add_prefix d = create_driver(CONFIG2) - assert_equal 'encoded, another_encoded', d.instance.key_names + assert_equal ['encoded', 'another_encoded'], d.instance.key_names end def test_tag_mangle From adb1acf6e29a7a57ac8e3f447688697ec8cc9b54 Mon Sep 17 00:00:00 2001 From: Kenji Okimoto Date: Fri, 6 Jan 2017 16:56:21 +0900 Subject: [PATCH 4/7] Use new Hash syntax rubocop -a --only Style/HashSyntax --- lib/fluent/plugin/out_uri_decode.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/fluent/plugin/out_uri_decode.rb b/lib/fluent/plugin/out_uri_decode.rb index 04986d8..4bdb2a2 100644 --- a/lib/fluent/plugin/out_uri_decode.rb +++ b/lib/fluent/plugin/out_uri_decode.rb @@ -6,11 +6,11 @@ class Fluent::URIDecoderOutput < Fluent::Plugin::Output helpers :event_emitter - config_param :tag, :string, :default => nil - config_param :remove_prefix, :string, :default => nil - config_param :add_prefix, :string, :default => nil - config_param :key_name, :string, :default => nil - config_param :key_names, :array, :default => [] + config_param :tag, :string, default: nil + config_param :remove_prefix, :string, default: nil + config_param :add_prefix, :string, default: nil + config_param :key_name, :string, default: nil + config_param :key_names, :array, default: [] def configure(conf) super From 25dac7b5e57a32c8c08ee4eb71610599863552c6 Mon Sep 17 00:00:00 2001 From: Kenji Okimoto Date: Fri, 6 Jan 2017 17:05:15 +0900 Subject: [PATCH 5/7] Fix namespace --- lib/fluent/plugin/out_uri_decode.rb | 2 +- test/plugin/test_out_uri_decoder.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fluent/plugin/out_uri_decode.rb b/lib/fluent/plugin/out_uri_decode.rb index 4bdb2a2..8636548 100644 --- a/lib/fluent/plugin/out_uri_decode.rb +++ b/lib/fluent/plugin/out_uri_decode.rb @@ -1,7 +1,7 @@ require 'fluent/plugin/output' require 'uri' -class Fluent::URIDecoderOutput < Fluent::Plugin::Output +class Fluent::Plugin::URIDecoderOutput < Fluent::Plugin::Output Fluent::Plugin.register_output('uri_decode', self) helpers :event_emitter diff --git a/test/plugin/test_out_uri_decoder.rb b/test/plugin/test_out_uri_decoder.rb index 600a491..3280830 100644 --- a/test/plugin/test_out_uri_decoder.rb +++ b/test/plugin/test_out_uri_decoder.rb @@ -27,7 +27,7 @@ def setup end def create_driver(conf=CONFIG0) - Fluent::Test::Driver::Output.new(Fluent::URIDecoderOutput).configure(conf) + Fluent::Test::Driver::Output.new(Fluent::Plugin::URIDecoderOutput).configure(conf) end def test_configure From bc4db8b91de441f2c89d582728f5768bcd26eadf Mon Sep 17 00:00:00 2001 From: Kenji Okimoto Date: Fri, 6 Jan 2017 17:05:45 +0900 Subject: [PATCH 6/7] Fix a typo in test class name --- test/plugin/test_out_uri_decoder.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plugin/test_out_uri_decoder.rb b/test/plugin/test_out_uri_decoder.rb index 3280830..64e230f 100644 --- a/test/plugin/test_out_uri_decoder.rb +++ b/test/plugin/test_out_uri_decoder.rb @@ -1,7 +1,7 @@ require 'test_helper' require 'fluent/plugin/out_uri_decode' -class Fluent::URIDecorderOutputTest < Test::Unit::TestCase +class Fluent::URIDecoderOutputTest < Test::Unit::TestCase CONFIG0 = %[ type uri_decode key_name encoded From ac592a51a1c72241961d4a3a384b00e003e36185 Mon Sep 17 00:00:00 2001 From: Kenji Okimoto Date: Fri, 6 Jan 2017 17:06:09 +0900 Subject: [PATCH 7/7] Rename variables emits -> events --- test/plugin/test_out_uri_decoder.rb | 56 ++++++++++++++--------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/test/plugin/test_out_uri_decoder.rb b/test/plugin/test_out_uri_decoder.rb index 64e230f..c29223e 100644 --- a/test/plugin/test_out_uri_decoder.rb +++ b/test/plugin/test_out_uri_decoder.rb @@ -60,11 +60,11 @@ def test_emit_with_tag_specification d.feed(time, {'encoded' => '%23hash', 'value' => 1}) end - emits = d.events - assert_equal 1, emits.size - assert_equal 'decoded', emits[0][0] - assert_equal time, emits[0][1] - assert_equal '#hash', emits[0][2]['encoded'] + events = d.events + assert_equal 1, events.size + assert_equal 'decoded', events[0][0] + assert_equal time, events[0][1] + assert_equal '#hash', events[0][2]['encoded'] end def test_emit_with_prefix_specification @@ -74,11 +74,11 @@ def test_emit_with_prefix_specification d.feed(time, {'encoded' => '%23hash', 'value' => 1}) end - emits = d.events - assert_equal 1, emits.size - assert_equal 'decoded.message', emits[0][0] - assert_equal time, emits[0][1] - assert_equal '#hash', emits[0][2]['encoded'] + events = d.events + assert_equal 1, events.size + assert_equal 'decoded.message', events[0][0] + assert_equal time, events[0][1] + assert_equal '#hash', events[0][2]['encoded'] end def test_emit_with_multi_key_names @@ -88,12 +88,12 @@ def test_emit_with_multi_key_names d.feed(time, {'encoded' => '%23hash', 'another_encoded' => '%23another_hash'}) end - emits = d.events - assert_equal 1, emits.size - assert_equal 'decoded.message', emits[0][0] - assert_equal time, emits[0][1] - assert_equal '#hash', emits[0][2]['encoded'] - assert_equal '#another_hash', emits[0][2]['another_encoded'] + events = d.events + assert_equal 1, events.size + assert_equal 'decoded.message', events[0][0] + assert_equal time, events[0][1] + assert_equal '#hash', events[0][2]['encoded'] + assert_equal '#another_hash', events[0][2]['another_encoded'] end def test_multiple_emit @@ -103,17 +103,17 @@ def test_multiple_emit d.feed(time, {'encoded' => '%23hash', 'another_encoded' => '%23another_hash'}) end - emits = d.events - emits << d.events.flatten - assert_equal 2, emits.size - assert_equal 'decoded.message', emits[0][0] - assert_equal time, emits[0][1] - assert_equal '#hash', emits[0][2]['encoded'] - assert_equal '#another_hash', emits[0][2]['another_encoded'] - - assert_equal 'decoded.message', emits[1][0] - assert_equal time, emits[1][1] - assert_equal '#hash', emits[1][2]['encoded'] - assert_equal '#another_hash', emits[1][2]['another_encoded'] + events = d.events + events << d.events.flatten + assert_equal 2, events.size + assert_equal 'decoded.message', events[0][0] + assert_equal time, events[0][1] + assert_equal '#hash', events[0][2]['encoded'] + assert_equal '#another_hash', events[0][2]['another_encoded'] + + assert_equal 'decoded.message', events[1][0] + assert_equal time, events[1][1] + assert_equal '#hash', events[1][2]['encoded'] + assert_equal '#another_hash', events[1][2]['another_encoded'] end end