Skip to content

Commit

Permalink
Added target config option
Browse files Browse the repository at this point in the history 
Allows to specify into what field you want the CEF data to be stored.

Closes logstash-plugins#35
  • Loading branch information
@breml
breml committed May 12, 2017
1 parent 7836663 commit bfcf104
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 31 deletions.
46 changes: 32 additions & 14 deletions lib/logstash/codecs/cef.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
# the provided name is added, which contains the raw data.
config :raw_data_field, :validate => :string

# Specify into what field you want the CEF data to be stored.
config :target, :validate => :string

HEADER_FIELDS = ['cefVersion','deviceVendor','deviceProduct','deviceVersion','deviceEventClassId','name','severity']

# Translating and flattening the CEF extensions with known field names as documented in the Common Event Format whitepaper
Expand All @@ -97,12 +100,27 @@ def initialize(params={})
@delimiter = @delimiter.gsub("\\r", "\r").gsub("\\n", "\n")
@buffer = FileWatch::BufferedTokenizer.new(@delimiter)
end

@target_field = structured_field(target)
end

private
def store_header_field(event,field_name,field_data)
#Unescape pipes and backslash in header fields
event.set(field_name,field_data.gsub(/\\\|/, '|').gsub(/\\\\/, '\\')) unless field_data.nil?
event.set(@target_field + structured_field(field_name),field_data.gsub(/\\\|/, '|').gsub(/\\\\/, '\\')) unless field_data.nil?
end

private
def structured_field(field_name)
if field_name.nil?
''
else
if field_name[0] == '[' && field_name[-1] == ']'
field_name
else
'[' + field_name + ']'
end
end
end

public
Expand All @@ -118,7 +136,7 @@ def decode(data, &block)

def handle(data, &block)
event = LogStash::Event.new
event.set(raw_data_field, data) unless raw_data_field.nil?
event.set(@target_field + structured_field(raw_data_field), data) unless raw_data_field.nil?

# Strip any quotations at the start and end, flex connectors seem to send this
if data[0] == "\""
Expand Down Expand Up @@ -146,14 +164,14 @@ def handle(data, &block)
message = split_data[HEADER_FIELDS.size..-1].join('|')

# Try and parse out the syslog header if there is one
if event.get('cefVersion').include? ' '
split_cef_version= event.get('cefVersion').rpartition(' ')
event.set('syslog', split_cef_version[0])
event.set('cefVersion',split_cef_version[2])
if event.get(@target_field + structured_field('cefVersion')).include? ' '
split_cef_version= event.get(@target_field + structured_field('cefVersion')).rpartition(' ')
event.set(@target_field + structured_field('syslog'), split_cef_version[0])
event.set(@target_field + structured_field('cefVersion'), split_cef_version[2])
end

# Get rid of the CEF bit in the version
event.set('cefVersion', event.get('cefVersion').sub(/^CEF:/, ''))
event.set(@target_field + structured_field('cefVersion'), event.get(@target_field + structured_field('cefVersion')).sub(/^CEF:/, ''))

# Strip any whitespace from the message
if not message.nil? and message.include? '='
Expand Down Expand Up @@ -184,7 +202,7 @@ def handle(data, &block)
message = message.map {|s| k, v = s.split('***'); "#{MAPPINGS[k] || k }=#{v}"}
message = message.each_with_object({}) do |k|
key, value = k.split(/\s*=\s*/,2)
event.set(key, value)
event.set(@target_field + structured_field(key), value)
end
end

Expand Down Expand Up @@ -318,14 +336,14 @@ def handle_v1_fields(event, split_data)
message = split_data[DEPRECATED_HEADER_FIELDS.size..-1].join('|')

# Try and parse out the syslog header if there is one
if event.get('cef_version').include? ' '
split_cef_version= event.get('cef_version').rpartition(' ')
event.set('syslog', split_cef_version[0])
event.set('cef_version',split_cef_version[2])
if event.get(@target_field + structured_field('cef_version')).include? ' '
split_cef_version= event.get(@target_field + structured_field('cef_version')).rpartition(' ')
event.set(@target_field + structured_field('syslog'), split_cef_version[0])
event.set(@target_field + structured_field('cef_version'),split_cef_version[2])
end

# Get rid of the CEF bit in the version
event.set('cef_version', event.get('cef_version').sub(/^CEF:/, ''))
event.set(@target_field + structured_field('cef_version'), event.get(@target_field + structured_field('cef_version')).sub(/^CEF:/, ''))

# Strip any whitespace from the message
if not message.nil? and message.include? '='
Expand All @@ -343,7 +361,7 @@ def handle_v1_fields(event, split_data)
extensions[key] = value.gsub(/\\=/, '=').gsub(/\\\\/, '\\')
Hash[*message].each{ |k, v| extensions[k] = v }
# And save the new has as the extensions
event.set('cef_ext', extensions)
event.set(@target_field + structured_field('cef_ext'), extensions)
end

end
Expand Down
93 changes: 76 additions & 17 deletions spec/codecs/cef_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,13 +335,13 @@
context "#decode" do
let (:message) { "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232" }

def validate(e)
def validate(e, target = '')
insist { e.is_a?(LogStash::Event) }
insist { e.get('cefVersion') } == "0"
insist { e.get('deviceVersion') } == "1.0"
insist { e.get('deviceEventClassId') } == "100"
insist { e.get('name') } == "trojan successfully stopped"
insist { e.get('severity') } == "10"
insist { e.get(target + '[cefVersion]') } == "0"
insist { e.get(target + '[deviceVersion]') } == "1.0"
insist { e.get(target + '[deviceEventClassId]') } == "100"
insist { e.get(target + '[name]') } == "trojan successfully stopped"
insist { e.get(target + '[severity]') } == "10"
end

context "with delimiter set" do
Expand Down Expand Up @@ -567,6 +567,35 @@ def validate(e)
end
end

context "with target set to a field" do
subject(:codec) { LogStash::Codecs::CEF.new("target" => "cef") }

it "should create all fields within this target (cef)" do
subject.decode(message) do |e|
validate(e, '[cef]')
end
end
end

context "with target set to a field (with brackets)" do
subject(:codec) { LogStash::Codecs::CEF.new("target" => "[cef]") }

it "should create all fields within this target ([cef])" do
subject.decode(message) do |e|
validate(e, '[cef]')
end
end
end

context "with target set to a nested field" do
subject(:codec) { LogStash::Codecs::CEF.new("target" => "[cef][nested]") }

it "should create all fields within this target ([cef][nested])" do
subject.decode(message) do |e|
validate(e, '[cef][nested]')
end
end
end
end

context "decode with deprecated version option" do
Expand All @@ -579,18 +608,18 @@ def validate(e)

subject(:codec) { LogStash::Codecs::CEF.new(options) }

def validate(e)
def validate(e, target = '')
insist { e.is_a?(LogStash::Event) }
insist { e.get('cef_version') } == "0"
insist { e.get('cef_device_version') } == "1.0"
insist { e.get('cef_sigid') } == "100"
insist { e.get('cef_name') } == "trojan successfully stopped"
insist { e.get('cef_severity') } == "10"
insist { e.get('cefVersion') } == "0"
insist { e.get('deviceVersion') } == "1.0"
insist { e.get('deviceEventClassId') } == "100"
insist { e.get('name') } == "trojan successfully stopped"
insist { e.get('severity') } == "10"
insist { e.get(target + '[cef_version]') } == "0"
insist { e.get(target + '[cef_device_version]') } == "1.0"
insist { e.get(target + '[cef_sigid]') } == "100"
insist { e.get(target + '[cef_name]') } == "trojan successfully stopped"
insist { e.get(target + '[cef_severity]') } == "10"
insist { e.get(target + '[cefVersion]') } == "0"
insist { e.get(target + '[deviceVersion]') } == "1.0"
insist { e.get(target + '[deviceEventClassId]') } == "100"
insist { e.get(target + '[name]') } == "trojan successfully stopped"
insist { e.get(target + '[severity]') } == "10"
end

it "should parse the cef headers" do
Expand Down Expand Up @@ -740,6 +769,36 @@ def validate(e)
end
end
end

context "with target set to a field" do
subject(:codec) { LogStash::Codecs::CEF.new("deprecated_v1_fields" => true, "target" => "cef") }

it "should create all fields within this target (cef)" do
subject.decode(message) do |e|
validate(e, '[cef]')
end
end
end

context "with target set to a field (with brackets)" do
subject(:codec) { LogStash::Codecs::CEF.new("deprecated_v1_fields" => true, "target" => "[cef]") }

it "should create all fields within this target ([cef])" do
subject.decode(message) do |e|
validate(e, '[cef]')
end
end
end

context "with target set to a nested field" do
subject(:codec) { LogStash::Codecs::CEF.new("deprecated_v1_fields" => true, "target" => "[cef][nested]") }

it "should create all fields within this target ([cef][nested])" do
subject.decode(message) do |e|
validate(e, '[cef][nested]')
end
end
end
end

context "encode and decode" do
Expand Down

0 comments on commit bfcf104

Please sign in to comment.