diff --git a/README.md b/README.md index 6e37d90..a68dfd7 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,30 @@ Overview -======== - +============== **bluekrabsetw** is a C++ library that simplifies interacting with ETW. It allows for any number of traces and providers to be enabled and for client code to register for event notifications from these traces. **bluekrabsetw** originates from the **krabsetw** c++ library and seeks to improve and include capabilities that have not yet been included in the former library. **bluekrabsetw** also provides code to simplify parsing generic event data into strongly typed data types. **Threathunters.BlueKrabsetw.Native.ETW** is a C++ CLI (.NET) wrapper around **bluekrabsetw**. It provides the same functionality as bluekrabsetw to .NET applications. -Examples & Documentation -======== +> ### Additional Capabilities +> The following capabilities have been integrated into the solution alongside the original krabsetw C++ library: +> * **Provider Enhanced Runtime Capabilities**: +> * Supports enabling and disabling providers dynamically to adapt to changing requirements. This includes: Provider Addition, Removal, and Update Functionality +> +> * **Session Enhanced Runtime Capabilities**: +> * todo +> +> * **Decoupling of Functionality to Allow Better Control of Sessions**: +> * Provides improved modularity and flexibility, making it easier to manage and control Sessions. +> +> * **Improved Pre-Filtering Mechanisms**: +> * Optimizes data processing by allowing more efficient filtering before data is collected. +> +> These enhancements extend the core features of the original krabsetw C++ library, providing a more robust and flexible solution. +Examples & Documentation +============== * An [ETW Primer](docs/EtwPrimer.md). * Simple examples can be found in the `examples` folder. * Please refer to [KrabsExample.md](docs/KrabsExample.md) and [LobstersExample.md](docs/LobstersExample.md) for detailed examples. diff --git a/Threathunters.BlueKrabsetw.Native.ETW.NetCore/Microsoft.O365.Security.Native.ETW.NetCore.vcxproj b/Threathunters.BlueKrabsetw.Native.ETW.NetCore/Microsoft.O365.Security.Native.ETW.NetCore.vcxproj index c3ea46a..c6df4dc 100644 --- a/Threathunters.BlueKrabsetw.Native.ETW.NetCore/Microsoft.O365.Security.Native.ETW.NetCore.vcxproj +++ b/Threathunters.BlueKrabsetw.Native.ETW.NetCore/Microsoft.O365.Security.Native.ETW.NetCore.vcxproj @@ -212,10 +212,10 @@ - + diff --git a/Threathunters.BlueKrabsetw.Native.ETW/Filtering/DirectEventFilter.hpp b/Threathunters.BlueKrabsetw.Native.ETW/Filtering/DirectEventFilter.hpp deleted file mode 100644 index b0e6313..0000000 --- a/Threathunters.BlueKrabsetw.Native.ETW/Filtering/DirectEventFilter.hpp +++ /dev/null @@ -1,157 +0,0 @@ -#pragma once - -#include "../Conversions.hpp" -#include "../EventRecordError.hpp" -#include "../EventRecord.hpp" -#include "../EventRecordMetadata.hpp" -#include "../Guid.hpp" -#include "../IEventRecord.hpp" -#include "../IEventRecordError.hpp" -#include "../NativePtr.hpp" -#include "Predicate.hpp" - -using namespace System; -using namespace System::Collections::Generic; -using namespace System::Runtime::InteropServices; - -namespace Microsoft { - namespace O365 { - namespace Security { - namespace ETW { - - public interface class IDirectEventFilter - { - public: - virtual unsigned int GetEventType(); - virtual unsigned long GetSize(); - }; - - public ref class SystemFlagsEventFilter : IDirectEventFilter - { - public: - SystemFlagsEventFilter(unsigned long long flags, unsigned long size) - : flags_(flags), - type_(EVENT_FILTER_TYPE_SYSTEM_FLAGS), - size_(size) - { - } - - virtual unsigned int GetEventType() - { - return type_; - } - - virtual unsigned long GetSize() - { - return size_; - } - - unsigned long long GetFlag() - { - return flags_; - } - - - private: - unsigned long long flags_; - unsigned int type_; - unsigned long size_; - }; - - public ref class EventIdFilter : IDirectEventFilter - { - public: - EventIdFilter(IEnumerable^ ids) - : ids_(gcnew List(ids)), - type_(EVENT_FILTER_TYPE_EVENT_ID), - size_(0) - { - } - - EventIdFilter(... array^ ids) - : ids_(gcnew List(ids)), - type_(EVENT_FILTER_TYPE_EVENT_ID), - size_(0) - { - } - - virtual unsigned int GetEventType() - { - return type_; - } - - virtual unsigned long GetSize() - { - return size_; - } - - List^ GetList() - { - return ids_; - } - - private: - List^ ids_; - unsigned int type_; - unsigned long size_; - }; - - - public ref class DirectEventFilters - { - public: - DirectEventFilters(IEnumerable^ filters) - : directFilterList_(gcnew List(filters)), - filter_(new krabs::direct_event_filters()) - { - } - - DirectEventFilters(... array^ filters) - : directFilterList_(gcnew List(filters)), - filter_(new krabs::direct_event_filters()) - { - } - - internal: - operator krabs::direct_event_filters& () - { - - for each (auto filter in directFilterList_) - { - switch (filter->GetEventType()) { - case EVENT_FILTER_TYPE_SYSTEM_FLAGS: { - if (auto typeFilter = dynamic_cast(filter)) - { - auto p = std::make_shared(typeFilter->GetFlag(), typeFilter->GetSize()); - filter_->list_.emplace_back(p); - } - break; - } - case EVENT_FILTER_TYPE_EVENT_ID: { - if (auto typeFilter = dynamic_cast(filter)) - { - std::set tmp; - for each (auto l in typeFilter->GetList()) - { - tmp.insert(static_cast(l)); - } - auto p = std::make_shared(tmp, TRUE); - filter_->list_.emplace_back(p); - } - break; - } - default: { - - } - } - } - return *filter_; - } - - NativePtr filter_; - List^ directFilterList_; - }; - } - } - } -} \ No newline at end of file diff --git a/Threathunters.BlueKrabsetw.Native.ETW/Filtering/PreEventfilter.hpp b/Threathunters.BlueKrabsetw.Native.ETW/Filtering/PreEventfilter.hpp new file mode 100644 index 0000000..9779ba5 --- /dev/null +++ b/Threathunters.BlueKrabsetw.Native.ETW/Filtering/PreEventfilter.hpp @@ -0,0 +1,171 @@ +#pragma once + +#include "../Conversions.hpp" +#include "../EventRecordError.hpp" +#include "../EventRecord.hpp" +#include "../EventRecordMetadata.hpp" +#include "../Guid.hpp" +#include "../IEventRecord.hpp" +#include "../IEventRecordError.hpp" +#include "../NativePtr.hpp" +#include "Predicate.hpp" + +using namespace System; +using namespace System::Collections::Generic; +using namespace System::Runtime::InteropServices; + +namespace Microsoft { + namespace O365 { + namespace Security { + namespace ETW { + + public interface class IPrePredicate + { + public: + virtual EVENT_FILTER_DESCRIPTOR operator()(); + }; + + public ref class SystemFlags : public IPrePredicate + { + public: + SystemFlags(unsigned long long a1, unsigned long a2) + : data_(a1) + , size_(a2) + {} + + virtual EVENT_FILTER_DESCRIPTOR operator()() + { + auto native_filter = new krabs::system_flags(data_, size_); + + return native_filter->operator()(); + } + + private: + unsigned long long data_; + unsigned long size_; + }; + + public ref class EventIds : IPrePredicate + { + public: + EventIds(IEnumerable^ a1) + : data_(gcnew List(a1)) + , filter_in_(true) + {} + + /*EventIds(... array^ a1) + : data_(gcnew List(a1)) + , filter_in_(true) + {}*/ + + virtual EVENT_FILTER_DESCRIPTOR operator()() + { + std::set x; + for each (auto y in data_) + { + x.insert(static_cast(y)); + } + + auto native_filter = new krabs::event_ids(x, filter_in_); + + return native_filter->operator()(); + } + + private: + List^ data_; + bool filter_in_; + }; + + public ref class ProcessIds : IPrePredicate + { + public: + ProcessIds(IEnumerable^ a1) + : data_(gcnew List(a1)) + {} + + ProcessIds(... array^ a1) + : data_(gcnew List(a1)) + {} + + virtual EVENT_FILTER_DESCRIPTOR operator()() + { + std::set x; + for each (auto y in data_) + { + x.insert(static_cast(y)); + } + + auto native_filter = new krabs::event_ids(x, 0); + + return native_filter->operator()(); + } + + private: + List^ data_; + }; + + public ref class EventNames : IPrePredicate + { + public: + EventNames(bool a2, IEnumerable^ a1) + : data_(gcnew List(a1)) + , filter_in_(a2) + {} + + EventNames(bool a2, ... array^ a1) + : data_(gcnew List(a1)) + , filter_in_(a2) + {} + + virtual EVENT_FILTER_DESCRIPTOR operator()() + { + std::set x; + for each (auto y in data_) + { + x.insert(msclr::interop::marshal_as(y)); + } + + auto native_filter = new krabs::event_names(x, filter_in_); + + return native_filter->operator()(); + } + + private: + List^ data_; + bool filter_in_; + }; + + public ref class PreEventFilter + { + public: + PreEventFilter(IEnumerable^ filters) + : directFilterList_(gcnew List(filters)), + filter_(new krabs::pre_event_filter()) + {} + + PreEventFilter(... array^ filters) + : directFilterList_(gcnew List(filters)), + filter_(new krabs::pre_event_filter()) + {} + + internal: + operator krabs::pre_event_filter& () + { + auto count = 0; + for each (auto filter in directFilterList_) + { + filter_->descriptor_.descriptor[count++] = filter->operator()(); + } + + filter_->descriptor_.count = count; + return *filter_; + } + + NativePtr filter_; + List^ directFilterList_; + }; + } + } + } +} + diff --git a/Threathunters.BlueKrabsetw.Native.ETW/ITrace.hpp b/Threathunters.BlueKrabsetw.Native.ETW/ITrace.hpp index bdf3a9b..d2b3783 100644 --- a/Threathunters.BlueKrabsetw.Native.ETW/ITrace.hpp +++ b/Threathunters.BlueKrabsetw.Native.ETW/ITrace.hpp @@ -23,6 +23,16 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { /// void SetTraceProperties(EventTraceProperties^ properties); + /// + /// TODO + /// + void SetTraceFilename(String^ filename); + + /// + /// Starts listening for events from the enabled providers. + /// + void Open(); + /// /// Starts listening for events from the enabled providers. /// @@ -33,6 +43,16 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { /// void Stop(); + /// + /// Stops listening for events. + /// + void Close(); + + /// + /// Stops listening for events. + /// + void Update(); + /// /// Get stats about events handled by this trace. /// @@ -57,6 +77,18 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { /// /// The to enable. void Enable(RawProvider^ provider); + + /// + /// Enables a provider for the given user trace. + /// + /// The to enable. + void Disable(Provider^ provider); + + /// + /// Enables a raw provider for the given user trace. + /// + /// The to enable. + void Disable(RawProvider^ provider); }; /// diff --git a/Threathunters.BlueKrabsetw.Native.ETW/KernelTrace.hpp b/Threathunters.BlueKrabsetw.Native.ETW/KernelTrace.hpp index 9b19cfe..f97c296 100644 --- a/Threathunters.BlueKrabsetw.Native.ETW/KernelTrace.hpp +++ b/Threathunters.BlueKrabsetw.Native.ETW/KernelTrace.hpp @@ -92,6 +92,12 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { /// virtual void SetTraceProperties(EventTraceProperties^ properties); + /// + /// + /// + /// + virtual void SetTraceFilename(String^ filename); + /// /// Opens a trace session. /// @@ -152,6 +158,28 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { /// virtual void Stop(); + /// + /// Stops listening for events. + /// + /// + /// KernelTrace trace = new KernelTrace(); + /// // ... + /// trace.Start(); + /// trace.Stop(); + /// + virtual void Close(); + + /// + /// Stops listening for events. + /// + /// + /// KernelTrace trace = new KernelTrace(); + /// // ... + /// trace.Start(); + /// trace.Stop(); + /// + virtual void Update(); + /// /// Get stats about events handled by this trace /// @@ -230,6 +258,12 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { ExecuteAndConvertExceptions(return trace_->set_trace_properties(&_properties)); } + inline void KernelTrace::SetTraceFilename(String^ filename) + { + std::wstring nativeName = msclr::interop::marshal_as(filename); + ExecuteAndConvertExceptions(return trace_->set_trace_filename(nativeName)); + } + inline void KernelTrace::Open() { ExecuteAndConvertExceptions((void)trace_->open()); @@ -250,6 +284,16 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { ExecuteAndConvertExceptions(return trace_->stop()); } + inline void KernelTrace::Close () + { + ExecuteAndConvertExceptions(return trace_->close()); + } + + inline void KernelTrace::Update() + { + ExecuteAndConvertExceptions(return trace_->update()); + } + inline TraceStats KernelTrace::QueryStats() { ExecuteAndConvertExceptions(return TraceStats(trace_->query_stats())); diff --git a/Threathunters.BlueKrabsetw.Native.ETW/Microsoft.O365.Security.Native.ETW.vcxproj b/Threathunters.BlueKrabsetw.Native.ETW/Microsoft.O365.Security.Native.ETW.vcxproj index 7993801..123a5ce 100644 --- a/Threathunters.BlueKrabsetw.Native.ETW/Microsoft.O365.Security.Native.ETW.vcxproj +++ b/Threathunters.BlueKrabsetw.Native.ETW/Microsoft.O365.Security.Native.ETW.vcxproj @@ -207,10 +207,10 @@ - + diff --git a/Threathunters.BlueKrabsetw.Native.ETW/Microsoft.O365.Security.Native.ETW.vcxproj.filters b/Threathunters.BlueKrabsetw.Native.ETW/Microsoft.O365.Security.Native.ETW.vcxproj.filters index 54467e7..f1532d7 100644 --- a/Threathunters.BlueKrabsetw.Native.ETW/Microsoft.O365.Security.Native.ETW.vcxproj.filters +++ b/Threathunters.BlueKrabsetw.Native.ETW/Microsoft.O365.Security.Native.ETW.vcxproj.filters @@ -122,7 +122,7 @@ Header Files - + Header Files\Filtering diff --git a/Threathunters.BlueKrabsetw.Native.ETW/Provider.hpp b/Threathunters.BlueKrabsetw.Native.ETW/Provider.hpp index 29ea463..f10437c 100644 --- a/Threathunters.BlueKrabsetw.Native.ETW/Provider.hpp +++ b/Threathunters.BlueKrabsetw.Native.ETW/Provider.hpp @@ -10,7 +10,7 @@ #include "Guid.hpp" #include "NativePtr.hpp" #include "Filtering/EventFilter.hpp" -#include "Filtering/DirectEventFilter.hpp" +#include "Filtering/PreEventfilter.hpp" using namespace System; using namespace System::Runtime::InteropServices; @@ -18,7 +18,7 @@ using namespace System::Runtime::InteropServices; namespace Microsoft { namespace O365 { namespace Security { namespace ETW { ref class UserTrace; - + // Flags as documented here: // https://msdn.microsoft.com/en-us/library/windows/desktop/dd392306(v=vs.85).aspx public enum class TraceFlags @@ -173,11 +173,7 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { provider_->add_filter(filter); } - /// - /// Adds a new EventFilter to the provider. - /// - /// the to add - void AddFilter(O365::Security::ETW::DirectEventFilters^ filter) { + void AddFilter(O365::Security::ETW::PreEventFilter^ filter) { provider_->add_filter(filter); } diff --git a/Threathunters.BlueKrabsetw.Native.ETW/TraceStats.hpp b/Threathunters.BlueKrabsetw.Native.ETW/TraceStats.hpp index 7178676..8feb60c 100644 --- a/Threathunters.BlueKrabsetw.Native.ETW/TraceStats.hpp +++ b/Threathunters.BlueKrabsetw.Native.ETW/TraceStats.hpp @@ -34,15 +34,55 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { /// count of events lost initonly uint32_t EventsLost; + /// count of trace buffers + initonly uint32_t BuffersSize; + + /// count of free buffers + initonly uint32_t MinimumBuffers; + + /// count of buffers written + initonly uint32_t MaximumBuffers; + + /// count of buffers lost + initonly uint32_t MaximumFileSize; + + /// count of total events + initonly uint32_t LogFileMode; + + /// count of events handled + initonly uint32_t FlushTimer; + + /// count of events lost + initonly uint32_t EnableFlags; + + /// count of total events + initonly String^ LogFileName; + + /// count of events handled + initonly String^ LoggerName; + + /// count of events lost + initonly uint32_t FlushThreshold; + internal: TraceStats(const krabs::trace_stats& stats) - : BuffersCount(stats.buffersCount) - , BuffersFree(stats.buffersFree) - , BuffersWritten(stats.buffersWritten) - , BuffersLost(stats.buffersLost) - , EventsTotal(stats.eventsTotal) - , EventsHandled(stats.eventsHandled) - , EventsLost(stats.eventsLost) + : BuffersCount(stats.buffers_count) + , BuffersFree(stats.buffers_free) + , BuffersWritten(stats.buffers_written) + , BuffersLost(stats.buffers_lost) + , EventsTotal(stats.events_total) + , EventsHandled(stats.events_handled) + , EventsLost(stats.events_lost) + , BuffersSize(stats.buffer_size) + , MinimumBuffers(stats.minimum_buffers) + , MaximumBuffers(stats.maximum_buffers) + , MaximumFileSize(stats.maximum_file_size) + , LogFileMode(stats.log_file_mode) + , FlushTimer(stats.flush_timer) + , EnableFlags(stats.enable_flags) + , LogFileName(msclr::interop::marshal_as(stats.log_file_name)) + , LoggerName(msclr::interop::marshal_as(stats.logger_name)) + , FlushThreshold(stats.flush_threshold) { } }; diff --git a/Threathunters.BlueKrabsetw.Native.ETW/UserTrace.hpp b/Threathunters.BlueKrabsetw.Native.ETW/UserTrace.hpp index 82a8d8f..213b747 100644 --- a/Threathunters.BlueKrabsetw.Native.ETW/UserTrace.hpp +++ b/Threathunters.BlueKrabsetw.Native.ETW/UserTrace.hpp @@ -73,6 +73,30 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { /// virtual void Enable(O365::Security::ETW::RawProvider ^provider); + /// + /// Enables a provider for the given user trace. + /// + /// the to enable on the trace + /// + /// UserTrace trace = new UserTrace(); + /// System.Guid powershell = System.Guid.Parse("{...}") + /// Provider provider = new Provider(powershell); + /// trace.Enable(provider); + /// + virtual void Disable(O365::Security::ETW::Provider^ provider); + + /// + /// Enables a raw provider for the given user trace. + /// + /// the to enable on the trace + /// + /// UserTrace trace = new UserTrace(); + /// System.Guid powershell = System.Guid.Parse("{...}") + /// Provider provider = new RawProvider(powershell); + /// trace.Enable(provider); + /// + virtual void Disable(O365::Security::ETW::RawProvider^ provider); + /// /// Sets the trace properties for a session. /// Must be called before Open()/Start(). @@ -100,6 +124,12 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { /// virtual void SetTraceProperties(EventTraceProperties ^properties); + /// + /// + /// + /// + virtual void SetTraceFilename(String^ filename); + /// /// Opens a trace session. /// @@ -160,6 +190,28 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { /// virtual void Stop(); + /// + /// Stops listening for events. + /// + /// + /// UserTrace trace = new UserTrace(); + /// // ... + /// trace.Start(); + /// trace.Stop(); + /// + virtual void Close(); + + /// + /// Stops listening for events. + /// + /// + /// UserTrace trace = new UserTrace(); + /// // ... + /// trace.Start(); + /// trace.Stop(); + /// + virtual void Update(); + /// /// Get stats about events handled by this trace /// @@ -206,6 +258,16 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { return trace_->enable(*provider->provider_); } + inline void UserTrace::Disable(O365::Security::ETW::Provider^ provider) + { + return trace_->disable(*provider->provider_); + } + + inline void UserTrace::Disable(O365::Security::ETW::RawProvider^ provider) + { + return trace_->disable(*provider->provider_); + } + inline void UserTrace::SetTraceProperties(EventTraceProperties ^properties) { EVENT_TRACE_PROPERTIES _properties; @@ -217,6 +279,12 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { ExecuteAndConvertExceptions(return trace_->set_trace_properties(&_properties)); } + inline void UserTrace::SetTraceFilename(String^ filename) + { + std::wstring nativeName = msclr::interop::marshal_as(filename); + ExecuteAndConvertExceptions(return trace_->set_trace_filename(nativeName)); + } + inline void UserTrace::Open() { ExecuteAndConvertExceptions((void)trace_->open()); @@ -237,6 +305,16 @@ namespace Microsoft { namespace O365 { namespace Security { namespace ETW { ExecuteAndConvertExceptions(return trace_->stop()); } + inline void UserTrace::Close() + { + ExecuteAndConvertExceptions(return trace_->close()); + } + + inline void UserTrace::Update() + { + ExecuteAndConvertExceptions(return trace_->update()); + } + inline TraceStats UserTrace::QueryStats() { ExecuteAndConvertExceptions(return TraceStats(trace_->query_stats())); diff --git a/bluekrabs/bluekrabs.sln b/bluekrabs/bluekrabs.sln index 3228316..2982981 100644 --- a/bluekrabs/bluekrabs.sln +++ b/bluekrabs/bluekrabs.sln @@ -20,19 +20,33 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution MTA.testsettings = MTA.testsettings EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "krabs headers", "krabs headers", "{1FD19105-D67C-492B-B98F-53E00A324269}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EtwTestsCS", "..\tests\ManagedETWTests\EtwTestsCS.csproj", "{600CFE03-FD84-4323-9439-839D81C31972}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{C4AB7F5F-2FB3-4C16-A1F3-F6700C655B02}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Examples", "Examples", "{2E00634C-7E8B-4656-9505-78FF2F5D0EDD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ManagedExamples", "..\examples\ManagedExamples\ManagedExamples.csproj", "{32E71DD0-D11A-44DE-8CA8-572995AF2373}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "NativeExamples", "..\examples\NativeExamples\NativeExamples.vcxproj", "{D31B1A4B-8282-4AED-99FC-9AA5974B9134}" + ProjectSection(ProjectDependencies) = postProject + {ED4E6027-541F-440A-A5EE-15DBB7B89423} = {ED4E6027-541F-440A-A5EE-15DBB7B89423} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Threathunters.BlueKrabsetw.Native.ETW.NetCore", "..\Threathunters.BlueKrabsetw.Native.ETW.NetCore\Microsoft.O365.Security.Native.ETW.NetCore.vcxproj", "{9DE6788C-5759-4A75-B484-ABA4C7EF5F08}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "bluekrabs headers", "bluekrabs headers", "{60104949-C1E9-45B1-A37A-2A0EE5F94E1B}" ProjectSection(SolutionItems) = preProject krabs.hpp = krabs.hpp EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "krabs", "krabs", "{371361C8-96EC-4D6D-B80B-2E47E3453264}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "bluekrabs", "bluekrabs", "{6345EFAF-43BD-42DE-9B8D-8D955E0C1FDC}" ProjectSection(SolutionItems) = preProject krabs\client.hpp = krabs\client.hpp krabs\collection_view.hpp = krabs\collection_view.hpp krabs\compiler_check.hpp = krabs\compiler_check.hpp krabs\errors.hpp = krabs\errors.hpp krabs\etw.hpp = krabs\etw.hpp - krabs\helping\file_mapping.hpp = krabs\helping\file_mapping.hpp krabs\guid.hpp = krabs\guid.hpp krabs\kernel_guids.hpp = krabs\kernel_guids.hpp krabs\kernel_providers.hpp = krabs\kernel_providers.hpp @@ -53,52 +67,40 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "krabs", "krabs", "{371361C8 krabs\wstring_convert.hpp = krabs\wstring_convert.hpp EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "filtering", "filtering", "{96FA58B5-A1F6-4107-9FB4-226290F9D696}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "filtering", "filtering", "{DE5DA5AA-BA9F-4D07-AF37-E1CE1487217C}" ProjectSection(SolutionItems) = preProject krabs\filtering\comparers.hpp = krabs\filtering\comparers.hpp - krabs\filtering\direct_event_filter.hpp = krabs\filtering\direct_event_filter.hpp krabs\filtering\event_filter.hpp = krabs\filtering\event_filter.hpp + krabs\filtering\post_event_filter.hpp = krabs\filtering\post_event_filter.hpp krabs\filtering\predicates.hpp = krabs\filtering\predicates.hpp + krabs\filtering\pre_event_filter.hpp = krabs\filtering\pre_event_filter.hpp krabs\filtering\view_adapters.hpp = krabs\filtering\view_adapters.hpp EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "testing", "testing", "{9ED1AE76-2EAA-4CCF-8F01-458BDC8BCD53}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "helping", "helping", "{531AF202-7C02-4203-9463-F10DE8A996AD}" ProjectSection(SolutionItems) = preProject - krabs\testing\event_filter_proxy.hpp = krabs\testing\event_filter_proxy.hpp - krabs\testing\extended_data_builder.hpp = krabs\testing\extended_data_builder.hpp - krabs\testing\filler.hpp = krabs\testing\filler.hpp - krabs\testing\proxy.hpp = krabs\testing\proxy.hpp - krabs\testing\record_builder.hpp = krabs\testing\record_builder.hpp - krabs\testing\record_property_thunk.hpp = krabs\testing\record_property_thunk.hpp - krabs\testing\synth_record.hpp = krabs\testing\synth_record.hpp - EndProjectSection -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EtwTestsCS", "..\tests\ManagedETWTests\EtwTestsCS.csproj", "{600CFE03-FD84-4323-9439-839D81C31972}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{C4AB7F5F-2FB3-4C16-A1F3-F6700C655B02}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Examples", "Examples", "{2E00634C-7E8B-4656-9505-78FF2F5D0EDD}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ManagedExamples", "..\examples\ManagedExamples\ManagedExamples.csproj", "{32E71DD0-D11A-44DE-8CA8-572995AF2373}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "NativeExamples", "..\examples\NativeExamples\NativeExamples.vcxproj", "{D31B1A4B-8282-4AED-99FC-9AA5974B9134}" - ProjectSection(ProjectDependencies) = postProject - {ED4E6027-541F-440A-A5EE-15DBB7B89423} = {ED4E6027-541F-440A-A5EE-15DBB7B89423} + krabs\helping\file_mapping.hpp = krabs\helping\file_mapping.hpp EndProjectSection EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Threathunters.BlueKrabsetw.Native.ETW.NetCore", "..\Threathunters.BlueKrabsetw.Native.ETW.NetCore\Microsoft.O365.Security.Native.ETW.NetCore.vcxproj", "{9DE6788C-5759-4A75-B484-ABA4C7EF5F08}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "searching", "searching", "{03BCA2C2-F363-45B0-8745-E193BFBD54C5}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "searching", "searching", "{2C63BA17-1E15-4B5B-B979-A00C3A331678}" ProjectSection(SolutionItems) = preProject discover.hpp = discover.hpp krabs\searching\wevt.hpp = krabs\searching\wevt.hpp EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "helping", "helping", "{40378D85-BA5E-4FA1-A450-44BFA680B203}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "testing", "testing", "{9EBF97B2-137A-42F1-830F-E644C9590C33}" ProjectSection(SolutionItems) = preProject - krabs\helping\file_mapping.hpp = krabs\helping\file_mapping.hpp + krabs\testing\event_filter_proxy.hpp = krabs\testing\event_filter_proxy.hpp + krabs\testing\extended_data_builder.hpp = krabs\testing\extended_data_builder.hpp + krabs\testing\filler.hpp = krabs\testing\filler.hpp + krabs\testing\proxy.hpp = krabs\testing\proxy.hpp + krabs\testing\record_builder.hpp = krabs\testing\record_builder.hpp + krabs\testing\record_property_thunk.hpp = krabs\testing\record_property_thunk.hpp + krabs\testing\synth_record.hpp = krabs\testing\synth_record.hpp EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "proxy", "proxy", "{E157A8E6-C44F-4D87-AA59-5D9F0A78820B}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -161,14 +163,15 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {880977B8-15CA-421B-BF48-D01626A530A2} = {C4AB7F5F-2FB3-4C16-A1F3-F6700C655B02} - {371361C8-96EC-4D6D-B80B-2E47E3453264} = {1FD19105-D67C-492B-B98F-53E00A324269} - {96FA58B5-A1F6-4107-9FB4-226290F9D696} = {371361C8-96EC-4D6D-B80B-2E47E3453264} - {9ED1AE76-2EAA-4CCF-8F01-458BDC8BCD53} = {371361C8-96EC-4D6D-B80B-2E47E3453264} {600CFE03-FD84-4323-9439-839D81C31972} = {C4AB7F5F-2FB3-4C16-A1F3-F6700C655B02} {32E71DD0-D11A-44DE-8CA8-572995AF2373} = {2E00634C-7E8B-4656-9505-78FF2F5D0EDD} {D31B1A4B-8282-4AED-99FC-9AA5974B9134} = {2E00634C-7E8B-4656-9505-78FF2F5D0EDD} - {03BCA2C2-F363-45B0-8745-E193BFBD54C5} = {371361C8-96EC-4D6D-B80B-2E47E3453264} - {40378D85-BA5E-4FA1-A450-44BFA680B203} = {371361C8-96EC-4D6D-B80B-2E47E3453264} + {6345EFAF-43BD-42DE-9B8D-8D955E0C1FDC} = {60104949-C1E9-45B1-A37A-2A0EE5F94E1B} + {DE5DA5AA-BA9F-4D07-AF37-E1CE1487217C} = {6345EFAF-43BD-42DE-9B8D-8D955E0C1FDC} + {531AF202-7C02-4203-9463-F10DE8A996AD} = {6345EFAF-43BD-42DE-9B8D-8D955E0C1FDC} + {2C63BA17-1E15-4B5B-B979-A00C3A331678} = {6345EFAF-43BD-42DE-9B8D-8D955E0C1FDC} + {9EBF97B2-137A-42F1-830F-E644C9590C33} = {6345EFAF-43BD-42DE-9B8D-8D955E0C1FDC} + {E157A8E6-C44F-4D87-AA59-5D9F0A78820B} = {6345EFAF-43BD-42DE-9B8D-8D955E0C1FDC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {82BAA012-2EF9-4303-A429-CDA3655D5009} diff --git a/bluekrabs/krabs.hpp b/bluekrabs/krabs.hpp index 1e1ae51..c8db64b 100644 --- a/bluekrabs/krabs.hpp +++ b/bluekrabs/krabs.hpp @@ -57,5 +57,6 @@ #include "krabs/filtering/comparers.hpp" #include "krabs/filtering/predicates.hpp" #include "krabs/filtering/event_filter.hpp" +//#include "krabs/filtering/pre_event_filter.hpp" -#pragma warning(pop) +#pragma warning(pop) \ No newline at end of file diff --git a/bluekrabs/krabs/etw.hpp b/bluekrabs/krabs/etw.hpp index 3bccca7..46960d4 100644 --- a/bluekrabs/krabs/etw.hpp +++ b/bluekrabs/krabs/etw.hpp @@ -70,6 +70,15 @@ namespace krabs { namespace details { */ void stop(); + /** + * + * Close the ETW trace identified by the info in the trace type. + * In conjunction with stopping, closing detaches a consumer + * from an active trace session but does not terminate the session. + * + */ + void close(); + /** * * Opens the ETW trace identified by the info in the trace type. @@ -85,19 +94,43 @@ namespace krabs { namespace details { */ void process(); + /** + * + * Starts processing the ETW trace identified by the info in the trace type. + * open() needs to called for this to work first. + * + */ + void disable(const typename T::trace_type::provider_type& p); + + /** + * + * Starts processing the ETW trace identified by the info in the trace type. + * open() needs to called for this to work first. + * + */ + void enable(const typename T::trace_type::provider_type& p); + + /** + * + * Starts processing the ETW trace identified by the info in the trace type. + * open() needs to called for this to work first. + * + */ + void update(); + /** * * Queries the ETW trace identified by the info in the trace type. * */ - EVENT_TRACE_PROPERTIES query(); + trace_info query(); /** * * Queries the ETW trace identified by the info in the trace type v2. * */ - EVENT_TRACE_PROPERTIES_V2 query_v2(); + trace_info_v2 query_v2(); /** * @@ -125,17 +158,21 @@ namespace krabs { namespace details { void on_event(const EVENT_RECORD &record); private: + trace_info fill_trace_update_info(); trace_info fill_trace_info(); trace_info_v2 fill_trace_info_v2(); - EVENT_TRACE_LOGFILE fill_logfile(); - void close_trace(); + EVENT_TRACE_LOGFILE fill_logfile(); void register_trace(); - EVENT_TRACE_PROPERTIES query_trace(); - EVENT_TRACE_PROPERTIES_V2 query_trace_v2(); void stop_trace(); EVENT_TRACE_LOGFILE open_trace(); + void close_trace(); + void update_trace(); + trace_info query_trace(); + trace_info_v2 query_trace_v2(); void process_trace(); void enable_providers(); + void disable_provider(const typename T::trace_type::provider_type& p); + void enable_provider(const typename T::trace_type::provider_type& p); private: T &trace_; @@ -156,7 +193,8 @@ namespace krabs { namespace details { * */ template - static void __stdcall trace_callback_thunk(EVENT_RECORD *pRecord) + static void __stdcall trace_callback_thunk( + EVENT_RECORD *pRecord) { auto *pUserTrace = (T*)(pRecord->UserContext); trace_manager trace(*pUserTrace); @@ -176,7 +214,8 @@ namespace krabs { namespace details { * */ template - static ULONG __stdcall trace_buffer_callback(EVENT_TRACE_LOGFILE *pLogFile) + static ULONG __stdcall trace_buffer_callback( + EVENT_TRACE_LOGFILE *pLogFile) { auto *pTrace = (T*)(pLogFile->Context); trace_manager trace(*pTrace); @@ -215,13 +254,45 @@ namespace krabs { namespace details { } template - EVENT_TRACE_PROPERTIES trace_manager::query() + void trace_manager::update() + { + if (trace_.sessionHandle_ == INVALID_PROCESSTRACE_HANDLE) { + throw open_trace_failure(); + } + + update_trace(); + } + + template + void trace_manager::enable( + const typename T::trace_type::provider_type& p) + { + if (trace_.sessionHandle_ == INVALID_PROCESSTRACE_HANDLE) { + throw open_trace_failure(); + } + + enable_provider(p); + } + + template + void trace_manager::disable( + const typename T::trace_type::provider_type& p) + { + if (trace_.sessionHandle_ == INVALID_PROCESSTRACE_HANDLE) { + throw open_trace_failure(); + } + + disable_provider(p); + } + + template + trace_info trace_manager::query() { return query_trace(); } template - EVENT_TRACE_PROPERTIES_V2 trace_manager::query_v2() + trace_info_v2 trace_manager::query_v2() { return query_trace(); } @@ -248,6 +319,12 @@ namespace krabs { namespace details { close_trace(); } + template + void trace_manager::close() + { + close_trace(); + } + template void trace_manager::set_buffers_processed(size_t processed) { @@ -354,9 +431,75 @@ namespace krabs { namespace details { } template - EVENT_TRACE_PROPERTIES trace_manager::query_trace() + trace_info trace_manager::fill_trace_update_info() { - trace_info info = fill_trace_info(); + trace_info info = query_trace(); + + /* + EnableFlags: Set this member to 0 to disable all kernel providers. + Otherwise, you must specify the kernel providers that you want to + enable or keep enabled. Applies only to system logger sessions. + + FlushTimer: Set this member if you want to change the time to wait + before flushing buffers. If this member is 0, the member is not + updated. + + LogFileNameOffset: Set this member if you want to switch to another + log file. If this member is 0, the file name is not updated. If the + offset is not zero and you do not change the log file name, the + function returns an error. + + LogFileMode: Set this member if you want to turn + EVENT_TRACE_REAL_TIME_MODE on and off. To turn real time consuming + off, set this member to 0. To turn real time consuming on, set + this member to EVENT_TRACE_REAL_TIME_MODE and it will be OR'd with + the current modes. + + MaximumBuffers: Set this member if you want to change the maximum + number of buffers that ETW uses. If this member is 0, the member + is not updated. + */ + if (auto enable_flags = T::trace_type::construct_enable_flags(trace_)) { + info.properties.EnableFlags = enable_flags; + } + + if (trace_.properties_.FlushTimer) { + info.properties.FlushTimer = trace_.properties_.FlushTimer; + } + + if (trace_.properties_.LogFileMode) { + info.properties.LogFileMode = trace_.properties_.LogFileMode; + } + + if (trace_.properties_.MaximumBuffers) { + info.properties.MaximumBuffers = trace_.properties_.MaximumBuffers; + } + + return info; + } + + template + void trace_manager::update_trace() + { + auto info = fill_trace_update_info(); + + ULONG status = ControlTrace( + NULL, + trace_.name_.c_str(), + &info.properties, + EVENT_TRACE_CONTROL_UPDATE); + + if (status != ERROR_WMI_INSTANCE_NOT_FOUND) { + error_check_common_conditions(status); + } + } + + template + trace_info trace_manager::query_trace() + { + //trace_info info = fill_trace_info(); + trace_info info = {}; + info.properties.Wnode.BufferSize = sizeof(trace_info); ULONG status = ControlTrace( NULL, @@ -367,15 +510,15 @@ namespace krabs { namespace details { if (status != ERROR_WMI_INSTANCE_NOT_FOUND) { error_check_common_conditions(status); - - return info.properties; + //return info.properties; + return info; } return { }; } template - EVENT_TRACE_PROPERTIES_V2 trace_manager::query_trace_v2() + trace_info_v2 trace_manager::query_trace_v2() { if (IsWindowsVersionOrGreater(10, 0, 15063)) { error_check_common_conditions(ERROR_NOT_SUPPORTED); @@ -383,20 +526,20 @@ namespace krabs { namespace details { return { }; } - trace_info_v2 info = fill_trace_info_v2(); + trace_info_v2 info = {}; - ULONG status = ControlTrace( - NULL, - trace_.name_.c_str(), - //info, - &info.properties, - EVENT_TRACE_CONTROL_QUERY); + //ULONG status = ControlTrace( + // NULL, + // trace_.name_.c_str(), + // //info, + // &info.properties, + // EVENT_TRACE_CONTROL_QUERY); - if (status != ERROR_WMI_INSTANCE_NOT_FOUND) { - error_check_common_conditions(status); - - return info.properties; - } + //if (status != ERROR_WMI_INSTANCE_NOT_FOUND) { + // error_check_common_conditions(status); + // //return info.properties; + // return info; + //} return { }; } @@ -507,4 +650,26 @@ namespace krabs { namespace details { { T::trace_type::enable_providers(trace_); } + + template + void trace_manager::disable_provider( + const typename T::trace_type::provider_type& p) + { + T::trace_type::disable_provider(trace_, p); + } + + template + void trace_manager::enable_provider( + const typename T::trace_type::provider_type& p) + { + if (trace_.registrationHandle_ == INVALID_PROCESSTRACE_HANDLE) { + trace_.properties_ = query_trace().properties; + trace_.registrationHandle_ = trace_.properties_.Wnode.HistoricalContext; + } + + if (trace_.registrationHandle_ != INVALID_PROCESSTRACE_HANDLE) { + T::trace_type::enable_provider(trace_, p); + } + } + } /* namespace details */ } /* namespace krabs */ diff --git a/bluekrabs/krabs/filter_descriptor.hpp b/bluekrabs/krabs/filter_descriptor.hpp new file mode 100644 index 0000000..007ed4e --- /dev/null +++ b/bluekrabs/krabs/filter_descriptor.hpp @@ -0,0 +1,111 @@ + +#pragma once + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + +#include +#include +//#include + +#include + +#include "filtering/direct_event_filter.hpp" + + +#pragma comment(lib, "tdh.lib") + +namespace krabs { namespace details { + + template + class filter_descriptor { + public: + filter_descriptor(T& trace); + + void set_event_payload(); + void set_event_id(const event_id_event_filter& direct_filter); + void set_event_pid(); + void set_event_name(); + void set_system_flags(const system_flags_event_filter& direct_filter); + + private: + ULONG filter_descriptor_count_ = 0; + EVENT_FILTER_DESCRIPTOR filter_desc_[MAX_EVENT_FILTERS_COUNT] = { 0 }; + std::unique_ptr id_cache_; + std::unique_ptr pid_cache_; + std::unique_ptr exe_name_cache_; + std::unique_ptr event_name_cache_; + unsigned int pids_cache_[MAX_EVENT_FILTER_PID_COUNT] = { 0 }; + PAYLOAD_FILTER_PREDICATE predicates_cache_[MAX_PAYLOAD_PREDICATES] = { 0 }; + + private: + //T& trace_; + + private: + template + friend class krabs::trace; + }; + + + // Implementation + // ------------------------------------------------------------------------ + + template + filter_descriptor::filter_descriptor(T& trace) + : trace_(trace) + {} + + template + void filter_descriptor::set_event_payload() { + + } + + template + void filter_descriptor::set_event_id(const event_id_event_filter& direct_filter) + { + /*typedef struct _EVENT_FILTER_EVENT_ID { + BOOLEAN FilterIn; + UCHAR Reserved; + USHORT Count; + USHORT Events[ANYSIZE_ARRAY]; + } EVENT_FILTER_EVENT_ID, * PEVENT_FILTER_EVENT_ID;*/ + + auto& filter_desc = filter_desc_[filter_descriptor_count_++]; + auto count = direct_filter.get_data().size(); + if (count > 0) { + auto cache_size = FIELD_OFFSET(EVENT_FILTER_EVENT_ID, Events[count]); + id_cache_ = std::make_unique(cache_size); + auto event_id_desc = reinterpret_cast(id_cache_.get()); + event_id_desc->FilterIn = TRUE; + event_id_desc->Count = static_cast(event_ids_count); + + auto i = 0; + for (auto event_id : direct_filter.get_data()) { + event_id_desc->Events[i++] = event_id; + } + + filter_desc.Type = EVENT_FILTER_TYPE_EVENT_ID; + filter_desc.Ptr = reinterpret_cast(event_id_desc); + filter_desc.Size = cache_size; + } + } + + template + void filter_descriptor::set_event_pid() { + + } + template + void filter_descriptor::set_event_name() { + + } + + template + void filter_descriptor::set_system_flags(const system_flags_event_filter& direct_filter) + { + auto& filter_desc = filter_desc_[filter_descriptor_count_++]; + filter_desc.Ptr = direct_filter.get_value(); + filter_desc.Size = direct_filter.get_size(); + filter_desc.Type = EVENT_FILTER_TYPE_SYSTEM_FLAGS; + } +} /* namespace details */ } /* namespace krabs */ \ No newline at end of file diff --git a/bluekrabs/krabs/filtering/direct_event_filter.hpp b/bluekrabs/krabs/filtering/direct_event_filter.hpp index 5b6c361..ec6cd3c 100644 --- a/bluekrabs/krabs/filtering/direct_event_filter.hpp +++ b/bluekrabs/krabs/filtering/direct_event_filter.hpp @@ -1,6 +1,6 @@ #include #include - +#include namespace krabs { @@ -63,6 +63,283 @@ namespace krabs { unsigned int type_; unsigned long size_; };*/ + + struct base_descriptor { + base_descriptor(unsigned int a1) + : type_(a1) + {} + + virtual EVENT_FILTER_DESCRIPTOR operator()() const = 0; + + unsigned int type_; + }; + + struct system_flags_descriptor : base_descriptor { + system_flags_descriptor(unsigned long long a1, unsigned long a2) + : base_descriptor(EVENT_FILTER_TYPE_SYSTEM_FLAGS) + , descriptor_({ 0 }) + , data_(a1) + , size_(a2) + {} + + EVENT_FILTER_DESCRIPTOR operator()() const override + { + descriptor_.Ptr = reinterpret_cast(&data_); + descriptor_.Size = size_; + descriptor_.Type = type_; + + return descriptor_; + } + + private: + mutable unsigned long long data_; + unsigned long size_; + mutable EVENT_FILTER_DESCRIPTOR descriptor_; + }; + + struct event_id_descriptor : base_descriptor { + event_id_descriptor(std::set a1, bool a2) + : base_descriptor(EVENT_FILTER_TYPE_EVENT_ID) + , descriptor_({ 0 }) + , data_(a1) + , filter_in_(a2) + {} + + EVENT_FILTER_DESCRIPTOR operator()() const override + { + /*typedef struct _EVENT_FILTER_EVENT_ID { + BOOLEAN FilterIn; + UCHAR Reserved; + USHORT Count; + USHORT Events[ANYSIZE_ARRAY]; + } EVENT_FILTER_EVENT_ID, * PEVENT_FILTER_EVENT_ID;*/ + + auto count = data_.size(); + if (count > 0) { + auto cache_size = FIELD_OFFSET(EVENT_FILTER_EVENT_ID, Events[count]); + cache_ = std::make_unique(cache_size); + auto tmp = reinterpret_cast(cache_.get()); + tmp->FilterIn = filter_in_; + tmp->Count = static_cast(count); + int i = 0; + for (auto item : data_) { + tmp->Events[i++] = item; + } + descriptor_.Ptr = reinterpret_cast(cache_.get()); + descriptor_.Size = cache_size; + descriptor_.Type = type_; + } + + return descriptor_; + } + + private: + std::set data_; + bool filter_in_; + mutable EVENT_FILTER_DESCRIPTOR descriptor_; + mutable std::unique_ptr cache_; + }; + + struct pid_descriptor : base_descriptor { + pid_descriptor(std::set a1) + : base_descriptor(EVENT_FILTER_TYPE_PID) + , descriptor_({ 0 }) + , data_(a1) + {} + + EVENT_FILTER_DESCRIPTOR operator()() const override + { + /*typedef struct _EVENT_FILTER_EVENT_ID { + BOOLEAN FilterIn; + UCHAR Reserved; + USHORT Count; + USHORT Events[ANYSIZE_ARRAY]; + } EVENT_FILTER_EVENT_ID, * PEVENT_FILTER_EVENT_ID;*/ + + auto count = data_.size(); + if (count > 0) { + int i = 0; + for (auto item : data_) { + if (i < MAX_EVENT_FILTER_PID_COUNT) { + cache_[i++] = item; + } + } + descriptor_.Ptr = reinterpret_cast(cache_); + descriptor_.Size = sizeof(unsigned int) * i; + descriptor_.Type = type_; + } + + return descriptor_; + } + + private: + std::set data_; + mutable EVENT_FILTER_DESCRIPTOR descriptor_; + mutable unsigned int cache_[MAX_EVENT_FILTER_PID_COUNT] = { 0 }; + }; + + struct event_name_descriptor : base_descriptor { + event_name_descriptor(std::set a1, bool a2) + : base_descriptor(EVENT_FILTER_TYPE_EVENT_NAME) + , descriptor_({ 0 }) + , data_(a1) + , filter_in_(a2) + {} + + EVENT_FILTER_DESCRIPTOR operator()() const override + { + /*typedef struct _EVENT_FILTER_EVENT_NAME { + ULONGLONG MatchAnyKeyword; + ULONGLONG MatchAllKeyword; + UCHAR Level; + BOOLEAN FilterIn; + USHORT NameCount; + UCHAR Names[ANYSIZE_ARRAY]; + } EVENT_FILTER_EVENT_NAME, * PEVENT_FILTER_EVENT_NAME;*/ + + auto count = data_.size(); + if (count > 0) { + auto cache_size = FIELD_OFFSET(EVENT_FILTER_EVENT_NAME, Names[count]); + cache_ = std::make_unique(cache_size); + auto tmp = reinterpret_cast(cache_.get()); + tmp->FilterIn = filter_in_; + tmp->Level = 0; + tmp->MatchAnyKeyword = 0; + tmp->MatchAllKeyword = 0; + tmp->NameCount = static_cast(count); + // The Names field should be a series of + // NameCount null terminated utf-8 + // event names. + auto i = 0; + for (auto item1 : data_) { + item1.push_back('\0'); + for (auto& item2 : item1) { + tmp->Names[i++] = item2; + } + } + + descriptor_.Ptr = reinterpret_cast(cache_.get()); + descriptor_.Size = cache_size; + descriptor_.Type = type_; + } + + return descriptor_; + } + + private: + std::set data_; + bool filter_in_; + mutable EVENT_FILTER_DESCRIPTOR descriptor_; + mutable std::unique_ptr cache_; + }; + + struct payload_descriptor : base_descriptor { + payload_descriptor(std::set a1, bool a2) + : base_descriptor(EVENT_FILTER_TYPE_PAYLOAD) + , descriptor_({ 0 }) + , data_(a1) + , filter_in_(a2) + {} + + EVENT_FILTER_DESCRIPTOR operator()() const override + { + /*typedef struct _PAYLOAD_FILTER_PREDICATE { + LPWSTR FieldName; + USHORT CompareOp; + LPWSTR Value; + } PAYLOAD_FILTER_PREDICATE, *PPAYLOAD_FILTER_PREDICATE;*/ + + + + return descriptor_; + } + + private: + std::set data_; + bool filter_in_; + mutable EVENT_FILTER_DESCRIPTOR descriptor_; + mutable PAYLOAD_FILTER_PREDICATE cache_[MAX_PAYLOAD_PREDICATES] = { 0 }; + //mutable std::unique_ptr cache_; + }; + + struct descriptor_info { + unsigned long count; + EVENT_FILTER_DESCRIPTOR descriptor[MAX_EVENT_FILTERS_COUNT]; + }; + + /** + * + * Accepts an event if any of the predicates in the vector matches + * + */ + struct direct_event_filters1 { + direct_event_filters1(std::vector list) + : list_(list) + , descriptor_({0}) + , count_(0) + {} + + descriptor_info operator()() const + { + auto& count = descriptor_.count; + if (count == 0) { + for (auto& item : list_) + { + descriptor_.descriptor[count++] = item->operator()(); + /*switch (item->type_) + { + case EVENT_FILTER_TYPE_SYSTEM_FLAGS: + { + auto tmp = static_cast(const_cast(item)); + if (tmp) + { + descriptor_.descriptor[count++] = (*tmp)(); + } + break; + } + case EVENT_FILTER_TYPE_EVENT_ID: { + auto tmp = static_cast(const_cast(item)); + if (tmp) { + descriptor_.descriptor[count++] = (*tmp)(); + } + break; + } + case EVENT_FILTER_TYPE_EVENT_NAME: { + auto tmp = static_cast(const_cast(item)); + if (tmp) { + descriptor_.descriptor[count++] = (*tmp)(); + } + break; + } + case EVENT_FILTER_TYPE_PAYLOAD: { + auto tmp = static_cast(const_cast(item)); + if (tmp) { + descriptor_.descriptor[count++] = (*tmp)(); + } + break; + } + case EVENT_FILTER_TYPE_PID: { + auto tmp = static_cast(const_cast(item)); + if (tmp) { + descriptor_.descriptor[count++] = (*tmp)(); + } + break; + } + default: { + break; + } + }*/ + } + } + + return descriptor_; + } + private: + mutable unsigned long count_; + mutable descriptor_info descriptor_; + std::vector list_; + }; + struct system_flags_event_filter : direct_event_filter_base { system_flags_event_filter(unsigned long long flags, unsigned long size) : flags_(flags), @@ -188,7 +465,6 @@ namespace krabs { unsigned long size_; }; - /* typedef struct _PAYLOAD_FILTER_PREDICATE { LPWSTR FieldName; diff --git a/bluekrabs/krabs/filtering/post_event_filter.hpp b/bluekrabs/krabs/filtering/post_event_filter.hpp new file mode 100644 index 0000000..e69de29 diff --git a/bluekrabs/krabs/filtering/pre_event_filter.hpp b/bluekrabs/krabs/filtering/pre_event_filter.hpp new file mode 100644 index 0000000..4eb1980 --- /dev/null +++ b/bluekrabs/krabs/filtering/pre_event_filter.hpp @@ -0,0 +1,254 @@ +#include +#include +#include + +namespace krabs { + + // + // EVENT_FILTER_TYPE values for the Type field of EVENT_FILTER_DESCRIPTOR. + // * / + //#define EVENT_FILTER_TYPE_NONE (0x00000000) + //#define EVENT_FILTER_TYPE_SCHEMATIZED (0x80000000) // Provider-side. + //#define EVENT_FILTER_TYPE_SYSTEM_FLAGS (0x80000001) // Internal use only. + //#define EVENT_FILTER_TYPE_TRACEHANDLE (0x80000002) // Initiate rundown. + //#define EVENT_FILTER_TYPE_PID (0x80000004) // Process ID. + //#define EVENT_FILTER_TYPE_EXECUTABLE_NAME (0x80000008) // EXE file name. + //#define EVENT_FILTER_TYPE_PACKAGE_ID (0x80000010) // Package ID. + //#define EVENT_FILTER_TYPE_PACKAGE_APP_ID (0x80000020) // Package Relative App Id (PRAID). + //#define EVENT_FILTER_TYPE_PAYLOAD (0x80000100) // TDH payload filter. + //#define EVENT_FILTER_TYPE_EVENT_ID (0x80000200) // Event IDs. + //#define EVENT_FILTER_TYPE_EVENT_NAME (0x80000400) // Event name (TraceLogging only). + //#define EVENT_FILTER_TYPE_STACKWALK (0x80001000) // Event IDs for stack. + //#define EVENT_FILTER_TYPE_STACKWALK_NAME (0x80002000) // Event name for stack (TraceLogging only). + //#define EVENT_FILTER_TYPE_STACKWALK_LEVEL_KW (0x80004000) // Filter stack collection by level and keyword. + //#define EVENT_FILTER_TYPE_CONTAINER (0x80008000) // Filter by Container ID. + + struct pre_predicate_base { + pre_predicate_base() {} + + virtual EVENT_FILTER_DESCRIPTOR operator()() const = 0; + }; + + struct filter_descriptor { + unsigned long count = 0; + EVENT_FILTER_DESCRIPTOR descriptor[MAX_EVENT_FILTERS_COUNT]; + }; + + struct system_flags : pre_predicate_base { + system_flags(unsigned long long a1, unsigned long a2) + : descriptor_({ 0 }) + , data_(a1) + , size_(a2) + {} + + EVENT_FILTER_DESCRIPTOR operator()() const override + { + descriptor_.Ptr = reinterpret_cast(&data_); + descriptor_.Size = size_; + descriptor_.Type = EVENT_FILTER_TYPE_SYSTEM_FLAGS; + + return descriptor_; + } + + private: + mutable unsigned long long data_; + unsigned long size_; + mutable EVENT_FILTER_DESCRIPTOR descriptor_; + }; + + struct event_ids : pre_predicate_base { + event_ids(std::set a1, bool a2) + : descriptor_({ 0 }) + , data_(a1) + , filter_in_(a2) + {} + + EVENT_FILTER_DESCRIPTOR operator()() const override + { + /*typedef struct _EVENT_FILTER_EVENT_ID { + BOOLEAN FilterIn; + UCHAR Reserved; + USHORT Count; + USHORT Events[ANYSIZE_ARRAY]; + } EVENT_FILTER_EVENT_ID, * PEVENT_FILTER_EVENT_ID;*/ + + auto count = data_.size(); + if (count > 0) { + auto cache_size = FIELD_OFFSET(EVENT_FILTER_EVENT_ID, Events[count]); + cache_ = std::make_unique(cache_size); + auto tmp = reinterpret_cast(cache_.get()); + tmp->FilterIn = filter_in_; + tmp->Count = static_cast(count); + int i = 0; + for (auto item : data_) { + tmp->Events[i++] = item; + } + descriptor_.Ptr = reinterpret_cast(cache_.get()); + descriptor_.Size = cache_size; + descriptor_.Type = EVENT_FILTER_TYPE_EVENT_ID; + } + + return descriptor_; + } + + private: + std::set data_; + bool filter_in_; // When this member is TRUE, filtering is enabled for the specified event IDs. When this member is FALSE, filtering is disabled for the event IDs. + mutable EVENT_FILTER_DESCRIPTOR descriptor_; + mutable std::unique_ptr cache_; + }; + + struct process_pids : pre_predicate_base { + process_pids(std::set a1) + : descriptor_({ 0 }) + , data_(a1) + {} + + EVENT_FILTER_DESCRIPTOR operator()() const override + { + /*typedef struct _EVENT_FILTER_EVENT_ID { + BOOLEAN FilterIn; + UCHAR Reserved; + USHORT Count; + USHORT Events[ANYSIZE_ARRAY]; + } EVENT_FILTER_EVENT_ID, * PEVENT_FILTER_EVENT_ID;*/ + + auto count = data_.size(); + if (count > 0) { + int i = 0; + for (auto item : data_) { + if (i < MAX_EVENT_FILTER_PID_COUNT) { + cache_[i++] = item; + } + } + descriptor_.Ptr = reinterpret_cast(cache_); + descriptor_.Size = sizeof(unsigned int) * i; + descriptor_.Type = EVENT_FILTER_TYPE_PID; + } + + return descriptor_; + } + + private: + std::set data_; + mutable EVENT_FILTER_DESCRIPTOR descriptor_; + mutable unsigned int cache_[MAX_EVENT_FILTER_PID_COUNT] = { 0 }; + }; + + struct event_names : pre_predicate_base { + event_names(std::set a1, bool a2) + : descriptor_({ 0 }) + , data_(a1) + , filter_in_(a2) + {} + + EVENT_FILTER_DESCRIPTOR operator()() const override + { + /*typedef struct _EVENT_FILTER_EVENT_NAME { + ULONGLONG MatchAnyKeyword; + ULONGLONG MatchAllKeyword; + UCHAR Level; + BOOLEAN FilterIn; + USHORT NameCount; + UCHAR Names[ANYSIZE_ARRAY]; + } EVENT_FILTER_EVENT_NAME, * PEVENT_FILTER_EVENT_NAME;*/ + + auto count = data_.size(); + if (count > 0) { + auto cache_size = FIELD_OFFSET(EVENT_FILTER_EVENT_NAME, Names[count]); + cache_ = std::make_unique(cache_size); + auto tmp = reinterpret_cast(cache_.get()); + tmp->FilterIn = filter_in_; + tmp->Level = 0; + tmp->MatchAnyKeyword = 0; + tmp->MatchAllKeyword = 0; + tmp->NameCount = static_cast(count); + // The Names field should be a series of + // NameCount null terminated utf-8 + // event names. + auto i = 0; + for (auto item1 : data_) { + item1.push_back('\0'); + for (auto& item2 : item1) { + tmp->Names[i++] = item2; + } + } + + descriptor_.Ptr = reinterpret_cast(cache_.get()); + descriptor_.Size = cache_size; + descriptor_.Type = EVENT_FILTER_TYPE_EVENT_NAME; + } + + return descriptor_; + } + + private: + std::set data_; + bool filter_in_; + mutable EVENT_FILTER_DESCRIPTOR descriptor_; + mutable std::unique_ptr cache_; + }; + + struct event_payloads : pre_predicate_base { + event_payloads(std::set a1, bool a2) + : descriptor_({ 0 }) + , data_(a1) + , filter_in_(a2) + {} + + EVENT_FILTER_DESCRIPTOR operator()() const override + { + /*typedef struct _PAYLOAD_FILTER_PREDICATE { + LPWSTR FieldName; + USHORT CompareOp; + LPWSTR Value; + } PAYLOAD_FILTER_PREDICATE, *PPAYLOAD_FILTER_PREDICATE; + + EVENT_FILTER_TYPE_PAYLOAD + + */ + + + + return descriptor_; + } + + private: + std::set data_; + bool filter_in_; + mutable EVENT_FILTER_DESCRIPTOR descriptor_; + mutable PAYLOAD_FILTER_PREDICATE cache_[MAX_PAYLOAD_PREDICATES] = { 0 }; + //mutable std::unique_ptr cache_; + }; + + + /** + * + * + * + */ + struct pre_event_filter { + pre_event_filter() {} + pre_event_filter(std::vector> list) + : descriptor_({ 0 }) + , list_(list) + {} + + filter_descriptor operator()() const + { + auto& count = descriptor_.count; + if (count == 0) { + for (auto& item : list_) + { + descriptor_.descriptor[count++] = item->operator()(); + } + } + + return descriptor_; + } + + std::vector> list_; + mutable filter_descriptor descriptor_; + + }; +} /* namespace krabs */ \ No newline at end of file diff --git a/bluekrabs/krabs/kt.hpp b/bluekrabs/krabs/kt.hpp index 512b694..623094d 100644 --- a/bluekrabs/krabs/kt.hpp +++ b/bluekrabs/krabs/kt.hpp @@ -25,7 +25,7 @@ namespace krabs { namespace details { struct kt { typedef krabs::kernel_provider provider_type; - + typedef int provider_enable_info; /** * * Used to assign a name to the trace instance that is being @@ -59,6 +59,15 @@ namespace krabs { namespace details { static void enable_providers( const krabs::trace &trace); + /** + * + * Enables the providers that are attached to the given trace. + * + */ + static void enable_provider( + krabs::trace& trace, + const krabs::details::kt::provider_type& p); + /** * * Enables the configured kernel rundown flags. @@ -112,7 +121,7 @@ namespace krabs { namespace details { const krabs::trace &trace) { unsigned long flags = 0; - for (auto &provider : trace.providers_) { + for (auto &provider : trace.enabled_providers_) { flags |= provider.get().flags(); } @@ -131,7 +140,7 @@ namespace krabs { namespace details { error_check_common_conditions(status); auto group_mask_set = false; - for (auto& provider : trace.providers_) { + for (auto& provider : trace.enabled_providers_) { auto group = provider.get().group_mask(); PERFINFO_OR_GROUP_WITH_GROUPMASK(group, &(gmi.EventTraceGroupMasks)); group_mask_set |= (group != 0); @@ -146,12 +155,26 @@ namespace krabs { namespace details { return; } + inline void kt::enable_provider( + krabs::trace& trace, + const krabs::details::kt::provider_type& p) + { + if (trace.registrationHandle_ == INVALID_PROCESSTRACE_HANDLE) + return; + GUID g; + if (g == p.guid()) { + + } + error_check_common_conditions(ERROR_NOT_FOUND); + return; + } + inline void kt::enable_rundown( const krabs::trace& trace) { bool rundown_enabled = false; ULONG rundown_flags = 0; - for (auto& provider : trace.providers_) { + for (auto& provider : trace.enabled_providers_) { rundown_enabled |= provider.get().rundown_enabled(); rundown_flags |= provider.get().rundown_flags(); } @@ -174,7 +197,7 @@ namespace krabs { namespace details { const EVENT_RECORD &record, const krabs::trace &trace) { - for (auto &provider : trace.providers_) { + for (auto &provider : trace.enabled_providers_) { if (provider.get().id() == record.EventHeader.ProviderId) { provider.get().on_event(record, trace.context_); return; diff --git a/bluekrabs/krabs/provider.hpp b/bluekrabs/krabs/provider.hpp index fb8c164..a61efff 100644 --- a/bluekrabs/krabs/provider.hpp +++ b/bluekrabs/krabs/provider.hpp @@ -10,7 +10,7 @@ #include "compiler_check.hpp" #include "filtering/event_filter.hpp" -#include "filtering/direct_event_filter.hpp" +#include "filtering/pre_event_filter.hpp" #include "perfinfo_groupmask.hpp" #include "trace_context.hpp" #include "wstring_convert.hpp" @@ -45,7 +45,6 @@ namespace krabs { typedef std::function provider_error_callback; namespace details { - /** * * Serves as a base for providers and kernel_providers. Handles event @@ -113,7 +112,7 @@ namespace krabs { */ void add_filter(const event_filter &f); - void add_filter(const direct_event_filters& f); + void add_filter(const pre_event_filter& f); protected: /** @@ -127,8 +126,7 @@ namespace krabs { std::deque callbacks_; std::deque error_callbacks_; std::deque filters_; - std::deque direct_filters_; - + filter_descriptor pre_filter_; private: template friend class details::trace_manager; @@ -275,6 +273,13 @@ namespace krabs { */ void enable_rundown_events(); + /** + * + * Retrieves the GUID associated with this provider. + * + */ + const GUID guid() const; + /** * * Turns a strongly typed provider to provider<> (useful for @@ -296,7 +301,7 @@ namespace krabs { T level_; T enable_property_; bool rundown_enabled_; - + private: template friend class details::trace_manager; @@ -353,11 +358,18 @@ namespace krabs { /** * - * Retrieves the GUID associated with this provider. + * Retrieves the krabs::guid associated with this provider. * */ const krabs::guid &id() const; + /** + * + * Retrieves the GUID associated with this provider. + * + */ + const GUID guid() const; + /** * * Sets flags to be enabled for the kernel rundown GUID. @@ -485,9 +497,9 @@ namespace krabs { } template - void base_provider::add_filter(const direct_event_filters& f) + void base_provider::add_filter(const pre_event_filter& f) { - direct_filters_.push_back(f); + pre_filter_ = f(); } template @@ -495,7 +507,8 @@ namespace krabs { { try { - for (auto& callback : callbacks_) { + for (auto& callback : callbacks_) + { callback(record, trace_context); } @@ -505,7 +518,8 @@ namespace krabs { } catch (krabs::could_not_find_schema& ex) { - for (auto& error_callback : error_callbacks_) { + for (auto& error_callback : error_callbacks_) + { error_callback(record, ex.what()); } } @@ -608,6 +622,7 @@ namespace krabs { } guid_ = providerGuid; + //guid2_ = krabs::guid(providerGuid); any_ = 0; all_ = 0; level_ = 5; @@ -667,9 +682,20 @@ namespace krabs { return tmp; } + template + inline const GUID provider::guid() const + { + return guid_; + } + inline const krabs::guid &kernel_provider::id() const { return id_; } + inline const GUID kernel_provider::guid() const + { + return id_.operator GUID(); + } + } diff --git a/bluekrabs/krabs/schema_locator.hpp b/bluekrabs/krabs/schema_locator.hpp index e3ec591..7ad8c41 100644 --- a/bluekrabs/krabs/schema_locator.hpp +++ b/bluekrabs/krabs/schema_locator.hpp @@ -149,15 +149,17 @@ namespace krabs { // allocate and fill the schema from TDH auto buffer = std::unique_ptr(new char[bufferSize]); - error_check_common_conditions( - TdhGetEventInformation( + status = TdhGetEventInformation( (PEVENT_RECORD)&record, 0, NULL, (PTRACE_EVENT_INFO)buffer.get(), - &bufferSize), - record); - + &bufferSize); + + if (status != ERROR_SUCCESS) { + error_check_common_conditions(status, record); + } + return buffer; } } diff --git a/bluekrabs/krabs/trace.hpp b/bluekrabs/krabs/trace.hpp index ffed74f..4dc42bd 100644 --- a/bluekrabs/krabs/trace.hpp +++ b/bluekrabs/krabs/trace.hpp @@ -4,6 +4,8 @@ #pragma once #include +#include +#include #include "compiler_check.hpp" #include "guid.hpp" @@ -31,25 +33,44 @@ namespace krabs { * Selected statistics about an ETW trace * */ - class trace_stats - { + class trace_stats { public: - const uint32_t buffersCount; - const uint32_t buffersFree; - const uint32_t buffersWritten; - const uint32_t buffersLost; - const uint64_t eventsTotal; - const uint64_t eventsHandled; - const uint32_t eventsLost; - - trace_stats(uint64_t eventsHandled, const EVENT_TRACE_PROPERTIES& props) - : buffersCount(props.NumberOfBuffers) - , buffersFree(props.FreeBuffers) - , buffersWritten(props.BuffersWritten) - , buffersLost(props.RealTimeBuffersLost) - , eventsTotal(eventsHandled + props.EventsLost) - , eventsHandled(eventsHandled) - , eventsLost(props.EventsLost) + const uint32_t buffers_count; + const uint32_t buffers_free; + const uint32_t buffers_written; + const uint32_t buffers_lost; + const uint64_t events_total; + const uint64_t events_handled; + const uint32_t events_lost; + const uint32_t buffer_size; + const uint32_t minimum_buffers; + const uint32_t maximum_buffers; + const uint32_t maximum_file_size; + const uint32_t log_file_mode; + const uint32_t flush_timer; + const uint32_t enable_flags; + const std::wstring log_file_name; + const std::wstring logger_name; + const uint32_t flush_threshold; + + trace_stats(uint64_t eventsHandled, const details::trace_info& props) + : buffers_count(props.properties.NumberOfBuffers) + , buffers_free(props.properties.FreeBuffers) + , buffers_written(props.properties.BuffersWritten) + , buffers_lost(props.properties.RealTimeBuffersLost) + , events_total(eventsHandled + props.properties.EventsLost) + , events_handled(eventsHandled) + , events_lost(props.properties.EventsLost) + , buffer_size(props.properties.BufferSize) + , minimum_buffers(props.properties.MinimumBuffers) + , maximum_buffers(props.properties.MaximumBuffers) + , maximum_file_size(props.properties.MaximumFileSize) + , log_file_mode(props.properties.LogFileMode) + , flush_timer(props.properties.FlushTimer) + , enable_flags(props.properties.EnableFlags) + , flush_threshold(props.properties.FlushThreshold) + , logger_name(props.traceName) + , log_file_name(props.logfileName) { } }; @@ -166,7 +187,22 @@ namespace krabs { /** * - * Enables the provider on the given user trace. + * Update the session configuration so that the session receives + * the requested events from the provider. + * + * + * krabs::trace trace; + * krabs::guid id(L"{A0C1853B-5C40-4B15-8766-3CF1C58F985A}"); + * provider<> powershell(id); + * trace.enable(powershell); + * + */ + void update(); + + /** + * + * Update the session configuration so that the session receives + * the requested events from the provider. * * * krabs::trace trace; @@ -177,6 +213,20 @@ namespace krabs { */ void enable(const typename T::provider_type &p); + /** + * + * Update the session configuration so that the session does not + * receive events from the provider. + * + * + * krabs::trace trace; + * krabs::guid id(L"{A0C1853B-5C40-4B15-8766-3CF1C58F985A}"); + * provider<> powershell(id); + * trace.disable(powershell); + * + */ + void disable(const typename T::provider_type& p); + /** * * Starts a trace session. @@ -204,7 +254,22 @@ namespace krabs { * trace.stop(); * */ - void stop(); + void stop(bool force = false); + + /** + * + * Closes a trace session. + * + * + * krabs::trace trace; + * krabs::guid id(L"{A0C1853B-5C40-4B15-8766-3CF1C58F985A}"); + * provider<> powershell(id); + * trace.enable(powershell); + * trace.start(); + * trace.stop(); + * + */ + void close(); /** * @@ -222,6 +287,13 @@ namespace krabs { */ EVENT_TRACE_LOGFILE open(); + /** + * + * Transition the ETW trace from real-time to file or vice versa. + * + */ + void transition(); + /** * * Start processing events for an already opened session. @@ -291,11 +363,27 @@ namespace krabs { */ void on_event(const EVENT_RECORD &); + /////** + //// * + //// * Updates a trace session. + //// * + //// * + //// * todo + //// * + //// */ + //void update_provider(const typename T::provider_type& p); + private: std::wstring name_; std::wstring logFilename_; bool non_stoppable_; - std::deque> providers_; + std::deque> enabled_providers_; + // This essentially takes the union of all the provider flags + // for a given provider. This comes about when multiple providers + // for the same XX are provided and request different provider flags. + // TODO: Only forward the calls that are requested to each provider. + typename T::provider_enable_info provider_enable_info_; + std::mutex providers_mutex_; TRACEHANDLE registrationHandle_; TRACEHANDLE sessionHandle_; @@ -354,11 +442,15 @@ namespace krabs { if (!non_stoppable_) { stop(); } + else { + close(); + } } template void trace::set_trace_properties(const PEVENT_TRACE_PROPERTIES properties) { + properties_ = {}; properties_.BufferSize = properties->BufferSize; properties_.MinimumBuffers = properties->MinimumBuffers; properties_.MaximumBuffers = properties->MaximumBuffers; @@ -386,29 +478,75 @@ namespace krabs { void trace::on_event(const EVENT_RECORD &record) { ++eventsHandled_; + std::lock_guard lock(providers_mutex_); T::forward_events(record, *this); } template - void trace::enable(const typename T::provider_type &p) - { - providers_.push_back(std::ref(p)); + void trace::enable(const typename T::provider_type& p) + { + auto insert_unique = [&](const auto& _p) { + auto it = std::find_if(enabled_providers_.begin(), enabled_providers_.end(), [&_p](const auto& x) { + return x.get().guid() == _p.guid(); + }); + if (it == enabled_providers_.end()) { + enabled_providers_.push_back(std::ref(_p)); + } + else { + *it = std::ref(_p); + } + }; + + if (registrationHandle_ == INVALID_PROCESSTRACE_HANDLE) { + insert_unique(p); + } + else { + std::lock_guard lock(providers_mutex_); + details::trace_manager manager(*this); + manager.enable(p); + insert_unique(p); + } + } + + template + void trace::disable(const typename T::provider_type& p) + { + if (registrationHandle_ == INVALID_PROCESSTRACE_HANDLE) { + auto it = std::find_if(enabled_providers_.begin(), enabled_providers_.end(), [&p](const auto& x) { + return x.get().guid() == p.guid(); + }); + + if (it != enabled_providers_.end()) { + std::lock_guard lock(providers_mutex_); + details::trace_manager manager(*this); + manager.disable(p); + enabled_providers_.erase(it); + } + } } template void trace::start() { eventsHandled_ = 0; - details::trace_manager manager(*this); manager.start(); } template - void trace::stop() + void trace::stop(bool force) + { + if (!non_stoppable_ || force) { + details::trace_manager manager(*this); + manager.stop(); + } + } + + template + void trace::close() { details::trace_manager manager(*this); - manager.stop(); + manager.close(); } template @@ -420,6 +558,12 @@ namespace krabs { return manager.open(); } + template + void trace::transition() + { + return; + } + template void trace::process() { @@ -429,6 +573,13 @@ namespace krabs { manager.process(); } + template + void trace::update() + { + details::trace_manager manager(*this); + manager.update(); + } + template trace_stats trace::query_stats() { diff --git a/bluekrabs/krabs/ut.hpp b/bluekrabs/krabs/ut.hpp index f27e52c..11d410e 100644 --- a/bluekrabs/krabs/ut.hpp +++ b/bluekrabs/krabs/ut.hpp @@ -4,6 +4,8 @@ #pragma once #include +#include +#include #include "compiler_check.hpp" #include "trace.hpp" @@ -20,46 +22,21 @@ namespace krabs { namespace details { * code. * */ - struct ut { - - typedef krabs::provider<> provider_type; - - struct filter_flags { - ULONG filter_type_; - std::set event_ids_; - std::set event_names_; - ULONGLONG custom_value_; - ULONG custom_size_; - std::wstring field_name_; - unsigned short compare_op_; - std::wstring value_; - }; - - struct event_filter_buffers { - std::vector event_id_buffer; - std::vector event_pid_buffer; - std::vector event_exe_name_buffer; - std::vector event_name_buffer; - unsigned int pids[MAX_EVENT_FILTER_PID_COUNT] = { 0 }; - EVENT_FILTER_DESCRIPTOR filter_desc[15] = { 0 }; - PAYLOAD_FILTER_PREDICATE predicates[MAX_PAYLOAD_PREDICATES] = { 0 }; - }; + struct ut { //ENABLE_TRACE_PARAMETERS - struct provider_enable_info { + struct enable_trace_info { + GUID guid; ENABLE_TRACE_PARAMETERS parameters; bool rundown_enabled = false; UCHAR level; ULONGLONG any; ULONGLONG all; ULONG enable_property; - - event_filter_buffers event_buffer; + EVENT_FILTER_DESCRIPTOR filter_desc[15] = { 0 }; }; - typedef std::map provider_enable_info_container; - - - + typedef krabs::provider<> provider_type; + typedef std::map provider_enable_info; /** * * Used to assign a name to the trace instance that is being @@ -86,53 +63,29 @@ namespace krabs { namespace details { /** * - * todo. - * - */ - static ULONG populate_system_flags_filter_desc(ut::provider_enable_info& info, const system_flags_event_filter* system_flags); - - /** - * - * todo. - * - */ - static ULONG populate_event_id_filter_desc(ut::provider_enable_info& info, const event_id_event_filter* system_flags); - - /** - * - * todo. - * - */ - static ULONG populate_event_pid_filter_desc(ut::provider_enable_info& info, const event_pid_event_filter* system_flags); - - /** - * - * todo. - * - */ - static ULONG populate_event_name_filter_desc(ut::provider_enable_info& info, const event_name_event_filter* event_names); - - /** - * - * todo. - * - */ - static ULONG populate_event_payload_filter_desc(ut::provider_enable_info& info, const event_payload_event_filter* event_names); - - /** - * - * todo. + * Enables the providers that are attached to the given trace. * */ - static void populate_provider_enable_info(const ut::provider_type& provider, ut::provider_enable_info& info); + static void enable_providers( + krabs::trace &trace); + + ///** + // * + // * Enables the providers that are attached to the given trace. + // * + // */ + static void enable_provider( + krabs::trace& trace, + const krabs::details::ut::provider_type& p); /** * * Enables the providers that are attached to the given trace. * */ - static void enable_providers( - const krabs::trace &trace); + static void disable_provider( + krabs::trace& trace, + const krabs::details::ut::provider_type& p); /** * @@ -141,8 +94,8 @@ namespace krabs { namespace details { * */ static void enable_rundown( - const krabs::trace& trace); - + krabs::trace& trace); + /** * * Decides to forward an event to any of the providers in the trace. @@ -150,7 +103,7 @@ namespace krabs { namespace details { */ static void forward_events( const EVENT_RECORD &record, - const krabs::trace &trace); + krabs::trace &trace); /** * @@ -186,289 +139,103 @@ namespace krabs { namespace details { { return 0; } - - inline ULONG ut::populate_system_flags_filter_desc(ut::provider_enable_info& info, const system_flags_event_filter* system_flags) - { - auto& filter_desc = info.parameters.EnableFilterDesc[info.parameters.FilterDescCount]; - filter_desc.Ptr = reinterpret_cast(&system_flags->get_value()); - filter_desc.Size = system_flags->get_size(); - filter_desc.Type = EVENT_FILTER_TYPE_SYSTEM_FLAGS; - - return 1; - } - - inline ULONG ut::populate_event_id_filter_desc(ut::provider_enable_info& info, const event_id_event_filter* event_ids) - { - /*typedef struct _EVENT_FILTER_EVENT_ID { - BOOLEAN FilterIn; - UCHAR Reserved; - USHORT Count; - USHORT Events[ANYSIZE_ARRAY]; - } EVENT_FILTER_EVENT_ID, * PEVENT_FILTER_EVENT_ID;*/ - auto& filter_desc = info.parameters.EnableFilterDesc[info.parameters.FilterDescCount]; - auto& buffer = info.event_buffer.event_id_buffer; - auto event_ids_count = event_ids->get_data().size(); - auto buffer_size = FIELD_OFFSET(EVENT_FILTER_EVENT_ID, Events[event_ids_count]); - if (event_ids_count > 0) { - filter_desc.Type = EVENT_FILTER_TYPE_EVENT_ID; - buffer.resize(buffer_size, 0); - auto event_ids_desc = reinterpret_cast(&buffer[0]); - event_ids_desc->FilterIn = TRUE; - event_ids_desc->Count = static_cast(event_ids_count); - auto index = 0; - for (auto id : event_ids->get_data()) { - event_ids_desc->Events[index] = id; - index++; - } - filter_desc.Ptr = reinterpret_cast(event_ids_desc); - filter_desc.Size = buffer_size; - - return 1; - } - - return 0; - } - - inline ULONG ut::populate_event_pid_filter_desc(ut::provider_enable_info& info, const event_pid_event_filter* event_ids) + + inline void ut::enable_providers( + krabs::trace& trace) { - /*typedef struct _EVENT_FILTER_EVENT_ID { - BOOLEAN FilterIn; - UCHAR Reserved; - USHORT Count; - USHORT Events[ANYSIZE_ARRAY]; - } EVENT_FILTER_EVENT_ID, * PEVENT_FILTER_EVENT_ID;*/ - auto& filter_desc = info.parameters.EnableFilterDesc[info.parameters.FilterDescCount]; - auto& buffer = info.event_buffer.pids; - auto event_ids_count = event_ids->get_data().size(); - //auto buffer_size = FIELD_OFFSET(EVENT_FILTER_EVENT_ID, Events[event_ids_count]); - if (event_ids_count > 0) { - /*filter_desc.Type = EVENT_FILTER_TYPE_PID; - buffer.resize(buffer_size, 0); - auto event_ids_desc = reinterpret_cast(&buffer[0]); - event_ids_desc->FilterIn = TRUE; - event_ids_desc->Count = static_cast(event_ids_count); - auto index = 0; - for (auto id : event_ids->get_data()) { - event_ids_desc->Events[index] = id; - index++; - }*/ - auto index = 0; - for (auto id : event_ids->get_data()) { - buffer[index] = id; - index++; - } - - auto size = sizeof(unsigned int) * index; - filter_desc.Type = EVENT_FILTER_TYPE_PID; - filter_desc.Ptr = reinterpret_cast(buffer); - filter_desc.Size = (ULONG)size; - - return 1; + if (trace.registrationHandle_ == INVALID_PROCESSTRACE_HANDLE) { + return; } - - return 0; - } - - inline ULONG ut::populate_event_name_filter_desc(ut::provider_enable_info& info, const event_name_event_filter* event_names) - { - /*typedef struct _EVENT_FILTER_EVENT_NAME { - ULONGLONG MatchAnyKeyword; - ULONGLONG MatchAllKeyword; - UCHAR Level; - BOOLEAN FilterIn; - USHORT NameCount; - UCHAR Names[ANYSIZE_ARRAY]; - } EVENT_FILTER_EVENT_NAME, * PEVENT_FILTER_EVENT_NAME;*/ - auto& filter_desc = info.parameters.EnableFilterDesc[info.parameters.FilterDescCount]; //todo check if slot set - auto& buffer = info.event_buffer.event_id_buffer; - auto names_count = event_names->get_data().size(); - auto buffer_size = FIELD_OFFSET(EVENT_FILTER_EVENT_ID, Events[names_count]); - if (names_count > 0) { - filter_desc.Type = EVENT_FILTER_TYPE_EVENT_NAME; - buffer.resize(buffer_size, 0); - auto event_name_desc = reinterpret_cast(&buffer[0]); - event_name_desc->FilterIn = TRUE; - event_name_desc->Level = info.level; - event_name_desc->MatchAnyKeyword = info.any; - event_name_desc->MatchAllKeyword = info.all; - - // The Names field should be a series of - // NameCount null terminated utf-8 - // event names. - auto index = 0; - for (auto name : event_names->get_data()) { - name.push_back('\0'); - for (auto& s : name) { - event_name_desc->Names[index] = s; - index++; - } - event_name_desc->NameCount = static_cast(names_count); - } - filter_desc.Ptr = reinterpret_cast(event_name_desc); - filter_desc.Size = buffer_size; - - return 1; + + for (auto& provider : trace.enabled_providers_) { + auto& _provider = provider.get(); + enable_provider(trace, _provider); } - - return 0; } - inline ULONG ut::populate_event_payload_filter_desc(ut::provider_enable_info& info, const event_payload_event_filter* event_payload) + inline void ut::enable_provider( + krabs::trace& trace, + const krabs::details::ut::provider_type& provider) { - /*typedef struct _PAYLOAD_FILTER_PREDICATE { - LPWSTR FieldName; - USHORT CompareOp; - LPWSTR Value; - } PAYLOAD_FILTER_PREDICATE, *PPAYLOAD_FILTER_PREDICATE;*/ - auto& filter_desc = info.parameters.EnableFilterDesc[info.parameters.FilterDescCount]; - auto& predicates = info.event_buffer.predicates; - ULONG predicates_count = 0; - EVENT_DESCRIPTOR ed = { 0 }; - PVOID event_filter[MAX_EVENT_FILTERS_COUNT] = { 0 }; - std::wstring mans{ L"C:\\data\\microsoft-windows-system-events.dll" }; - ULONG Status = TdhLoadManifestFromBinary((PWSTR)mans.c_str()); - if (Status != ERROR_SUCCESS) { - printf("TdhCreatePayloadFilter() failed with %lu\n", Status); - } - //0. check if manifest provider - //1. load find eval and load manifest - //2. polpulate first payload filter - predicates[predicates_count].CompareOp = event_payload->get_compare_op(); - predicates[predicates_count].FieldName = static_cast(const_cast(event_payload->get_field_name().c_str())); - predicates[predicates_count].Value = static_cast(const_cast(event_payload->get_value().c_str())); - ed.Id = 5; - //TdhCreatePayloadFilter(); - Status = TdhCreatePayloadFilter( - &info.parameters.SourceId, - &ed, - TRUE, // TRUE Match any predicates (OR); FALSE Match all predicates (AND) - 1, - predicates, - &event_filter[predicates_count++]); - if (Status != ERROR_SUCCESS) { - printf("TdhCreatePayloadFilter() failed with %lu\n", Status); - } - Status = TdhAggregatePayloadFilters( - predicates_count, - event_filter, - NULL, - &filter_desc); - if (Status != ERROR_SUCCESS) { - printf("TdhAggregatePayloadFilters() failed with %lu\n", Status); + if (trace.registrationHandle_ == INVALID_PROCESSTRACE_HANDLE) { + return; } - return 1; - } + // This essentially takes the union of all the provider flags + // for a given provider GUID. This comes about when multiple providers + // for the same GUID are provided and request different provider flags. + auto& provider_enable_info = trace.provider_enable_info_[provider.guid_]; + provider_enable_info.parameters.ControlFlags = 0; + provider_enable_info.parameters.Version = ENABLE_TRACE_PARAMETERS_VERSION_2; + provider_enable_info.guid = provider.guid_; + // TODO: Only forward the calls that are requested to each provider. + provider_enable_info.level |= provider.level_; + provider_enable_info.any |= provider.any_; + provider_enable_info.all |= provider.all_; + provider_enable_info.rundown_enabled |= provider.rundown_enabled_; + provider_enable_info.parameters.EnableProperty |= provider.enable_property_; - inline void ut::populate_provider_enable_info(const ut::provider_type& provider, ut::provider_enable_info& info) - { - info.parameters.ControlFlags = 0; - info.parameters.Version = ENABLE_TRACE_PARAMETERS_VERSION_2; - info.parameters.SourceId = provider.guid_; - - info.level |= provider.level_; - info.any |= provider.any_; - info.all |= provider.all_; - info.rundown_enabled |= provider.rundown_enabled_; - - info.parameters.EnableProperty |= provider.enable_property_; - info.parameters.FilterDescCount = 0; - info.parameters.EnableFilterDesc = &info.event_buffer.filter_desc[0]; // There can only be one descriptor for each filter // type as specified by the Type member of the // EVENT_FILTER_DESCRIPTOR structure. - for (const auto& direct_filters : provider.direct_filters_) { - for (const auto& direct_filter : direct_filters.list_) { - switch (direct_filter->get_type()) { - case EVENT_FILTER_TYPE_SYSTEM_FLAGS: { - auto system_flags = reinterpret_cast(direct_filter.get()); - if (ULONG count = populate_system_flags_filter_desc(info, system_flags)) - { - info.parameters.FilterDescCount += count; - } - break; - } - case EVENT_FILTER_TYPE_EVENT_ID: { - auto event_ids = reinterpret_cast(direct_filter.get()); - if (ULONG count = populate_event_id_filter_desc(info, event_ids)) - { - info.parameters.FilterDescCount += count; - } - break; - } - case EVENT_FILTER_TYPE_EVENT_NAME: { - auto event_names = reinterpret_cast(direct_filter.get()); - if (ULONG count = populate_event_name_filter_desc(info, event_names)) - { - info.parameters.FilterDescCount += count; - } - break; - } - case EVENT_FILTER_TYPE_PAYLOAD: { - auto event_payload = dynamic_cast(direct_filter.get()); - if (ULONG count = populate_event_payload_filter_desc(info, event_payload)) - { - info.parameters.FilterDescCount += count; - } - break; - } - case EVENT_FILTER_TYPE_PID: { - auto event_pids = reinterpret_cast(direct_filter.get()); - if (ULONG count = populate_event_pid_filter_desc(info, event_pids)) - { - info.parameters.FilterDescCount += count; - } - break; - } - default: { - break; - } - } - } + if (provider.pre_filter_.count == 0) + { + provider_enable_info.parameters.FilterDescCount = 0; + provider_enable_info.parameters.EnableFilterDesc = &provider_enable_info.filter_desc[0]; + } + else + { + provider_enable_info.parameters.FilterDescCount = provider.pre_filter_.count; + provider_enable_info.parameters.EnableFilterDesc = const_cast(&provider.pre_filter_.descriptor[0]); } + ULONG status = EnableTraceEx2(trace.registrationHandle_, + &provider.guid_, + EVENT_CONTROL_CODE_ENABLE_PROVIDER, + provider_enable_info.level, + provider_enable_info.any, + provider_enable_info.all, + 0, + &provider_enable_info.parameters); - - - //return enable_trace_parameters{ 0 }; + error_check_common_conditions(status); } - inline void ut::enable_providers( - const krabs::trace& trace) + inline void ut::disable_provider( + krabs::trace& trace, + const krabs::details::ut::provider_type& provider) { - if (trace.registrationHandle_ == INVALID_PROCESSTRACE_HANDLE) + if (trace.registrationHandle_ == INVALID_PROCESSTRACE_HANDLE) { return; + } - provider_enable_info_container providers_enable_info; - // This function essentially takes the union of all the provider flags - // for a given provider GUID. This comes about when multiple providers - // for the same GUID are provided and request different provider flags. - // TODO: Only forward the calls that are requested to each provider. - for (auto& provider : trace.providers_) { - //auto& a = provider.get(); - auto& enable_info = providers_enable_info[provider.get().guid_]; - populate_provider_enable_info(provider, enable_info); - + auto it = trace.provider_enable_info_.find(provider.guid_); + if (it != trace.provider_enable_info_.end()) { ULONG status = EnableTraceEx2(trace.registrationHandle_, - &enable_info.parameters.SourceId, - EVENT_CONTROL_CODE_ENABLE_PROVIDER, - enable_info.level, - enable_info.any, - enable_info.all, + &provider.guid_, + EVENT_CONTROL_CODE_DISABLE_PROVIDER, + 0, + 0, + 0, 0, - &enable_info.parameters); + NULL); - error_check_common_conditions(status); + error_check_common_conditions(status); + if (status == ERROR_SUCCESS) { + trace.provider_enable_info_.erase(it); + } } } + + inline void ut::enable_rundown( - const krabs::trace& trace) + krabs::trace& trace) { if (trace.registrationHandle_ == INVALID_PROCESSTRACE_HANDLE) return; - for (auto& provider : trace.providers_) { + for (auto& provider : trace.enabled_providers_) { if (!provider.get().rundown_enabled_) continue; @@ -486,10 +253,10 @@ namespace krabs { namespace details { inline void ut::forward_events( const EVENT_RECORD &record, - const krabs::trace &trace) + krabs::trace &trace) { // for manifest providers, EventHeader.ProviderId is the Provider GUID - for (auto& provider : trace.providers_) { + for (auto& provider : trace.enabled_providers_) { if (record.EventHeader.ProviderId == provider.get().guid_) { provider.get().on_event(record, trace.context_); return; @@ -499,15 +266,17 @@ namespace krabs { namespace details { // for MOF providers, EventHeader.Provider is the *Message* GUID // we need to ask TDH for event information in order to determine the // correct provider to pass this event to - auto schema = get_event_schema_from_tdh(record); - auto eventInfo = reinterpret_cast(schema.get()); - for (auto& provider : trace.providers_) { - if (eventInfo->ProviderGuid == provider.get().guid_) { - provider.get().on_event(record, trace.context_); - return; + if ((record.EventHeader.EventProperty & EVENT_HEADER_PROPERTY_LEGACY_EVENTLOG) == 1) { + auto schema = get_event_schema_from_tdh(record); + auto eventInfo = reinterpret_cast(schema.get()); + for (auto& provider : trace.enabled_providers_) { + if (eventInfo->ProviderGuid == provider.get().guid_) { + provider.get().on_event(record, trace.context_); + return; + } } } - + if (trace.default_callback_ != nullptr) trace.default_callback_(record, trace.context_); } diff --git a/examples/ManagedExamples/ManagedExamples.csproj b/examples/ManagedExamples/ManagedExamples.csproj index ee8666e..3055935 100644 --- a/examples/ManagedExamples/ManagedExamples.csproj +++ b/examples/ManagedExamples/ManagedExamples.csproj @@ -13,6 +13,21 @@ 512 true + publish\ + true + Disk + false + Foreground + 7 + Days + false + false + true + 0 + 1.0.0.%2a + false + false + true x64 @@ -22,7 +37,7 @@ bin\Debug\ DEBUG;TRACE prompt - 4 + 5 false @@ -78,6 +93,9 @@ + + + @@ -86,12 +104,24 @@ {ed4e6027-541f-440a-a5ee-15dbb7b89423} - Microsoft.O365.Security.Native.ETW + Threathunters.BlueKrabsetw.Native.ETW + + + False + Microsoft .NET Framework 4.7.2 %28x86 and x64%29 + true + + + False + .NET Framework 3.5 SP1 + false + + trace has stopped - Assert::IsTrue(0 == trace.query_stats().buffersCount); + Assert::IsTrue(0 == trace.query_stats().buffers_count); } TEST_METHOD(should_get_same_trace_flags_as_set)