From a83f2cc0167023f7fdb77b59d3a0f945c25b9984 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Mon, 31 Jan 2022 17:13:51 -0300 Subject: [PATCH 001/152] chore(configOptions): add ConfigOptions command to ServiceProvider --- src/MetamorphosisServiceProvider.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/MetamorphosisServiceProvider.php b/src/MetamorphosisServiceProvider.php index 687c739a..4938fbac 100644 --- a/src/MetamorphosisServiceProvider.php +++ b/src/MetamorphosisServiceProvider.php @@ -3,6 +3,7 @@ namespace Metamorphosis; use Illuminate\Support\ServiceProvider; +use Metamorphosis\Console\ConfigOptionsCommand; use Metamorphosis\Console\ConsumerCommand; use Metamorphosis\Console\ConsumerMakeCommand; use Metamorphosis\Console\MiddlewareMakeCommand; @@ -26,6 +27,7 @@ public function register() ConsumerMakeCommand::class, MiddlewareMakeCommand::class, ProducerMakeCommand::class, + ConfigOptionsCommand::class, ]); $this->app->bind('metamorphosis', function ($app) { From 7b0061f4e517838bb059c4d3baaa2c6d5e8c1033 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Mon, 31 Jan 2022 17:16:15 -0300 Subject: [PATCH 002/152] chore(configOptions): call new configOptions artisan command --- .../ProducerWithConfigOptionsTest.php | 41 +++---------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 3382b590..5f5017a0 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -24,14 +24,13 @@ public function testShouldRunAProducerMessagesWithConfigOptions(): void $this->haveAHandlerConfigured(); // I Expect That - $this->myMessagesHaveBeenProduced(); - $this->expectNotToPerformAssertions(); + //$this->myMessagesHaveBeenProduced(); // When I - $this->haveSomeRandomMessageProduced(); + //$this->haveSomeRandomMessageProduced(); // I Expect That - $this->myMessagesHaveBeenLogged(); + //$this->myMessagesHaveBeenLogged(); // When I $this->runTheConsumer(); @@ -41,39 +40,11 @@ protected function runTheConsumer(): void { $dummy = new MessageConsumer($this->consumerConfigOptions); $this->instance('\App\Kafka\Consumers\ConsumerOverride', $dummy); - config([ - 'kafka_new_config' => [ - 'brokers' => [ - 'override' => [ - 'connections' => env( - 'KAFKA_BROKER_CONNECTIONS', - 'kafka:9092' - ), - ], - ], - 'topics' => [ - 'default' => [ - 'broker' => 'override', - 'consumer' => [ - 'consumer_groups' => [ - 'test-consumer-group' => [ - 'handler' => '\App\Kafka\Consumers\ConsumerOverride', - 'offset_reset' => 'earliest', - ], - ], - ], - ], - ], - ], - ]); + $this->artisan( - 'kafka:consume', + 'kafka:consume-config-class', [ - 'topic' => 'default', - 'consumer_group' => 'test-consumer-group', - '--timeout' => 20000, - '--times' => 2, - '--config_name' => 'kafka_new_config', + 'handler' => '\\App\\Kafka\\Consumers\\ConsumerOverride', ] ); } From 574f219b81c38f9530c1ab5794fd606a680603ba Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Mon, 31 Jan 2022 17:19:05 -0300 Subject: [PATCH 003/152] chore(configOptions): add ConfigOptionsCommand --- src/Console/ConfigOptionsCommand.php | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 src/Console/ConfigOptionsCommand.php diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php new file mode 100644 index 00000000..ccd8bb37 --- /dev/null +++ b/src/Console/ConfigOptionsCommand.php @@ -0,0 +1,49 @@ +make($this->option(), $this->argument()); + +// $this->writeStartingConsumer($configManager); + + $config = $this->argument()['handler']; + + $manager = Factory::make($this->argument()['handler']); + + $runner = app(Runner::class, compact('manager')); + //$runner->run($manager->get('times')); + } + + private function writeStartingConsumer(AbstractConfigManager $configManager) + { + $text = 'Starting consumer for topic: '.$configManager->get('topic').PHP_EOL; + $text .= ' on consumer group: '.$configManager->get('consumer_group').PHP_EOL; + $text .= 'Connecting in '.$configManager->get('connections').PHP_EOL; + $text .= 'Running consumer..'; + + $this->output->writeln($text); + } +} From a0e26ea2553919cbef4b7fe9994271615d680d21 Mon Sep 17 00:00:00 2001 From: David Franca Date: Mon, 31 Jan 2022 17:55:13 -0300 Subject: [PATCH 004/152] chore: make config manager with config options --- src/Connectors/Consumer/Config.php | 8 ++++++++ src/Console/ConfigOptionsCommand.php | 13 ++++++------- tests/Integration/ProducerWithConfigOptionsTest.php | 6 +++--- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 306600aa..57b8b92d 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -39,6 +39,14 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; + public function makeWithConfigOptions(string $handlerClass): AbstractConfigManager + { + $configManager = app(ConsumerConfigManager::class); + $configManager->set(['handler' => $handlerClass]); + + return $configManager; + } + public function make(array $options, array $arguments): AbstractConfigManager { $configName = $options['config_name'] ?? 'kafka'; diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index ccd8bb37..37dab14c 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -3,6 +3,7 @@ use Illuminate\Console\Command as BaseCommand; use Metamorphosis\AbstractConfigManager; +use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; @@ -23,18 +24,16 @@ class ConfigOptionsCommand extends BaseCommand */ protected $signature = 'kafka:consume-config-class {handler : handler.}'; - public function handle() + public function handle(Config $config) { -// $configManager = $config->make($this->option(), $this->argument()); + $configManager = $config->makeWithConfigOptions($this->argument()['handler']); -// $this->writeStartingConsumer($configManager); + $this->writeStartingConsumer($configManager); - $config = $this->argument()['handler']; - - $manager = Factory::make($this->argument()['handler']); + $manager = Factory::make($configManager); $runner = app(Runner::class, compact('manager')); - //$runner->run($manager->get('times')); + $runner->run(2); } private function writeStartingConsumer(AbstractConfigManager $configManager) diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 5f5017a0..4a45d8e0 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -24,13 +24,13 @@ public function testShouldRunAProducerMessagesWithConfigOptions(): void $this->haveAHandlerConfigured(); // I Expect That - //$this->myMessagesHaveBeenProduced(); + $this->myMessagesHaveBeenProduced(); // When I - //$this->haveSomeRandomMessageProduced(); + $this->haveSomeRandomMessageProduced(); // I Expect That - //$this->myMessagesHaveBeenLogged(); + $this->myMessagesHaveBeenLogged(); // When I $this->runTheConsumer(); From 1407d9f008c3fd39c5fe634e54a94819819994b4 Mon Sep 17 00:00:00 2001 From: David Franca Date: Mon, 31 Jan 2022 18:44:46 -0300 Subject: [PATCH 005/152] chore: replace config manager to config options --- src/Authentication/Factory.php | 16 +++---- src/Authentication/SASLAuthentication.php | 26 +++------- src/Authentication/SSLAuthentication.php | 26 +++------- src/Connectors/Consumer/Config.php | 4 +- .../Consumer/ConnectorInterface.php | 4 +- src/Connectors/Consumer/Factory.php | 48 ++++++++----------- src/Connectors/Consumer/HighLevel.php | 21 ++++---- src/Connectors/Consumer/LowLevel.php | 31 +++++------- src/Connectors/Producer/Connector.php | 9 ++-- src/Console/ConfigOptionsCommand.php | 23 +++++---- src/Consumers/LowLevel.php | 8 ++-- src/Producer.php | 30 ++++-------- src/Producer/Poll.php | 15 +++--- .../ProducerWithConfigOptionsTest.php | 1 + 14 files changed, 107 insertions(+), 155 deletions(-) diff --git a/src/Authentication/Factory.php b/src/Authentication/Factory.php index 67e3e1fa..c679209a 100644 --- a/src/Authentication/Factory.php +++ b/src/Authentication/Factory.php @@ -4,6 +4,8 @@ use Metamorphosis\AbstractConfigManager; use Metamorphosis\Exceptions\AuthenticationException; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\AuthInterface; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\Conf; class Factory @@ -14,9 +16,9 @@ class Factory private const TYPE_NONE = 'none'; - public static function authenticate(Conf $conf, AbstractConfigManager $configManager): void + public static function authenticate(Conf $conf, AuthInterface $configOptions): void { - $type = $configManager->get('auth.type'); + $type = $configOptions->getType(); switch ($type) { case null: case self::TYPE_NONE: @@ -24,17 +26,11 @@ public static function authenticate(Conf $conf, AbstractConfigManager $configMan break; case self::TYPE_SSL: - app( - SSLAuthentication::class, - compact('conf', 'configManager') - ); + app(SSLAuthentication::class, compact('conf', 'configOptions')); break; case self::TYPE_SASL_SSL: - app( - SASLAuthentication::class, - compact('conf', 'configManager') - ); + app(SASLAuthentication::class, compact('conf', 'configOptions')); break; default: diff --git a/src/Authentication/SASLAuthentication.php b/src/Authentication/SASLAuthentication.php index c0ebb42e..9b2fb0d0 100644 --- a/src/Authentication/SASLAuthentication.php +++ b/src/Authentication/SASLAuthentication.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Authentication; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\SaslSsl; use RdKafka\Conf; class SASLAuthentication implements AuthenticationInterface @@ -11,35 +11,23 @@ class SASLAuthentication implements AuthenticationInterface private AbstractConfigManager $configManager; - public function __construct(Conf $conf, AbstractConfigManager $configManager) + public function __construct(Conf $conf, SaslSsl $config) { $this->conf = $conf; - $this->configManager = $configManager; + $this->config = $config; $this->authenticate(); } private function authenticate(): void { - $this->conf->set( - 'security.protocol', - $this->configManager->get('auth.type') - ); + $this->conf->set('security.protocol', $this->config->getType()); // The mechanisms key is optional when configuring this kind of authentication // If the user does not specify the mechanism, the default will be 'PLAIN'. // But, to make config more clear, we are asking the user every time. - $this->conf->set( - 'sasl.mechanisms', - $this->configManager->get('auth.mechanisms') - ); - $this->conf->set( - 'sasl.username', - $this->configManager->get('auth.username') - ); - $this->conf->set( - 'sasl.password', - $this->configManager->get('auth.password') - ); + $this->conf->set('sasl.mechanisms', $this->config->getMechanisms()); + $this->conf->set('sasl.username', $this->config->getUsername()); + $this->conf->set('sasl.password', $this->config->getPassword()); } } diff --git a/src/Authentication/SSLAuthentication.php b/src/Authentication/SSLAuthentication.php index 246ab5e7..750552e8 100644 --- a/src/Authentication/SSLAuthentication.php +++ b/src/Authentication/SSLAuthentication.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Authentication; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\Ssl; use RdKafka\Conf; class SSLAuthentication implements AuthenticationInterface @@ -11,31 +11,19 @@ class SSLAuthentication implements AuthenticationInterface private AbstractConfigManager $configManager; - public function __construct(Conf $conf, AbstractConfigManager $configManager) + public function __construct(Conf $conf, Ssl $configSsl) { $this->conf = $conf; - $this->configManager = $configManager; + $this->configSsl = $configSsl; $this->authenticate(); } private function authenticate(): void { - $this->conf->set( - 'security.protocol', - $this->configManager->get('auth.type') - ); - $this->conf->set( - 'ssl.ca.location', - $this->configManager->get('auth.ca') - ); - $this->conf->set( - 'ssl.certificate.location', - $this->configManager->get('auth.certificate') - ); - $this->conf->set( - 'ssl.key.location', - $this->configManager->get('auth.key') - ); + $this->conf->set('security.protocol', $this->configSsl->getType()); + $this->conf->set('ssl.ca.location', $this->configSsl->getCa()); + $this->conf->set('ssl.certificate.location', $this->configSsl->getCertificate()); + $this->conf->set('ssl.key.location', $this->configSsl->getKey()); } } diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 57b8b92d..9216cf18 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -39,10 +39,10 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; - public function makeWithConfigOptions(string $handlerClass): AbstractConfigManager + public function makeWithConfigOptions(string $handlerClass, ?int $times = null): AbstractConfigManager { $configManager = app(ConsumerConfigManager::class); - $configManager->set(['handler' => $handlerClass]); + $configManager->set(['handler' => $handlerClass], ['times' => $times]); return $configManager; } diff --git a/src/Connectors/Consumer/ConnectorInterface.php b/src/Connectors/Consumer/ConnectorInterface.php index 3163fca6..738a6d0e 100644 --- a/src/Connectors/Consumer/ConnectorInterface.php +++ b/src/Connectors/Consumer/ConnectorInterface.php @@ -2,10 +2,10 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use Metamorphosis\Consumers\ConsumerInterface; interface ConnectorInterface { - public function getConsumer(bool $autoCommit, AbstractConfigManager $configManager): ConsumerInterface; + public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface; } diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index bc7258e9..f48fe56d 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -5,6 +5,7 @@ use Metamorphosis\AbstractConfigManager; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Middlewares\Handler\Dispatcher; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; /** * This factory will determine what kind of connector will be used. @@ -13,44 +14,33 @@ */ class Factory { - public static function make(AbstractConfigManager $configManager): Manager + public static function make(ConfigOptions $configOptions): Manager { - $autoCommit = $configManager->get('auto_commit', true); - $commitAsync = $configManager->get('commit_async', true); - - $consumer = self::getConsumer($autoCommit, $configManager); - $handler = app($configManager->get('handler')); - - $dispatcher = self::getMiddlewareDispatcher( - $configManager->middlewares() - ); - - return new Manager( - $consumer, - $handler, - $dispatcher, - $autoCommit, - $commitAsync - ); + $autoCommit = $configOptions->isAutoCommit(); + $commitAsync = $configOptions->isCommitASync(); + + $consumer = self::getConsumer($autoCommit, $configOptions); + $handler = app($configOptions->getHandler()); + + $dispatcher = self::getMiddlewareDispatcher($configOptions->getMiddlewares()); + + return new Manager($consumer, $handler, $dispatcher, $autoCommit, $commitAsync); } - public static function getConsumer(bool $autoCommit, AbstractConfigManager $configManager): ConsumerInterface + protected static function requiresPartition(ConfigOptions $configOptions): bool { - if (self::requiresPartition($configManager)) { - return app(LowLevel::class)->getConsumer( - $autoCommit, - $configManager - ); - } + $partition = $configOptions->getPartition(); - return app(HighLevel::class)->getConsumer($autoCommit, $configManager); + return !is_null($partition) && $partition >= 0; } - protected static function requiresPartition(AbstractConfigManager $configManager): bool + public static function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface { - $partition = $configManager->get('partition'); + if (self::requiresPartition($configOptions)) { + return app(LowLevel::class)->getConsumer($autoCommit, $configOptions); + } - return !is_null($partition) && $partition >= 0; + return app(HighLevel::class)->getConsumer($autoCommit, $configOptions); } private static function getMiddlewareDispatcher(array $middlewares): Dispatcher diff --git a/src/Connectors/Consumer/HighLevel.php b/src/Connectors/Consumer/HighLevel.php index 7a61a171..ede233a4 100644 --- a/src/Connectors/Consumer/HighLevel.php +++ b/src/Connectors/Consumer/HighLevel.php @@ -2,38 +2,39 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Authentication\Factory; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Consumers\HighLevel as HighLevelConsumer; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\Conf; use RdKafka\KafkaConsumer; class HighLevel implements ConnectorInterface { - public function getConsumer(bool $autoCommit, AbstractConfigManager $configManager): ConsumerInterface + public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface { - $conf = $this->getConf($configManager); + $conf = $this->getConf($configOptions); - $conf->set('group.id', $configManager->get('consumer_group')); - $conf->set('auto.offset.reset', $configManager->get('offset_reset')); + $conf->set('group.id', $configOptions->getConsumerGroup()); + $conf->set('auto.offset.reset', $configOptions->getOffsetReset()); if (!$autoCommit) { $conf->set('enable.auto.commit', 'false'); } $consumer = app(KafkaConsumer::class, ['conf' => $conf]); - $consumer->subscribe([$configManager->get('topic_id')]); - $timeout = $configManager->get('timeout'); + $consumer->subscribe([$configOptions->getTopicId()]); + $timeout = $configOptions->getTimeout(); return app(HighLevelConsumer::class, compact('consumer', 'timeout')); } - protected function getConf(AbstractConfigManager $configManager): Conf + protected function getConf(ConfigOptions $configOptions): Conf { $conf = resolve(Conf::class); - Factory::authenticate($conf, $configManager); + $broker = $configOptions->getBroker(); + Factory::authenticate($conf, $broker->getAuth()); - $conf->set('metadata.broker.list', $configManager->get('connections')); + $conf->set('metadata.broker.list', $broker->getConnections()); return $conf; } diff --git a/src/Connectors/Consumer/LowLevel.php b/src/Connectors/Consumer/LowLevel.php index 6c4d3993..9f47da26 100644 --- a/src/Connectors/Consumer/LowLevel.php +++ b/src/Connectors/Consumer/LowLevel.php @@ -6,50 +6,43 @@ use Metamorphosis\Authentication\Factory; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Consumers\LowLevel as LowLevelConsumer; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\Conf; use RdKafka\Consumer; use RdKafka\TopicConf; class LowLevel implements ConnectorInterface { - public function getConsumer(bool $autoCommit, AbstractConfigManager $configManager): ConsumerInterface + public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface { $conf = $this->getConf(); - $conf->set('group.id', $configManager->get('consumer_group')); + $conf->set('group.id', $configOptions->getConsumerGroup()); if (!$autoCommit) { $conf->set('enable.auto.commit', 'false'); } - Factory::authenticate($conf, $configManager); + $broker = $configOptions->getBroker(); + Factory::authenticate($conf, $broker->getAuth()); $consumer = new Consumer($conf); - $consumer->addBrokers($configManager->get('connections')); + $consumer->addBrokers($broker->getConnections()); - $topicConf = $this->getTopicConfigs($configManager); - $topicConsumer = $consumer->newTopic( - $configManager->get('topic_id'), - $topicConf - ); + $topicConf = $this->getTopicConfigs($configOptions); + $topicConsumer = $consumer->newTopic($configOptions->getTopicId(), $topicConf); - $topicConsumer->consumeStart( - $configManager->get('partition'), - $configManager->get('offset') - ); + $topicConsumer->consumeStart($configOptions->getPartition(), $configOptions->getOffset()); - return new LowLevelConsumer($topicConsumer, $configManager); + return new LowLevelConsumer($topicConsumer, $configOptions); } - protected function getTopicConfigs(AbstractConfigManager $configManager) + protected function getTopicConfigs(ConfigOptions $configOptions) { $topicConfig = new TopicConf(); // Set where to start consuming messages when there is no initial offset in // offset store or the desired offset is out of range. // 'smallest': start from the beginning - $topicConfig->set( - 'auto.offset.reset', - $configManager->get('offset_reset') - ); + $topicConfig->set('auto.offset.reset', $configOptions->getOffsetReset()); return $topicConfig; } diff --git a/src/Connectors/Producer/Connector.php b/src/Connectors/Producer/Connector.php index ebe696ea..68eafb07 100644 --- a/src/Connectors/Producer/Connector.php +++ b/src/Connectors/Producer/Connector.php @@ -2,8 +2,8 @@ namespace Metamorphosis\Connectors\Producer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Authentication\Factory; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; use Metamorphosis\TopicHandler\Producer\HandleableResponseInterface; use Metamorphosis\TopicHandler\Producer\HandlerInterface; use RdKafka\Conf; @@ -12,7 +12,7 @@ class Connector { - public function getProducerTopic(HandlerInterface $handler, AbstractConfigManager $configManager): KafkaProducer + public function getProducerTopic(HandlerInterface $handler, ConfigOptions $configOptions): KafkaProducer { $conf = resolve(Conf::class); @@ -29,9 +29,10 @@ function ($kafka, Message $message) use ($handler) { ); } - $conf->set('metadata.broker.list', $configManager->get('connections')); + $broker = $configOptions->getBroker(); + $conf->set('metadata.broker.list', $broker->getConnections()); - Factory::authenticate($conf, $configManager); + Factory::authenticate($conf, $broker->getAuth()); return app(KafkaProducer::class, compact('conf')); } diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index 37dab14c..4d37240b 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -6,6 +6,7 @@ use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; class ConfigOptionsCommand extends BaseCommand { @@ -22,25 +23,29 @@ class ConfigOptionsCommand extends BaseCommand /** * @var {inheritdoc} */ - protected $signature = 'kafka:consume-config-class {handler : handler.}'; + protected $signature = 'kafka:consume-config-class + {handler : handler.} + {--times= : Amount of messages to be consumed.}'; public function handle(Config $config) { - $configManager = $config->makeWithConfigOptions($this->argument()['handler']); + $consumerHandler = app($this->argument('handler')); - $this->writeStartingConsumer($configManager); + $configOptions = $consumerHandler->getConfigOptions(); - $manager = Factory::make($configManager); + $this->writeStartingConsumer($configOptions); + + $manager = Factory::make($configOptions); $runner = app(Runner::class, compact('manager')); - $runner->run(2); + $runner->run($this->option('times')); } - private function writeStartingConsumer(AbstractConfigManager $configManager) + private function writeStartingConsumer(ConfigOptions $configOptions) { - $text = 'Starting consumer for topic: '.$configManager->get('topic').PHP_EOL; - $text .= ' on consumer group: '.$configManager->get('consumer_group').PHP_EOL; - $text .= 'Connecting in '.$configManager->get('connections').PHP_EOL; + $text = 'Starting consumer for topic: '.$configOptions->getTopicId().PHP_EOL; + $text .= ' on consumer group: '.$configOptions->getConsumerGroup().PHP_EOL; + $text .= 'Connecting in '.$configOptions->getBroker()->getConnections().PHP_EOL; $text .= 'Running consumer..'; $this->output->writeln($text); diff --git a/src/Consumers/LowLevel.php b/src/Consumers/LowLevel.php index 0a1d762b..e4a31ea0 100644 --- a/src/Consumers/LowLevel.php +++ b/src/Consumers/LowLevel.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Consumers; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\ConsumerTopic; use RdKafka\Message; @@ -14,12 +14,12 @@ class LowLevel implements ConsumerInterface private ?int $timeout; - public function __construct(ConsumerTopic $consumer, AbstractConfigManager $configManager) + public function __construct(ConsumerTopic $consumer, ConfigOptions $configOptions) { $this->consumer = $consumer; - $this->partition = $configManager->get('partition'); - $this->timeout = $configManager->get('timeout'); + $this->partition = $configOptions->getPartition(); + $this->timeout = $configOptions->getTimeout(); } public function consume(): ?Message diff --git a/src/Producer.php b/src/Producer.php index 61c155a3..e1187cf0 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -7,6 +7,7 @@ use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\Middlewares\Handler\Producer as ProducerMiddleware; use Metamorphosis\Producer\Poll; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Metamorphosis\TopicHandler\Producer\HandlerInterface; @@ -31,32 +32,21 @@ public function produce(HandlerInterface $producerHandler): void public function build(HandlerInterface $producerHandler): Dispatcher { - $configManager = $this->getConfigManager($producerHandler); + $configOptions = $producerHandler->getConfigOptions(); - $middlewares = $configManager->middlewares(); - $middlewares[] = $this->getProducerMiddleware( - $producerHandler, - $configManager - ); + $middlewares = $configOptions->getMiddlewares(); + $middlewares[] = $this->getProducerMiddleware($producerHandler, $configOptions); return new Dispatcher($middlewares); } - public function getProducerMiddleware( - HandlerInterface $producerHandler, - AbstractConfigManager $configManager - ): ProducerMiddleware { - $producer = $this->connector->getProducerTopic( - $producerHandler, - $configManager - ); + public function getProducerMiddleware(HandlerInterface $producerHandler, ConfigOptions $configOptions): ProducerMiddleware + { + $producer = $this->connector->getProducerTopic($producerHandler, $configOptions); - $topic = $producer->newTopic($configManager->get('topic_id')); - $poll = app( - Poll::class, - ['producer' => $producer, 'configManager' => $configManager] - ); - $partition = $configManager->get('partition'); + $topic = $producer->newTopic($configOptions->getTopicId()); + $poll = app(Poll::class, ['producer' => $producer, 'configOptions' => $configOptions]); + $partition = $configOptions->getPartition(); return app( ProducerMiddleware::class, diff --git a/src/Producer/Poll.php b/src/Producer/Poll.php index afb8d081..d6a64626 100644 --- a/src/Producer/Poll.php +++ b/src/Producer/Poll.php @@ -3,6 +3,7 @@ namespace Metamorphosis\Producer; use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; use RdKafka\Producer; use RuntimeException; @@ -24,15 +25,13 @@ class Poll private Producer $producer; - public function __construct(Producer $producer, AbstractConfigManager $configManager) + public function __construct(Producer $producer, ConfigOptions $configOptions) { - $this->isAsync = $configManager->get('is_async'); - $this->maxPollRecords = $configManager->get('max_poll_records'); - $this->requiredAcknowledgment = $configManager->get( - 'required_acknowledgment' - ); - $this->maxFlushAttempts = $configManager->get('flush_attempts'); - $this->timeout = $configManager->get('timeout'); + $this->isAsync = $configOptions->isAsync(); + $this->maxPollRecords = $configOptions->getMaxPollRecords(); + $this->requiredAcknowledgment = $configOptions->isRequiredAcknowledgment(); + $this->maxFlushAttempts = $configOptions->getFlushAttempts(); + $this->timeout = $configOptions->getTimeout(); $this->producer = $producer; } diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 4a45d8e0..4e256a42 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -45,6 +45,7 @@ protected function runTheConsumer(): void 'kafka:consume-config-class', [ 'handler' => '\\App\\Kafka\\Consumers\\ConsumerOverride', + '--times' => 2, ] ); } From a57b4deb908865492efb683cfa516f385aa1a902 Mon Sep 17 00:00:00 2001 From: David Franca Date: Tue, 1 Feb 2022 09:46:24 -0300 Subject: [PATCH 006/152] chore: migrate Avro to use ConfigOptions\AvroSchema --- src/Avro/ClientFactory.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Avro/ClientFactory.php b/src/Avro/ClientFactory.php index 0e537e4c..b30d0c1b 100644 --- a/src/Avro/ClientFactory.php +++ b/src/Avro/ClientFactory.php @@ -3,29 +3,31 @@ namespace Metamorphosis\Avro; use GuzzleHttp\Client as GuzzleClient; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema; class ClientFactory { - public function make(AbstractConfigManager $configManager): CachedSchemaRegistryClient + const REQUEST_TIMEOUT = 2000; + + public function make(AvroSchema $avroSchema): CachedSchemaRegistryClient { - $guzzleHttp = $this->getGuzzleHttpClient($configManager); + $guzzleHttp = $this->getGuzzleHttpClient($avroSchema); $client = app(Client::class, ['client' => $guzzleHttp]); return app(CachedSchemaRegistryClient::class, compact('client')); } - private function getGuzzleHttpClient(AbstractConfigManager $configManager): GuzzleClient + private function getGuzzleHttpClient(AvroSchema $avroSchema): GuzzleClient { - $config = $configManager->get('request_options') ?: []; - $config['timeout'] = $configManager->get('timeout'); - $config['base_uri'] = $configManager->get('url'); + $config = $avroSchema->getRequestOptions(); + $config['timeout'] = self::REQUEST_TIMEOUT; + $config['base_uri'] = $avroSchema->getUrl(); $config['headers'] = array_merge( $this->getDefaultHeaders(), $config['headers'] ?? [] ); - $config['verify'] = $configManager->get('ssl_verify') ?? false; + $config['verify'] = $avroSchema->getRequestOptions()['ssl_verify'] ?? false; return app(GuzzleClient::class, compact('config')); } From 14046edf1d1f473d2a83b8026e97ae9d8453ccd4 Mon Sep 17 00:00:00 2001 From: David Franca Date: Tue, 1 Feb 2022 10:00:06 -0300 Subject: [PATCH 007/152] chore: migrate Aith factory to use config options --- src/Authentication/Factory.php | 2 -- src/Authentication/SASLAuthentication.php | 17 +++++---- src/Authentication/SSLAuthentication.php | 17 +++++---- src/Connectors/Consumer/Config.php | 5 ++- src/Console/ConsumerCommand.php | 1 + src/Middlewares/AvroSchemaDecoder.php | 19 +++++----- tests/Unit/Authentication/FactoryTest.php | 43 ++++++++++------------- 7 files changed, 52 insertions(+), 52 deletions(-) diff --git a/src/Authentication/Factory.php b/src/Authentication/Factory.php index c679209a..c8a8f38a 100644 --- a/src/Authentication/Factory.php +++ b/src/Authentication/Factory.php @@ -2,10 +2,8 @@ namespace Metamorphosis\Authentication; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Exceptions\AuthenticationException; use Metamorphosis\TopicHandler\ConfigOptions\Auth\AuthInterface; -use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\Conf; class Factory diff --git a/src/Authentication/SASLAuthentication.php b/src/Authentication/SASLAuthentication.php index 9b2fb0d0..3918558d 100644 --- a/src/Authentication/SASLAuthentication.php +++ b/src/Authentication/SASLAuthentication.php @@ -9,25 +9,28 @@ class SASLAuthentication implements AuthenticationInterface { private Conf $conf; - private AbstractConfigManager $configManager; + /** + * @var SaslSsl + */ + private $configOptions; - public function __construct(Conf $conf, SaslSsl $config) + public function __construct(Conf $conf, SaslSsl $configOptions) { $this->conf = $conf; - $this->config = $config; + $this->configOptions = $configOptions; $this->authenticate(); } private function authenticate(): void { - $this->conf->set('security.protocol', $this->config->getType()); + $this->conf->set('security.protocol', $this->configOptions->getType()); // The mechanisms key is optional when configuring this kind of authentication // If the user does not specify the mechanism, the default will be 'PLAIN'. // But, to make config more clear, we are asking the user every time. - $this->conf->set('sasl.mechanisms', $this->config->getMechanisms()); - $this->conf->set('sasl.username', $this->config->getUsername()); - $this->conf->set('sasl.password', $this->config->getPassword()); + $this->conf->set('sasl.mechanisms', $this->configOptions->getMechanisms()); + $this->conf->set('sasl.username', $this->configOptions->getUsername()); + $this->conf->set('sasl.password', $this->configOptions->getPassword()); } } diff --git a/src/Authentication/SSLAuthentication.php b/src/Authentication/SSLAuthentication.php index 750552e8..5dfc06e4 100644 --- a/src/Authentication/SSLAuthentication.php +++ b/src/Authentication/SSLAuthentication.php @@ -9,21 +9,24 @@ class SSLAuthentication implements AuthenticationInterface { private Conf $conf; - private AbstractConfigManager $configManager; + /** + * @var Ssl + */ + private $configOptions; - public function __construct(Conf $conf, Ssl $configSsl) + public function __construct(Conf $conf, Ssl $configOptions) { $this->conf = $conf; - $this->configSsl = $configSsl; + $this->configOptions = $configOptions; $this->authenticate(); } private function authenticate(): void { - $this->conf->set('security.protocol', $this->configSsl->getType()); - $this->conf->set('ssl.ca.location', $this->configSsl->getCa()); - $this->conf->set('ssl.certificate.location', $this->configSsl->getCertificate()); - $this->conf->set('ssl.key.location', $this->configSsl->getKey()); + $this->conf->set('security.protocol', $this->configOptions->getType()); + $this->conf->set('ssl.ca.location', $this->configOptions->getCa()); + $this->conf->set('ssl.certificate.location', $this->configOptions->getCertificate()); + $this->conf->set('ssl.key.location', $this->configOptions->getKey()); } } diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 9216cf18..fa6d1398 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -6,6 +6,7 @@ use Metamorphosis\Connectors\AbstractConfig; use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\ConfigurationException; +use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; /** * This class is responsible for handling all configuration made on the @@ -75,10 +76,8 @@ public function make(array $options, array $arguments): AbstractConfigManager ); $this->validate(array_merge($config, $override)); - $configManager = app(ConsumerConfigManager::class); - $configManager->set($config, $override); - return $configManager; + return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); } /** diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index c4bdab76..bd80175e 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -7,6 +7,7 @@ use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; +use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; class ConsumerCommand extends BaseCommand { diff --git a/src/Middlewares/AvroSchemaDecoder.php b/src/Middlewares/AvroSchemaDecoder.php index 4ae626c9..8d185fdd 100644 --- a/src/Middlewares/AvroSchemaDecoder.php +++ b/src/Middlewares/AvroSchemaDecoder.php @@ -3,28 +3,29 @@ namespace Metamorphosis\Middlewares; use Closure; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Avro\ClientFactory; use Metamorphosis\Avro\Serializer\MessageDecoder; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema; class AvroSchemaDecoder implements MiddlewareInterface { private MessageDecoder $decoder; - private AbstractConfigManager $configManager; + /** + * @var AvroSchema + */ + private $avroSchema; - public function __construct(AbstractConfigManager $configManager, ClientFactory $factory) + public function __construct(AvroSchema $avroSchema, ClientFactory $factory) { - $this->configManager = $configManager; - if (!$this->configManager->get('url')) { - throw new ConfigurationException( - "Avro schema url not found, it's required to use AvroSchemaDecoder Middleware" - ); + $this->avroSchema = $avroSchema; + if (!$this->avroSchema->getUrl()) { + throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaDecoder Middleware"); } - $this->decoder = new MessageDecoder($factory->make($configManager)); + $this->decoder = new MessageDecoder($factory->make($avroSchema)); } public function process(RecordInterface $record, Closure $next) diff --git a/tests/Unit/Authentication/FactoryTest.php b/tests/Unit/Authentication/FactoryTest.php index 53999f8a..717abe16 100644 --- a/tests/Unit/Authentication/FactoryTest.php +++ b/tests/Unit/Authentication/FactoryTest.php @@ -3,8 +3,11 @@ namespace Tests\Unit\Authentication; use Metamorphosis\Authentication\Factory; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\AuthenticationException; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\AuthInterface; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\SaslSsl; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\Ssl; +use Mockery as m; use RdKafka\Conf; use Tests\LaravelTestCase; @@ -13,15 +16,7 @@ class FactoryTest extends LaravelTestCase public function testItMakesSslAuthenticationClass(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'auth' => [ - 'type' => 'ssl', - 'ca' => 'path/to/ca', - 'certificate' => 'path/to/certificate', - 'key' => 'path/to/key', - ], - ]); + $configOptionsSsl = new Ssl('path/to/ca', 'path/to/certificate', 'path/to/key'); $conf = new Conf(); $expected = [ 'security.protocol' => 'ssl', @@ -31,7 +26,7 @@ public function testItMakesSslAuthenticationClass(): void ]; // Actions - Factory::authenticate($conf, $configManager); + Factory::authenticate($conf, $configOptionsSsl); // Assertions $this->assertArraySubset($expected, $conf->dump()); @@ -40,15 +35,11 @@ public function testItMakesSslAuthenticationClass(): void public function testItMakesSASLAuthenticationClass(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'auth' => [ - 'type' => 'sasl_ssl', - 'mechanisms' => 'PLAIN', - 'username' => 'some-username', - 'password' => 'some-password', - ], - ]); + $configOptionsSaslSsl = new SaslSsl( + 'PLAIN', + 'some-username', + 'some-password' + ); $conf = new Conf(); $expected = [ 'security.protocol' => 'sasl_ssl', @@ -58,7 +49,7 @@ public function testItMakesSASLAuthenticationClass(): void ]; // Actions - Factory::authenticate($conf, $configManager); + Factory::authenticate($conf, $configOptionsSaslSsl); // Assertions $this->assertArraySubset($expected, $conf->dump()); @@ -67,13 +58,17 @@ public function testItMakesSASLAuthenticationClass(): void public function testItThrowsExceptionWhenInvalidProtocolIsPassed(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set(['auth' => ['type' => 'some-invalid-type']]); + $invalidAuth = m::mock(AuthInterface::class); $conf = new Conf(); + // Expectations + $invalidAuth->expects() + ->getType() + ->andReturn('some-invalid-type'); + $this->expectException(AuthenticationException::class); // Actions - Factory::authenticate($conf, $configManager); + Factory::authenticate($conf, $invalidAuth); } } From 9b3c02884909ccbac041f1d474bcb99efedd5d20 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Tue, 1 Feb 2022 10:28:02 -0300 Subject: [PATCH 008/152] chore: remove configManager and add Sasl config class --- .../Authentication/SASLAuthenticationTest.php | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/Unit/Authentication/SASLAuthenticationTest.php b/tests/Unit/Authentication/SASLAuthenticationTest.php index ce1078b3..f9e9a474 100644 --- a/tests/Unit/Authentication/SASLAuthenticationTest.php +++ b/tests/Unit/Authentication/SASLAuthenticationTest.php @@ -3,7 +3,7 @@ namespace Tests\Unit\Authentication; use Metamorphosis\Authentication\SASLAuthentication; -use Metamorphosis\ConsumerConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\SaslSsl; use RdKafka\Conf; use Tests\LaravelTestCase; @@ -12,16 +12,9 @@ class SASLAuthenticationTest extends LaravelTestCase public function testItShouldValidateAuthenticationConfigurations(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'auth' => [ - 'type' => 'sasl_ssl', - 'mechanisms' => 'PLAIN', - 'username' => 'some-username', - 'password' => 'some-password', - ], - ]); + $configSaslSsl = new SaslSsl('PLAIN', 'some-username', 'some-password'); $conf = new Conf(); + $expected = [ 'security.protocol' => 'sasl_ssl', 'sasl.username' => 'some-username', @@ -30,7 +23,7 @@ public function testItShouldValidateAuthenticationConfigurations(): void ]; // Actions - new SASLAuthentication($conf, $configManager); + new SASLAuthentication($conf, $configSaslSsl); // Assertions $this->assertArraySubset($expected, $conf->dump()); From 2e937f631017064e4bd06df54c187736d11cd29a Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Tue, 1 Feb 2022 10:31:05 -0300 Subject: [PATCH 009/152] chore: remove configManager and add SSL config class --- .../Unit/Authentication/SSLAuthenticationTest.php | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/Unit/Authentication/SSLAuthenticationTest.php b/tests/Unit/Authentication/SSLAuthenticationTest.php index 11c93691..35a9168e 100644 --- a/tests/Unit/Authentication/SSLAuthenticationTest.php +++ b/tests/Unit/Authentication/SSLAuthenticationTest.php @@ -3,7 +3,7 @@ namespace Tests\Unit\Authentication; use Metamorphosis\Authentication\SSLAuthentication; -use Metamorphosis\ConsumerConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\Ssl; use RdKafka\Conf; use Tests\LaravelTestCase; @@ -12,16 +12,8 @@ class SSLAuthenticationTest extends LaravelTestCase public function testItShouldValidateAuthenticationConfigurations(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'auth' => [ - 'type' => 'ssl', - 'ca' => 'path/to/ca', - 'certificate' => 'path/to/certificate', - 'key' => 'path/to/key', - ], - ]); $conf = new Conf(); + $configSsl = new Ssl('path/to/ca', 'path/to/certificate', 'path/to/key'); $expected = [ 'security.protocol' => 'ssl', 'ssl.ca.location' => 'path/to/ca', @@ -30,7 +22,7 @@ public function testItShouldValidateAuthenticationConfigurations(): void ]; // Actions - new SSLAuthentication($conf, $configManager); + new SSLAuthentication($conf, $configSsl); // Assertions $this->assertArraySubset($expected, $conf->dump()); From 96bfd526718c66caeb59db6fb10a7898a35b1214 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Tue, 1 Feb 2022 12:04:03 -0300 Subject: [PATCH 010/152] chore: fix code standard with phpcbf --- src/Connectors/Consumer/ConnectorInterface.php | 2 +- src/Connectors/Consumer/Factory.php | 1 - src/Connectors/Consumer/LowLevel.php | 1 - src/Console/ConfigOptionsCommand.php | 1 - src/Console/ConsumerCommand.php | 1 - src/Producer/Poll.php | 1 - 6 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Connectors/Consumer/ConnectorInterface.php b/src/Connectors/Consumer/ConnectorInterface.php index 738a6d0e..f9a7934d 100644 --- a/src/Connectors/Consumer/ConnectorInterface.php +++ b/src/Connectors/Consumer/ConnectorInterface.php @@ -2,8 +2,8 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use Metamorphosis\Consumers\ConsumerInterface; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; interface ConnectorInterface { diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index f48fe56d..893e9030 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; diff --git a/src/Connectors/Consumer/LowLevel.php b/src/Connectors/Consumer/LowLevel.php index 9f47da26..e897d467 100644 --- a/src/Connectors/Consumer/LowLevel.php +++ b/src/Connectors/Consumer/LowLevel.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Authentication\Factory; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Consumers\LowLevel as LowLevelConsumer; diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index 4d37240b..eb4c1071 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Console; use Illuminate\Console\Command as BaseCommand; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index bd80175e..c4bdab76 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -7,7 +7,6 @@ use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; -use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; class ConsumerCommand extends BaseCommand { diff --git a/src/Producer/Poll.php b/src/Producer/Poll.php index d6a64626..4c3b9c81 100644 --- a/src/Producer/Poll.php +++ b/src/Producer/Poll.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Producer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; use RdKafka\Producer; use RuntimeException; From 865e5438a525581099c0715ff33ad76185b142c4 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 7 Mar 2022 18:11:01 -0300 Subject: [PATCH 011/152] chore: debug config options --- src/Connectors/Consumer/Config.php | 3 +- src/Console/ConsumerCommand.php | 24 ++++------- src/Consumer.php | 4 +- .../Producer/AbstractProducer.php | 15 ++++--- tests/Integration/ConsumerTest.php | 11 ++--- tests/Integration/Dummies/MessageProducer.php | 14 +------ tests/Integration/ProducerTest.php | 41 ++++++++++++------- tests/Unit/ProducerTest.php | 16 ++++---- 8 files changed, 64 insertions(+), 64 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index fa6d1398..d14770fb 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -6,6 +6,7 @@ use Metamorphosis\Connectors\AbstractConfig; use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\ConfigurationException; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer; use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; /** @@ -48,7 +49,7 @@ public function makeWithConfigOptions(string $handlerClass, ?int $times = null): return $configManager; } - public function make(array $options, array $arguments): AbstractConfigManager + public function make(array $options, array $arguments): Consumer { $configName = $options['config_name'] ?? 'kafka'; $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index c4bdab76..5c233db4 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -3,10 +3,10 @@ namespace Metamorphosis\Console; use Illuminate\Console\Command as BaseCommand; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer; class ConsumerCommand extends BaseCommand { @@ -38,27 +38,21 @@ class ConsumerCommand extends BaseCommand public function handle(Config $config) { - $configManager = $config->make($this->option(), $this->argument()); + $consumer = $config->make($this->option(), $this->argument()); - $this->writeStartingConsumer($configManager); + $this->writeStartingConsumer($consumer); - $manager = Factory::make($configManager); + $manager = Factory::make($consumer); $runner = app(Runner::class, compact('manager')); - $runner->run($configManager->get('times')); + $runner->run($this->option('times')); } - private function writeStartingConsumer(AbstractConfigManager $configManager) + private function writeStartingConsumer(Consumer $consumer) { - $text = 'Starting consumer for topic: ' . $configManager->get( - 'topic' - ) . PHP_EOL; - $text .= ' on consumer group: ' . $configManager->get( - 'consumer_group' - ) . PHP_EOL; - $text .= 'Connecting in ' . $configManager->get( - 'connections' - ) . PHP_EOL; + $text = 'Starting consumer for topic: '.$consumer->getTopicId().PHP_EOL; + $text .= ' on consumer group: '.$consumer->getConsumerGroup().PHP_EOL; + $text .= 'Connecting in '.$consumer->getBroker()->getConnections().PHP_EOL; $text .= 'Running consumer..'; $this->output->writeln($text); diff --git a/src/Consumer.php b/src/Consumer.php index ba04c783..1d35a0c1 100644 --- a/src/Consumer.php +++ b/src/Consumer.php @@ -19,8 +19,8 @@ public function __construct(ConsumerConfigManager $configManager, ConsumerConfig { $configManager->set($configOptions->toArray()); - $this->consumer = Factory::getConsumer(true, $configManager); - $this->dispatcher = new Dispatcher($configManager->middlewares()); + $this->consumer = Factory::getConsumer(true, $configOptions); + $this->dispatcher = new Dispatcher($configOptions->getMiddlewares()); } public function consume(): ?RecordInterface diff --git a/src/TopicHandler/Producer/AbstractProducer.php b/src/TopicHandler/Producer/AbstractProducer.php index e95a6f63..946db3ce 100644 --- a/src/TopicHandler/Producer/AbstractProducer.php +++ b/src/TopicHandler/Producer/AbstractProducer.php @@ -4,7 +4,7 @@ use Metamorphosis\Exceptions\JsonException; use Metamorphosis\Record\ProducerRecord; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptionsProducer; +use Metamorphosis\TopicHandler\ConfigOptions\Producer; class AbstractProducer implements HandlerInterface { @@ -15,18 +15,21 @@ class AbstractProducer implements HandlerInterface protected ?string $key; - private ConfigOptionsProducer $configOptions; + /** + * @var Producer + */ + protected $producer; - public function __construct($record, ConfigOptionsProducer $configOptions, ?string $key = null) + public function __construct($record, Producer $producer, string $key = null) { $this->record = $record; $this->key = $key; - $this->configOptions = $configOptions; + $this->producer = $producer; } - public function getConfigOptions(): ConfigOptionsProducer + public function getConfigOptions(): Producer { - return $this->configOptions; + return $this->producer; } public function getRecord() diff --git a/tests/Integration/ConsumerTest.php b/tests/Integration/ConsumerTest.php index 8c107499..1d4397e8 100644 --- a/tests/Integration/ConsumerTest.php +++ b/tests/Integration/ConsumerTest.php @@ -59,16 +59,13 @@ public function testItShouldSetup(): void $saleOrderDispatcher = Metamorphosis::build($messageProducer); $saleOrderDispatcher->handle($messageProducer->createRecord()); - $consumer = $this->app->make( - Consumer::class, - ['configOptions' => $consumerConfigOptions] - ); - $expected = ['id' => 'MESSAGE_ID']; + $consumer = $this->app->make(Consumer::class, ['configOptions' => $consumerConfigOptions]); + $expected = '{"id":"MESSAGE_ID"}'; // Actions - $result = $consumer->consume(); + $result = $consumer->consume()->getPayload(); // Assertions - $this->assertSame($expected, $result->getPayload()); + $this->assertSame($expected, $result); } } diff --git a/tests/Integration/Dummies/MessageProducer.php b/tests/Integration/Dummies/MessageProducer.php index 22aac6f8..38fb8090 100644 --- a/tests/Integration/Dummies/MessageProducer.php +++ b/tests/Integration/Dummies/MessageProducer.php @@ -3,22 +3,12 @@ namespace Tests\Integration\Dummies; use Illuminate\Support\Facades\Log; -use Metamorphosis\TopicHandler\Producer\AbstractHandler; +use Metamorphosis\TopicHandler\Producer\AbstractProducer; use RdKafka\Message; use RuntimeException; -class MessageProducer extends AbstractHandler +class MessageProducer extends AbstractProducer { - public string $topic = 'default'; - - public function __construct($record, string $topic, ?string $key = null, ?int $partition = null) - { - $this->record = $record; - $this->topic = $topic; - $this->key = $key ?? 'recordId123'; - $this->partition = $partition; - } - public function success(Message $message): void { Log::info('Record successfully sent to broker.', [ diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index f7d13966..1689fe87 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -5,6 +5,9 @@ use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; use Metamorphosis\Facades\Metamorphosis; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Tests\Integration\Dummies\MessageConsumer; use Tests\Integration\Dummies\MessageProducer; use Tests\LaravelTestCase; @@ -17,11 +20,13 @@ class ProducerTest extends LaravelTestCase protected string $secondLowLevelMessage; + /** + * @group runProducer + */ public function testShouldRunAProducerAndReceiveMessagesWithAHighLevelConsumer(): void { // Given That I $this->haveAConsumerHandlerConfigured(); - $this->haveNoPartitionConfigured(); $this->haveSomeRandomMessagesProduced(); // I Expect That @@ -128,13 +133,9 @@ protected function runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingT private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $producer = app( - MessageProducer::class, - [ - 'record' => $this->highLevelMessage, - 'topic' => 'default', - ] - ); + $configOptionsProducer = $this->createConfigOptionsProducer('kafka-test'); + //$producer = app(MessageProducer::class, ['record' => $this->highLevelMessage, 'configOptions'=> $configOptionsProducer]); + $producer = new MessageProducer($this->highLevelMessage, $configOptionsProducer, 'recordId123'); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -142,8 +143,10 @@ private function haveSomeRandomMessagesProduced(): void private function produceRecordMessage(string $record): string { - $topic = 'low_level'; - $producer = app(MessageProducer::class, compact('record', 'topic')); + $configOptionsProducer = $this->createConfigOptionsProducer('low_level'); + $producer = new MessageProducer($record, $configOptionsProducer, 'recordId123'); + //$producer = app(MessageProducer::class, ['record'=>$record, 'configOptions' => $a]); + //$producer->topic = 'low_level'; Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -183,10 +186,20 @@ private function haveFourProducedMessages(): void $this->produceRecordMessage($this->secondLowLevelMessage); } - private function haveNoPartitionConfigured(): void - { - config( - ['kafka.topics.default.consumer.consumer_groups.test-consumer-group.partition' => -1] + private function createConfigOptionsProducer(string $topicId = 'kafka-test'): ProducerConfigOptions + { + $brokerOptions = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( + $topicId, + $brokerOptions, + null, + null, + [], + 20000, + false, + true, + 10, + 500 ); } } diff --git a/tests/Unit/ProducerTest.php b/tests/Unit/ProducerTest.php index 5a93dc8c..31650aef 100644 --- a/tests/Unit/ProducerTest.php +++ b/tests/Unit/ProducerTest.php @@ -12,7 +12,6 @@ use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; use Metamorphosis\TopicHandler\ConfigOptions\Broker; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; -use Metamorphosis\TopicHandler\Producer\AbstractHandler; use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Mockery as m; use RdKafka\Producer as KafkaProducer; @@ -38,7 +37,7 @@ public function testItShouldProduceRecordAsArrayThroughMiddlewareQueue(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class ($record, $topic) extends AbstractHandler { + $producerHandler = new class($record, $topic) extends AbstractProducer { }; // Expectations @@ -95,7 +94,7 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class ($record, $topic) extends AbstractHandler { + $producerHandler = new class($record, $topic) extends AbstractProducer { }; // Expectations @@ -128,7 +127,10 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void ->withAnyArgs(); // Actions - $producer->produce($producerHandler); + $result = $producer->produce($producerHandler); + + // Assertions + $this->assertNull($result); } public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): void @@ -147,7 +149,7 @@ public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): vo $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class ($record, $topic) extends AbstractHandler { + $producerHandler = new class($record, $topic) extends AbstractProducer { }; // Expectations @@ -223,7 +225,7 @@ public function testShouldBuildDispatcher(): void $producerTopic = m::mock(ProducerTopic::class); $configManager = m::mock(ProducerConfigManager::class); - $producerHandler = new class ($record, $topic) extends AbstractHandler { + $producerHandler = new class($record, $topic) extends AbstractProducer { }; // Expectations @@ -298,7 +300,7 @@ public function testShouldBuildDispatcherWithConfigOptions(): void $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); $broker = new Broker($connections, new None()); $configOptions = new ProducerConfigOptions($topicId, $broker); - $producerHandler = new class ($record, $configOptions) extends AbstractProducer { + $producerHandler = new class($record, $configOptions) extends AbstractProducer { }; // Expectations From a89f497b38110d3722e7a91568eb0bb98c9ac3a7 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 9 Mar 2022 09:45:27 -0300 Subject: [PATCH 012/152] chore: debug config options --- src/Connectors/Consumer/Config.php | 19 ++++-- src/ConsumerConfigManager.php | 7 +-- tests/Unit/AbstractConfigManagerTest.php | 75 ---------------------- tests/Unit/ConsumerConfigManagerTest.php | 79 ------------------------ 4 files changed, 16 insertions(+), 164 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index d14770fb..d2c97dd3 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -2,12 +2,14 @@ namespace Metamorphosis\Connectors\Consumer; +use InvalidArgumentException; use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\AbstractConfig; use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\TopicHandler\ConfigOptions\Consumer; use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; +use Metamorphosis\TopicHandler\Consumer\AbstractHandler; /** * This class is responsible for handling all configuration made on the @@ -41,12 +43,21 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; - public function makeWithConfigOptions(string $handlerClass, ?int $times = null): AbstractConfigManager + /** + * @param string $handlerClass + * @param int|null $times + * @return Consumer + */ + public function makeWithConfigOptions(string $handlerClass, ?int $times = null): ?Consumer { - $configManager = app(ConsumerConfigManager::class); - $configManager->set(['handler' => $handlerClass], ['times' => $times]); + /** @var AbstractHandler */ + $handler = app($handlerClass); + $configOptions = $handler->getConfigOptions(); + if(is_null($configOptions)){ + throw new InvalidArgumentException('Handler class cannot be null'); + } - return $configManager; + return $configOptions; } public function make(array $options, array $arguments): Consumer diff --git a/src/ConsumerConfigManager.php b/src/ConsumerConfigManager.php index 36d31c40..88207c97 100644 --- a/src/ConsumerConfigManager.php +++ b/src/ConsumerConfigManager.php @@ -22,12 +22,7 @@ public function set(array $config, ?array $commandConfig = null): void $this->remove('middlewares'); foreach ($middlewares as $middleware) { - $this->middlewares[] = is_string($middleware) - ? app( - $middleware, - ['configManager' => $this] - ) - : $middleware; + $this->middlewares[] = is_string($middleware) ? app($middleware) : $middleware; } if (!$consumerHandler) { diff --git a/tests/Unit/AbstractConfigManagerTest.php b/tests/Unit/AbstractConfigManagerTest.php index 7cf86812..e69de29b 100644 --- a/tests/Unit/AbstractConfigManagerTest.php +++ b/tests/Unit/AbstractConfigManagerTest.php @@ -1,75 +0,0 @@ -instance( - AbstractHandler::class, - m::mock(AbstractHandler::class) - ); - $config = [ - 'middlewares' => [], - 'handler' => AbstractHandler::class, - 'broker' => [ - 'default' => [ - 'connections' => 'kafka:9092', - ], - ], - 'topic_id' => 'kafka-test', - ]; - $broker = new Broker('kafka:9092', new None()); - $configOptions = new ConsumerConfigOptions( - 'kafka-override', - $broker, - '\App\Kafka\Consumers\ConsumerExample', - null, - null, - 'default', - null, - [MiddlewareDummy::class], - 200, - false, - true - ); - - $expected = [ - 'topic_id' => 'kafka-override', - 'connections' => 'kafka:9092', - 'auth' => null, - 'timeout' => 200, - 'handler' => '\App\Kafka\Consumers\ConsumerExample', - 'partition' => -1, - 'offset' => null, - 'consumer_group' => 'default', - 'auto_commit' => false, - 'commit_async' => true, - 'offset_reset' => 'smallest', - ]; - - $configManager = new ConsumerConfigManager(); - - // Expectations - $handler->expects() - ->getConfigOptions() - ->andReturn($configOptions); - - // Actions - $configManager->set($config); - - // Expectations - $this->assertEquals($expected, $configManager->get()); - } -} diff --git a/tests/Unit/ConsumerConfigManagerTest.php b/tests/Unit/ConsumerConfigManagerTest.php index 347aea6c..e69de29b 100644 --- a/tests/Unit/ConsumerConfigManagerTest.php +++ b/tests/Unit/ConsumerConfigManagerTest.php @@ -1,79 +0,0 @@ -instance( - AbstractHandler::class, - m::mock(AbstractHandler::class) - ); - $config = [ - 'middlewares' => [], - 'handler' => AbstractHandler::class, - 'broker' => [ - 'default' => [ - 'connections' => 'kafka:9092', - ], - ], - 'topic_id' => 'kafka-test', - ]; - $broker = new Broker('kafka:9092', new None()); - $configOptions = new ConsumerConfigOptions( - 'kafka-override', - $broker, - '\App\Kafka\Consumers\ConsumerExample', - null, - null, - 'default', - null, - [MiddlewareDummy::class], - 200, - false - ); - - $expected = [ - 'topic_id' => 'kafka-override', - 'connections' => 'kafka:9092', - 'auth' => null, - 'timeout' => 1000, - 'handler' => '\App\Kafka\Consumers\ConsumerExample', - 'partition' => -1, - 'offset' => null, - 'consumer_group' => 'default', - 'auto_commit' => false, - 'commit_async' => true, - 'offset_reset' => 'smallest', - 'times' => 2, - ]; - - $configManager = new ConsumerConfigManager(); - - $commandConfig = [ - 'timeout' => 1000, - 'times' => 2, - ]; - - // Expectations - $handler->expects() - ->getConfigOptions() - ->andReturn($configOptions); - - // Actions - $configManager->set($config, $commandConfig); - - // Expectations - $this->assertEquals($expected, $configManager->get()); - } -} From de86b27060d65c36a6a9e214a9f35fbbab74bf2d Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 10 Mar 2022 17:23:57 -0300 Subject: [PATCH 013/152] chore: create interfaces for classes --- src/Avro/CachedSchemaRegistryClient.php | 4 ++-- src/Avro/Client.php | 2 +- src/Avro/Serializer/Encoders/SchemaId.php | 8 +++++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Avro/CachedSchemaRegistryClient.php b/src/Avro/CachedSchemaRegistryClient.php index c91e643c..97706cd5 100644 --- a/src/Avro/CachedSchemaRegistryClient.php +++ b/src/Avro/CachedSchemaRegistryClient.php @@ -5,7 +5,7 @@ use AvroSchemaParseException; use RuntimeException; -class CachedSchemaRegistryClient +class CachedSchemaRegistryClient implements CachedSchemaRegistryClientInterface { private Client $client; @@ -20,7 +20,7 @@ class CachedSchemaRegistryClient */ private array $subjectVersionToSchema = []; - public function __construct(Client $client) + public function __construct(AvroClientInterface $client) { $this->client = $client; } diff --git a/src/Avro/Client.php b/src/Avro/Client.php index e0b714ac..02516b70 100644 --- a/src/Avro/Client.php +++ b/src/Avro/Client.php @@ -5,7 +5,7 @@ use GuzzleHttp\Client as GuzzleHttp; use Psr\Http\Message\ResponseInterface; -class Client +class Client implements AvroClientInterface { private GuzzleHttp $client; diff --git a/src/Avro/Serializer/Encoders/SchemaId.php b/src/Avro/Serializer/Encoders/SchemaId.php index ea428472..fd796333 100644 --- a/src/Avro/Serializer/Encoders/SchemaId.php +++ b/src/Avro/Serializer/Encoders/SchemaId.php @@ -6,6 +6,7 @@ use AvroIODatumWriter; use AvroStringIO; use Metamorphosis\Avro\CachedSchemaRegistryClient; +use Metamorphosis\Avro\CachedSchemaRegistryClientInterface; use Metamorphosis\Avro\Schema; use Metamorphosis\Avro\Serializer\SchemaFormats; @@ -13,7 +14,7 @@ class SchemaId implements EncoderInterface { private CachedSchemaRegistryClient $registry; - public function __construct(CachedSchemaRegistryClient $registry) + public function __construct(CachedSchemaRegistryClientInterface $registry) { $this->registry = $registry; } @@ -41,4 +42,9 @@ public function encode(Schema $schema, $message): string return $io->string(); } + + public function getRegistry() + { + return $this->registry; + } } From 7294e95d5527adb3f5c5edbd3c914c61bd00c8e4 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 10 Mar 2022 17:25:28 -0300 Subject: [PATCH 014/152] chore: update calls from configManager to configOptions --- src/Middlewares/AvroSchemaMixedEncoder.php | 29 ++++++++++------------ 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/Middlewares/AvroSchemaMixedEncoder.php b/src/Middlewares/AvroSchemaMixedEncoder.php index 4df0beb7..11534359 100644 --- a/src/Middlewares/AvroSchemaMixedEncoder.php +++ b/src/Middlewares/AvroSchemaMixedEncoder.php @@ -3,12 +3,12 @@ namespace Metamorphosis\Middlewares; use Closure; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Avro\CachedSchemaRegistryClient; use Metamorphosis\Avro\ClientFactory; use Metamorphosis\Avro\Serializer\Encoders\SchemaId; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptionsProducer; /** * Fetches a schema for a topic by subject and version (currently only 'latest') @@ -21,30 +21,27 @@ class AvroSchemaMixedEncoder implements MiddlewareInterface private CachedSchemaRegistryClient $schemaRegistry; - private AbstractConfigManager $configManager; + /** + * @var ConfigOptionsProducer + */ + private $configOptionsProducer; - public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, AbstractConfigManager $configManager) + public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, ConfigOptionsProducer $configOptionsProducer) { - if (!$configManager->get('url')) { - throw new ConfigurationException( - "Avro schema url not found, it's required to use AvroSchemaEncoder Middleware" - ); + if (!$configOptionsProducer->getAvroSchema()->getUrl()) { + throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaEncoder Middleware"); } - $schemaRegistry = $factory->make($configManager); +// $schemaRegistry = $factory->make($configOptionsProducer->getAvroSchema()); $this->schemaIdEncoder = $schemaIdEncoder; - $this->schemaRegistry = $schemaRegistry; - $this->configManager = $configManager; + $this->schemaRegistry = $schemaIdEncoder->getRegistry(); + $this->configOptionsProducer = $configOptionsProducer; } public function process(RecordInterface $record, Closure $next) { - $topic = $this->configManager->get('topic_id'); - $schema = $this->schemaRegistry->getBySubjectAndVersion( - "{$topic}-value", - 'latest' - ); - + $topic = $this->configOptionsProducer->getTopicId(); + $schema = $this->schemaRegistry->getBySubjectAndVersion("{$topic}-value", 'latest'); $arrayPayload = json_decode($record->getPayload(), true); $encodedPayload = $this->schemaIdEncoder->encode( $schema, From 3af47115ba4cbd8a519f6e31906165d7e50fd7b8 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 10 Mar 2022 17:26:51 -0300 Subject: [PATCH 015/152] chore: add interfaces --- src/Avro/AvroClientInterface.php | 6 ++++++ src/Avro/CachedSchemaRegistryClientInterface.php | 7 +++++++ 2 files changed, 13 insertions(+) create mode 100644 src/Avro/AvroClientInterface.php create mode 100644 src/Avro/CachedSchemaRegistryClientInterface.php diff --git a/src/Avro/AvroClientInterface.php b/src/Avro/AvroClientInterface.php new file mode 100644 index 00000000..d7617023 --- /dev/null +++ b/src/Avro/AvroClientInterface.php @@ -0,0 +1,6 @@ + Date: Thu, 10 Mar 2022 17:30:37 -0300 Subject: [PATCH 016/152] chore: add AvroSchemaMixedEncoderTest --- .../AvroSchemaMixedEncoderTest.php | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php diff --git a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php new file mode 100644 index 00000000..9412bd4b --- /dev/null +++ b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php @@ -0,0 +1,69 @@ +getSchemaTest(); + $configOptionsProducer = $this->createProducer(); + + $record = new ProducerRecord($schemaTest, 'kafka-test'); + $cachedSchemaRegistryClient = m::mock(CachedSchemaRegistryClient::class); + + $schema = (new Schema())->parse($schemaTest, '123'); + $clientFactory = new ClientFactory(); + $schemaIdEncoder = m::mock(SchemaId::class, [$cachedSchemaRegistryClient])->makePartial(); + + $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder($schemaIdEncoder, $clientFactory, $configOptionsProducer); + + //expect + $cachedSchemaRegistryClient->shouldReceive('getBySubjectAndVersion')->andReturn($schema); + $schemaIdEncoder->shouldReceive('encode')->andReturn('string'); + + //act + $avroSchemaMixedEncoder->process($record, $closure); + + //assert + $this->assertInstanceOf(AvroSchemaMixedEncoder::class, $avroSchemaMixedEncoder); + } + + private function createProducer() + { + $brokerOptions = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( + 'kafka-test', + $brokerOptions, + null, + new AvroSchema('http://url.teste', []), + [], + 20000, + false, + true, + 10, + 500 + ); + } + + private function getSchemaTest(): string + { + return file_get_contents(__DIR__.'/../fixtures/schemas/sales_price.avsc'); + } +} From 61dfbab8f5ae578a2c37d2c1510e14c3d77b5db1 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 21 Mar 2022 15:25:33 -0300 Subject: [PATCH 017/152] chore: remove inutilized interfaces --- src/Avro/CachedSchemaRegistryClient.php | 4 ++-- src/Avro/Client.php | 2 +- src/Avro/Serializer/Encoders/SchemaId.php | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Avro/CachedSchemaRegistryClient.php b/src/Avro/CachedSchemaRegistryClient.php index 97706cd5..c91e643c 100644 --- a/src/Avro/CachedSchemaRegistryClient.php +++ b/src/Avro/CachedSchemaRegistryClient.php @@ -5,7 +5,7 @@ use AvroSchemaParseException; use RuntimeException; -class CachedSchemaRegistryClient implements CachedSchemaRegistryClientInterface +class CachedSchemaRegistryClient { private Client $client; @@ -20,7 +20,7 @@ class CachedSchemaRegistryClient implements CachedSchemaRegistryClientInterface */ private array $subjectVersionToSchema = []; - public function __construct(AvroClientInterface $client) + public function __construct(Client $client) { $this->client = $client; } diff --git a/src/Avro/Client.php b/src/Avro/Client.php index 02516b70..e0b714ac 100644 --- a/src/Avro/Client.php +++ b/src/Avro/Client.php @@ -5,7 +5,7 @@ use GuzzleHttp\Client as GuzzleHttp; use Psr\Http\Message\ResponseInterface; -class Client implements AvroClientInterface +class Client { private GuzzleHttp $client; diff --git a/src/Avro/Serializer/Encoders/SchemaId.php b/src/Avro/Serializer/Encoders/SchemaId.php index fd796333..75d08216 100644 --- a/src/Avro/Serializer/Encoders/SchemaId.php +++ b/src/Avro/Serializer/Encoders/SchemaId.php @@ -6,7 +6,6 @@ use AvroIODatumWriter; use AvroStringIO; use Metamorphosis\Avro\CachedSchemaRegistryClient; -use Metamorphosis\Avro\CachedSchemaRegistryClientInterface; use Metamorphosis\Avro\Schema; use Metamorphosis\Avro\Serializer\SchemaFormats; @@ -14,7 +13,7 @@ class SchemaId implements EncoderInterface { private CachedSchemaRegistryClient $registry; - public function __construct(CachedSchemaRegistryClientInterface $registry) + public function __construct(CachedSchemaRegistryClient $registry) { $this->registry = $registry; } From ca175afbdaaba6ddb6fafff77a88d5298e2cf16c Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 21 Mar 2022 15:37:01 -0300 Subject: [PATCH 018/152] chore: add method to get consumerGroupId --- src/Connectors/Consumer/Config.php | 54 +++++++++++++----------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index d2c97dd3..aace08a9 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -64,22 +64,11 @@ public function make(array $options, array $arguments): Consumer { $configName = $options['config_name'] ?? 'kafka'; $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); - $consumerConfig = $this->getConsumerConfig( - $topicConfig, - $arguments['consumer_group'] - ); - $brokerConfig = $this->getBrokerConfig( - $configName, - $topicConfig['broker'] - ); - $schemaConfig = $this->getSchemaConfig( - $configName, - $arguments['topic'] - ); - $override = array_merge( - $this->filterValues($options), - $this->filterValues($arguments) - ); + $consumerGroupId = $this->getConsumerGroup($topicConfig, $arguments['consumer_group']); + $consumerConfig = $this->getConsumerConfig($topicConfig, $arguments, $consumerGroupId); + $brokerConfig = $this->getBrokerConfig($configName, $topicConfig['broker']); + $schemaConfig = $this->getSchemaConfig($configName, $arguments['topic']); + $override = array_merge($this->filterValues($options), $this->filterValues($arguments)); $config = array_merge( $topicConfig, $brokerConfig, @@ -89,6 +78,9 @@ public function make(array $options, array $arguments): Consumer $this->validate(array_merge($config, $override)); + $topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'] = $consumerConfig['partition']; + $topicConfig['consumer_group'] = $consumerGroupId; + return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); } @@ -110,31 +102,31 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } - private function getConsumerConfig(array $topicConfig, ?string $consumerGroupId = null): array + private function getConsumerConfig(array $topicConfig, array $arguments, string $consumerGroupId): array { - if ( - !$consumerGroupId && 1 === count( - $topicConfig['consumer']['consumer_groups'] - ) - ) { - $consumerGroupId = current( - array_keys($topicConfig['consumer']['consumer_groups']) - ); + $consumerConfig = $topicConfig['consumer']['consumer_groups'][$consumerGroupId] ?? null; + if (!$consumerConfig) { + throw new ConfigurationException("Consumer group '{$consumerGroupId}' not found"); } - $consumerGroupId = $consumerGroupId ?? 'default'; - $consumerConfig = $topicConfig['consumer']['consumer_groups'][$consumerGroupId] ?? null; $consumerConfig['consumer_group'] = $consumerGroupId; - if (!$consumerConfig) { - throw new ConfigurationException( - "Consumer group '{$consumerGroupId}' not found" - ); + if(isset($arguments['partition'])){ + $consumerConfig['partition'] = $arguments['partition']; } return $consumerConfig; } + private function getConsumerGroup(array $topicConfig, ?string $consumerGroupId): string + { + if (!$consumerGroupId && 1 === count($topicConfig['consumer']['consumer_groups'])) { + $consumerGroupId = current(array_keys($topicConfig['consumer']['consumer_groups'])); + } + + return $consumerGroupId ?? 'default'; + } + private function getMiddlewares(string $configName, array $topicConfig): array { return array_merge( From 8bddb6b634a9046be3cfadeeb34837731c559b7c Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 21 Mar 2022 15:58:27 -0300 Subject: [PATCH 019/152] chore: replace configManager by configOptions --- src/Connectors/Consumer/Config.php | 17 +- src/Connectors/Consumer/Factory.php | 12 +- src/Connectors/Producer/Connector.php | 6 +- src/Consumers/LowLevel.php | 8 +- src/Middlewares/AvroSchemaDecoder.php | 8 +- src/Middlewares/AvroSchemaMixedEncoder.php | 18 +- src/Producer.php | 23 +- src/Producer/Poll.php | 14 +- .../Factories/ConsumerFactory.php | 14 +- tests/Integration/ConsumerTest.php | 2 +- tests/Integration/ProducerTest.php | 31 ++- tests/Unit/Connectors/Consumer/ConfigTest.php | 8 +- .../Unit/Connectors/Consumer/FactoryTest.php | 75 +++--- .../Connectors/Consumer/HighLevelTest.php | 26 +- .../Unit/Connectors/Consumer/LowLevelTest.php | 27 +- .../Connectors/Producer/ConnectorTest.php | 47 ++-- tests/Unit/Consumers/LowLevelTest.php | 19 +- .../Middlewares/AvroSchemaDecoderTest.php | 70 ++++++ .../AvroSchemaMixedEncoderTest.php | 97 ++++--- tests/Unit/Producer/PollTest.php | 69 +++-- tests/Unit/ProducerTest.php | 238 +++++------------- .../Factories/ConsumerFactoryTest.php | 1 + 22 files changed, 430 insertions(+), 400 deletions(-) create mode 100644 tests/Unit/Middlewares/AvroSchemaDecoderTest.php diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index aace08a9..4db8f76a 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -3,13 +3,10 @@ namespace Metamorphosis\Connectors\Consumer; use InvalidArgumentException; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\AbstractConfig; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\TopicHandler\ConfigOptions\Consumer; use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; -use Metamorphosis\TopicHandler\Consumer\AbstractHandler; /** * This class is responsible for handling all configuration made on the @@ -43,17 +40,11 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; - /** - * @param string $handlerClass - * @param int|null $times - * @return Consumer - */ public function makeWithConfigOptions(string $handlerClass, ?int $times = null): ?Consumer { - /** @var AbstractHandler */ $handler = app($handlerClass); $configOptions = $handler->getConfigOptions(); - if(is_null($configOptions)){ + if (is_null($configOptions)) { throw new InvalidArgumentException('Handler class cannot be null'); } @@ -77,8 +68,10 @@ public function make(array $options, array $arguments): Consumer ); $this->validate(array_merge($config, $override)); + if (isset($topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'])) { + $topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'] = $consumerConfig['partition']; + } - $topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'] = $consumerConfig['partition']; $topicConfig['consumer_group'] = $consumerGroupId; return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); @@ -111,7 +104,7 @@ private function getConsumerConfig(array $topicConfig, array $arguments, string $consumerConfig['consumer_group'] = $consumerGroupId; - if(isset($arguments['partition'])){ + if (isset($arguments['partition'])) { $consumerConfig['partition'] = $arguments['partition']; } diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index 893e9030..6a7c61aa 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -4,16 +4,20 @@ use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Middlewares\Handler\Dispatcher; -use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; /** * This factory will determine what kind of connector will be used. * Basically, if the user pass --partition and --offset as argument * means that we will use the low level approach. */ + +/** + * TODO Rename this class to ConsumerFactory, it will improve semantics + */ class Factory { - public static function make(ConfigOptions $configOptions): Manager + public static function make(ConsumerConfigOptions $configOptions): Manager { $autoCommit = $configOptions->isAutoCommit(); $commitAsync = $configOptions->isCommitASync(); @@ -26,14 +30,14 @@ public static function make(ConfigOptions $configOptions): Manager return new Manager($consumer, $handler, $dispatcher, $autoCommit, $commitAsync); } - protected static function requiresPartition(ConfigOptions $configOptions): bool + protected static function requiresPartition(ConsumerConfigOptions $configOptions): bool { $partition = $configOptions->getPartition(); return !is_null($partition) && $partition >= 0; } - public static function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface + public static function getConsumer(bool $autoCommit, ConsumerConfigOptions $configOptions): ConsumerInterface { if (self::requiresPartition($configOptions)) { return app(LowLevel::class)->getConsumer($autoCommit, $configOptions); diff --git a/src/Connectors/Producer/Connector.php b/src/Connectors/Producer/Connector.php index 68eafb07..1109a598 100644 --- a/src/Connectors/Producer/Connector.php +++ b/src/Connectors/Producer/Connector.php @@ -3,7 +3,7 @@ namespace Metamorphosis\Connectors\Producer; use Metamorphosis\Authentication\Factory; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Metamorphosis\TopicHandler\Producer\HandleableResponseInterface; use Metamorphosis\TopicHandler\Producer\HandlerInterface; use RdKafka\Conf; @@ -12,7 +12,7 @@ class Connector { - public function getProducerTopic(HandlerInterface $handler, ConfigOptions $configOptions): KafkaProducer + public function getProducerTopic(HandlerInterface $handler, ProducerConfigOptions $producerConfigOptions): KafkaProducer { $conf = resolve(Conf::class); @@ -29,7 +29,7 @@ function ($kafka, Message $message) use ($handler) { ); } - $broker = $configOptions->getBroker(); + $broker = $producerConfigOptions->getBroker(); $conf->set('metadata.broker.list', $broker->getConnections()); Factory::authenticate($conf, $broker->getAuth()); diff --git a/src/Consumers/LowLevel.php b/src/Consumers/LowLevel.php index e4a31ea0..e9fee56c 100644 --- a/src/Consumers/LowLevel.php +++ b/src/Consumers/LowLevel.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Consumers; -use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; use RdKafka\ConsumerTopic; use RdKafka\Message; @@ -14,12 +14,12 @@ class LowLevel implements ConsumerInterface private ?int $timeout; - public function __construct(ConsumerTopic $consumer, ConfigOptions $configOptions) + public function __construct(ConsumerTopic $consumer, ConsumerConfigOptions $consumerConfigOptions) { $this->consumer = $consumer; - $this->partition = $configOptions->getPartition(); - $this->timeout = $configOptions->getTimeout(); + $this->partition = $consumerConfigOptions->getPartition(); + $this->timeout = $consumerConfigOptions->getTimeout(); } public function consume(): ?Message diff --git a/src/Middlewares/AvroSchemaDecoder.php b/src/Middlewares/AvroSchemaDecoder.php index 8d185fdd..1aeffcff 100644 --- a/src/Middlewares/AvroSchemaDecoder.php +++ b/src/Middlewares/AvroSchemaDecoder.php @@ -8,6 +8,7 @@ use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; class AvroSchemaDecoder implements MiddlewareInterface { @@ -18,14 +19,13 @@ class AvroSchemaDecoder implements MiddlewareInterface */ private $avroSchema; - public function __construct(AvroSchema $avroSchema, ClientFactory $factory) + public function __construct(ClientFactory $factory, ConsumerConfigOptions $consumerConfigOptions) { - $this->avroSchema = $avroSchema; - if (!$this->avroSchema->getUrl()) { + if (!$consumerConfigOptions->getAvroSchema()->getUrl()) { throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaDecoder Middleware"); } - $this->decoder = new MessageDecoder($factory->make($avroSchema)); + $this->decoder = new MessageDecoder($factory->make($consumerConfigOptions->getAvroSchema())); } public function process(RecordInterface $record, Closure $next) diff --git a/src/Middlewares/AvroSchemaMixedEncoder.php b/src/Middlewares/AvroSchemaMixedEncoder.php index 11534359..6969a80e 100644 --- a/src/Middlewares/AvroSchemaMixedEncoder.php +++ b/src/Middlewares/AvroSchemaMixedEncoder.php @@ -8,7 +8,7 @@ use Metamorphosis\Avro\Serializer\Encoders\SchemaId; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptionsProducer; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; /** * Fetches a schema for a topic by subject and version (currently only 'latest') @@ -22,25 +22,25 @@ class AvroSchemaMixedEncoder implements MiddlewareInterface private CachedSchemaRegistryClient $schemaRegistry; /** - * @var ConfigOptionsProducer + * @var ProducerConfigOptions */ - private $configOptionsProducer; + private $producerConfigOptions; - public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, ConfigOptionsProducer $configOptionsProducer) + public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, ProducerConfigOptions $producerConfigOptions) { - if (!$configOptionsProducer->getAvroSchema()->getUrl()) { + if (!$producerConfigOptions->getAvroSchema()->getUrl()) { throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaEncoder Middleware"); } -// $schemaRegistry = $factory->make($configOptionsProducer->getAvroSchema()); + $schemaRegistry = $factory->make($producerConfigOptions->getAvroSchema()); $this->schemaIdEncoder = $schemaIdEncoder; - $this->schemaRegistry = $schemaIdEncoder->getRegistry(); - $this->configOptionsProducer = $configOptionsProducer; + $this->schemaRegistry = $schemaRegistry; + $this->producerConfigOptions = $producerConfigOptions; } public function process(RecordInterface $record, Closure $next) { - $topic = $this->configOptionsProducer->getTopicId(); + $topic = $this->producerConfigOptions->getTopicId(); $schema = $this->schemaRegistry->getBySubjectAndVersion("{$topic}-value", 'latest'); $arrayPayload = json_decode($record->getPayload(), true); $encodedPayload = $this->schemaIdEncoder->encode( diff --git a/src/Producer.php b/src/Producer.php index e1187cf0..ee277fdf 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -7,7 +7,7 @@ use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\Middlewares\Handler\Producer as ProducerMiddleware; use Metamorphosis\Producer\Poll; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Metamorphosis\TopicHandler\Producer\HandlerInterface; @@ -32,21 +32,26 @@ public function produce(HandlerInterface $producerHandler): void public function build(HandlerInterface $producerHandler): Dispatcher { - $configOptions = $producerHandler->getConfigOptions(); + $producerConfigOptions = $producerHandler->getConfigOptions(); - $middlewares = $configOptions->getMiddlewares(); - $middlewares[] = $this->getProducerMiddleware($producerHandler, $configOptions); + $middlewares = $producerConfigOptions->getMiddlewares(); + + foreach ($middlewares as &$middleware) { + $middleware = is_string($middleware) ? app($middleware, ['producerConfigOptions' => $producerConfigOptions]) : $middleware; + } + + $middlewares[] = $this->getProducerMiddleware($producerHandler, $producerConfigOptions); return new Dispatcher($middlewares); } - public function getProducerMiddleware(HandlerInterface $producerHandler, ConfigOptions $configOptions): ProducerMiddleware + public function getProducerMiddleware(HandlerInterface $producerHandler, ProducerConfigOptions $producerConfigOptions): ProducerMiddleware { - $producer = $this->connector->getProducerTopic($producerHandler, $configOptions); + $producer = $this->connector->getProducerTopic($producerHandler, $producerConfigOptions); - $topic = $producer->newTopic($configOptions->getTopicId()); - $poll = app(Poll::class, ['producer' => $producer, 'configOptions' => $configOptions]); - $partition = $configOptions->getPartition(); + $topic = $producer->newTopic($producerConfigOptions->getTopicId()); + $poll = app(Poll::class, ['producer' => $producer, 'producerConfigOptions' => $producerConfigOptions]); + $partition = $producerConfigOptions->getPartition(); return app( ProducerMiddleware::class, diff --git a/src/Producer/Poll.php b/src/Producer/Poll.php index 4c3b9c81..c07508ec 100644 --- a/src/Producer/Poll.php +++ b/src/Producer/Poll.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Producer; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use RdKafka\Producer; use RuntimeException; @@ -24,13 +24,13 @@ class Poll private Producer $producer; - public function __construct(Producer $producer, ConfigOptions $configOptions) + public function __construct(Producer $producer, ProducerConfigOptions $producerConfigOptions) { - $this->isAsync = $configOptions->isAsync(); - $this->maxPollRecords = $configOptions->getMaxPollRecords(); - $this->requiredAcknowledgment = $configOptions->isRequiredAcknowledgment(); - $this->maxFlushAttempts = $configOptions->getFlushAttempts(); - $this->timeout = $configOptions->getTimeout(); + $this->isAsync = $producerConfigOptions->isAsync(); + $this->maxPollRecords = $producerConfigOptions->getMaxPollRecords(); + $this->requiredAcknowledgment = $producerConfigOptions->isRequiredAcknowledgment(); + $this->maxFlushAttempts = $producerConfigOptions->getFlushAttempts(); + $this->timeout = $producerConfigOptions->getTimeout(); $this->producer = $producer; } diff --git a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php index 103b882e..7bd58c93 100644 --- a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php +++ b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php @@ -22,20 +22,16 @@ public static function make( private static function getConsumerGroupConfig(array $topicData): array { $topicData['topicId'] = $topicData['topic_id']; + $topicData['consumerGroup'] = $topicData['consumer_group']; - $consumer = current($topicData['consumer']); - $topicData['consumerGroup'] = key($consumer); + $consumerGroup = $topicData['consumerGroup']; + $consumer = current($topicData['consumer'])[$consumerGroup]; - return array_merge( - $topicData, - self::convertConfigAttributes($consumer) - ); + return array_merge_recursive($topicData, self::convertConfigAttributes($consumer)); } - private static function convertConfigAttributes(array $topic): array + private static function convertConfigAttributes(array $consumerConfig): array { - $consumerConfig = current($topic); - if (isset($consumerConfig['auto_commit'])) { $consumerConfig['autoCommit'] = $consumerConfig['auto_commit']; } diff --git a/tests/Integration/ConsumerTest.php b/tests/Integration/ConsumerTest.php index 1d4397e8..7088e46b 100644 --- a/tests/Integration/ConsumerTest.php +++ b/tests/Integration/ConsumerTest.php @@ -51,7 +51,7 @@ public function testItShouldSetup(): void MessageProducerWithConfigOptions::class, [ 'record' => ['id' => 'MESSAGE_ID'], - 'configOptions' => $producerConfigOptions, + 'producer' => $producerConfigOptions, 'key' => 1, ] ); diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 1689fe87..9780d74c 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -27,6 +27,7 @@ public function testShouldRunAProducerAndReceiveMessagesWithAHighLevelConsumer() { // Given That I $this->haveAConsumerHandlerConfigured(); + $this->haveAConsumerPartitionConfigured(); $this->haveSomeRandomMessagesProduced(); // I Expect That @@ -69,6 +70,11 @@ protected function haveAConsumerHandlerConfigured(): void ); } + protected function haveAConsumerPartitionConfigured(): void + { + config(['kafka.topics.default.consumer.consumer_groups.test-consumer-group.partition' => -1]); + } + protected function runTheConsumer(): void { $this->artisan( @@ -133,9 +139,9 @@ protected function runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingT private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $configOptionsProducer = $this->createConfigOptionsProducer('kafka-test'); - //$producer = app(MessageProducer::class, ['record' => $this->highLevelMessage, 'configOptions'=> $configOptionsProducer]); - $producer = new MessageProducer($this->highLevelMessage, $configOptionsProducer, 'recordId123'); + $producerConfigOptions = $this->createProducerConfigOptions('kafka-test'); +// $producer = app(MessageProducer::class, ['record' => $this->highLevelMessage, 'producer'=> $producerConfigOptions]); + $producer = new MessageProducer($this->highLevelMessage, $producerConfigOptions, 'recordId123'); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -143,10 +149,8 @@ private function haveSomeRandomMessagesProduced(): void private function produceRecordMessage(string $record): string { - $configOptionsProducer = $this->createConfigOptionsProducer('low_level'); - $producer = new MessageProducer($record, $configOptionsProducer, 'recordId123'); - //$producer = app(MessageProducer::class, ['record'=>$record, 'configOptions' => $a]); - //$producer->topic = 'low_level'; + $producerConfigOptions = $this->createProducerConfigOptions('low_level'); + $producer = new MessageProducer($record, $producerConfigOptions, 'recordId123'); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -186,20 +190,15 @@ private function haveFourProducedMessages(): void $this->produceRecordMessage($this->secondLowLevelMessage); } - private function createConfigOptionsProducer(string $topicId = 'kafka-test'): ProducerConfigOptions + private function createProducerConfigOptions(string $topicId = 'kafka-test'): ProducerConfigOptions { - $brokerOptions = new Broker('kafka:9092', new None()); + $broker = new Broker('kafka:9092', new None()); return new ProducerConfigOptions( $topicId, - $brokerOptions, + $broker, null, null, - [], - 20000, - false, - true, - 10, - 500 + [] ); } } diff --git a/tests/Unit/Connectors/Consumer/ConfigTest.php b/tests/Unit/Connectors/Consumer/ConfigTest.php index 9e1c26f9..539b6ea0 100644 --- a/tests/Unit/Connectors/Consumer/ConfigTest.php +++ b/tests/Unit/Connectors/Consumer/ConfigTest.php @@ -84,10 +84,10 @@ public function testShouldValidateConsumerConfig(): void ]); // Actions - $configManager = $config->make($options, $arguments); + $configManager = $config->makeWithConfigOptions(ConsumerHandlerDummy::class); // Assertions - $this->assertArraySubset($expected, $configManager->get()); + $this->assertArraySubset($expected, $configManager->toArray()); } public function testShouldNotSetRuntimeConfigWhenOptionsIsInvalid(): void @@ -109,7 +109,7 @@ public function testShouldNotSetRuntimeConfigWhenOptionsIsInvalid(): void $configManager = $config->make($options, $arguments); // Assertions - $this->assertEmpty($configManager->get()); + $this->assertEmpty($configManager->toArray()); } public function testShouldNotSetRuntimeConfigWhenKafkaConfigIsInvalid(): void @@ -132,6 +132,6 @@ public function testShouldNotSetRuntimeConfigWhenKafkaConfigIsInvalid(): void $configManager = $config->make($options, $arguments); // Assertions - $this->assertEmpty($configManager->get()); + $this->assertEmpty($configManager->toArray()); } } diff --git a/tests/Unit/Connectors/Consumer/FactoryTest.php b/tests/Unit/Connectors/Consumer/FactoryTest.php index a90471d6..50a158f2 100644 --- a/tests/Unit/Connectors/Consumer/FactoryTest.php +++ b/tests/Unit/Connectors/Consumer/FactoryTest.php @@ -11,48 +11,6 @@ class FactoryTest extends LaravelTestCase { - public function testItMakesManagerWithLowLevelConsumer(): void - { - // Set - $config = new Config(); - $configManager = $config->make( - ['timeout' => 61], - ['topic' => 'topic_key', 'consumer_group' => 'with-partition'] - ); - $manager = Factory::make($configManager); - - // Assertions - $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); - } - - public function testItMakesManagerWithLowLevelConsumerWhenPartitionIsNotValid(): void - { - // Set - $config = new Config(); - $configManager = $config->make( - ['timeout' => 61], - ['topic' => 'topic_key', 'consumer_group' => 'with-partition', 'partition' => -1] - ); - $manager = Factory::make($configManager); - - // Assertions - $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); - } - - public function testItMakesHighLevelClass(): void - { - // Set - $config = new Config(); - $configManager = $config->make( - ['timeout' => 61], - ['topic' => 'topic_key', 'consumer_group' => 'without-partition'] - ); - $manager = Factory::make($configManager); - - // Assertions - $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); - } - protected function setUp(): void { parent::setUp(); @@ -88,4 +46,37 @@ protected function setUp(): void ], ]); } + + public function testItMakesManagerWithLowLevelConsumer(): void + { + // Set + $config = new Config(); + $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); + } + + public function testItMakesManagerWithLowLevelConsumerWhenPartitionIsNotValid(): void + { + // Set + $config = new Config(); + $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition', 'partition' => -1]); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); + } + + public function testItMakesHighLevelClass(): void + { + // Set + $config = new Config(); + $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'without-partition']); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); + } } diff --git a/tests/Unit/Connectors/Consumer/HighLevelTest.php b/tests/Unit/Connectors/Consumer/HighLevelTest.php index 52d58fc4..97ab8e91 100644 --- a/tests/Unit/Connectors/Consumer/HighLevelTest.php +++ b/tests/Unit/Connectors/Consumer/HighLevelTest.php @@ -3,8 +3,11 @@ namespace Tests\Unit\Connectors\Consumer; use Metamorphosis\Connectors\Consumer\HighLevel; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\HighLevel as HighLevelConsumer; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; use Tests\LaravelTestCase; class HighLevelTest extends LaravelTestCase @@ -12,19 +15,20 @@ class HighLevelTest extends LaravelTestCase public function testItShouldMakeConnectorSetup(): void { // Set - $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'connections' => $connections, - 'consumer_group' => 'some-group', - 'topic_id' => 'some_topic', - 'offset_reset' => 'earliest', - 'timeout' => 1000, - ]); $connector = new HighLevel(); + $brokerOptions = new Broker('kafka:9092', new None()); + $consumerConfigOptions = new ConsumerConfigOptions( + 'kafka-test', + $brokerOptions, + null, + 1, + 0, + 'some-group', + new AvroSchemaConfigOptions('http://url.teste') + ); // Actions - $result = $connector->getConsumer(false, $configManager); + $result = $connector->getConsumer(false, $consumerConfigOptions); // Assertions $this->assertInstanceOf(HighLevelConsumer::class, $result); diff --git a/tests/Unit/Connectors/Consumer/LowLevelTest.php b/tests/Unit/Connectors/Consumer/LowLevelTest.php index 039d096c..21c12291 100644 --- a/tests/Unit/Connectors/Consumer/LowLevelTest.php +++ b/tests/Unit/Connectors/Consumer/LowLevelTest.php @@ -3,8 +3,11 @@ namespace Tests\Unit\Connectors\Consumer; use Metamorphosis\Connectors\Consumer\LowLevel; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\LowLevel as LowLevelConsumer; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; use Tests\LaravelTestCase; class LowLevelTest extends LaravelTestCase @@ -12,20 +15,20 @@ class LowLevelTest extends LaravelTestCase public function testItShouldMakeConnectorSetup(): void { // Set - $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'connections' => $connections, - 'consumer_group' => 'some-group', - 'topic' => 'some_topic', - 'offset_reset' => 'earliest', - 'offset' => 0, - 'partition' => 1, - ]); $connector = new LowLevel(); + $brokerOptions = new Broker('kafka:9092', new None()); + $consumerConfigOptions = new ConsumerConfigOptions( + 'kafka-test', + $brokerOptions, + null, + 1, + 0, + 'some-group', + new AvroSchemaConfigOptions('http://url.teste') + ); // Actions - $result = $connector->getConsumer(true, $configManager); + $result = $connector->getConsumer(true, $consumerConfigOptions); // Assertions $this->assertInstanceOf(LowLevelConsumer::class, $result); diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index e0cb6123..f456dfb7 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -3,7 +3,8 @@ namespace Tests\Unit\Connectors\Producer; use Metamorphosis\Connectors\Producer\Connector; -use Metamorphosis\ProducerConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Metamorphosis\TopicHandler\Producer\HandleableResponseInterface; @@ -27,12 +28,12 @@ public function testItShouldMakeSetup(): void KafkaProducer::class, m::mock(KafkaProducer::class) ); - $configManager = m::mock(ProducerConfigManager::class); - $configOptions = m::mock(ProducerConfigOptions::class); + + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); - $handler = new class ('record', $configOptions) extends AbstractProducer implements HandleableResponseInterface { - /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */ + $handler = new class('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface { public function success(Message $message): void { } @@ -49,18 +50,14 @@ public function failed(Message $message): void ->withAnyArgs(); $conf->expects() - ->set('metadata.broker.list', 'kafka:9092'); - - $configManager->expects() - ->get('connections') - ->andReturn('kafka:9092'); + ->set('metadata.broker.list', 0); - $configManager->expects() - ->get('auth.type') - ->andReturn('none'); + $producerConfigOptions->expects() + ->getBroker() + ->andReturn($broker); // Actions - $result = $connector->getProducerTopic($handler, $configManager); + $result = $connector->getProducerTopic($handler, $producerConfigOptions); // Assertions $this->assertInstanceOf(KafkaProducer::class, $result); @@ -77,12 +74,12 @@ public function testItShouldMakeSetupWithoutHandleResponse(): void KafkaProducer::class, m::mock(KafkaProducer::class) ); - $configManager = m::mock(ProducerConfigManager::class); - $configOptions = m::mock(ProducerConfigOptions::class); + + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); - $handler = new class ('record', $configOptions) extends AbstractProducer implements HandlerInterface { - /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */ + $handler = new class('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface { public function success(Message $message): void { } @@ -98,18 +95,14 @@ public function failed(Message $message): void ->never(); $conf->expects() - ->set('metadata.broker.list', 'kafka:9092'); - - $configManager->expects() - ->get('connections') - ->andReturn('kafka:9092'); + ->set('metadata.broker.list', 0); - $configManager->expects() - ->get('auth.type') - ->andReturn('none'); + $producerConfigOptions->expects() + ->getBroker() + ->andReturn($broker); // Actions - $result = $connector->getProducerTopic($handler, $configManager); + $result = $connector->getProducerTopic($handler, $producerConfigOptions); // Assertions $this->assertInstanceOf(KafkaProducer::class, $result); diff --git a/tests/Unit/Consumers/LowLevelTest.php b/tests/Unit/Consumers/LowLevelTest.php index 6b35f993..cc8bb988 100644 --- a/tests/Unit/Consumers/LowLevelTest.php +++ b/tests/Unit/Consumers/LowLevelTest.php @@ -4,6 +4,10 @@ use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\LowLevel; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; use Mockery as m; use RdKafka\ConsumerTopic; use RdKafka\Message; @@ -19,10 +23,23 @@ public function testItShouldConsume(): void $configManager = new ConsumerConfigManager(); $configManager->set(compact('timeout', 'partition')); + $brokerOptions = new Broker('kafka:9092', new None()); + $consumerConfigOptions = new ConsumerConfigOptions( + 'kafka-test', + $brokerOptions, + null, + $partition, + null, + '', + new AvroSchemaConfigOptions('http://url.teste'), + [], + $timeout + ); + $consumerTopic = m::mock(ConsumerTopic::class); $message = new Message(); - $lowLevelConsumer = new LowLevel($consumerTopic, $configManager); + $lowLevelConsumer = new LowLevel($consumerTopic, $consumerConfigOptions); // Expectations $consumerTopic->expects() diff --git a/tests/Unit/Middlewares/AvroSchemaDecoderTest.php b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php new file mode 100644 index 00000000..6c8d31b0 --- /dev/null +++ b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php @@ -0,0 +1,70 @@ +getAvroSchema(); + $avroSchema = new AvroSchema('string'); + $decoder = m::mock(Schema::class); + $clientFactory = m::mock(ClientFactory::class); + $cachedSchemaRegistryClient = m::mock(CachedSchemaRegistryClient::class); + $expected = 'my awesome message'; + + $message = new Message(); + $message->payload = "\x01\x00\x00\x00\fmy-topic-key\x00\x00\x00\x05\$my awesome message"; + $message->err = 0; + + $closure = Closure::fromCallable(function ($producerRecord) { + return $producerRecord; + }); + + $consumerRecord = new ConsumerRecord($message); + + // Expectations + $clientFactory->expects() + ->make($avroSchemaConfigOptions) + ->andReturn($cachedSchemaRegistryClient); + + $cachedSchemaRegistryClient->expects() + ->getBySubjectAndVersion('my-topic-key', 5) + ->andReturn($decoder); + + $decoder->expects() + ->getAvroSchema() + ->andReturn($avroSchema); + + $avroSchemaDecoder = new AvroSchemaDecoder($clientFactory, $consumerConfigOptions); + + $result = $avroSchemaDecoder->process($consumerRecord, $closure); + + $this->assertSame($expected, $result->getPayload()); + } +} diff --git a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php index 9412bd4b..fcff2d6c 100644 --- a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php +++ b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php @@ -1,6 +1,8 @@ getSchemaFixture(); + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + 'kafka-test', + $broker, + null, + new AvroSchemaConfigOptions('subjects/kafka-test-value/versions/latest', []) + ); + $avroSchemaConfigOptions = $producerConfigOptions->getAvroSchema(); - $schemaTest = $this->getSchemaTest(); - $configOptionsProducer = $this->createProducer(); + $clientFactory = m::mock(ClientFactory::class); - $record = new ProducerRecord($schemaTest, 'kafka-test'); $cachedSchemaRegistryClient = m::mock(CachedSchemaRegistryClient::class); + $schemaIdEncoder = m::mock(SchemaId::class, [$cachedSchemaRegistryClient]); + + $schema = new Schema(); + $parsedSchema = $schema->parse($avroSchema, '123', 'kafka-test-value', 'latest'); + $record = $this->getRecord($parsedSchema->getAvroSchema()); + $producerRecord = new ProducerRecord($record, 'kafka-test'); + + $closure = Closure::fromCallable(function ($producerRecord) { + return $producerRecord; + }); + + $payload = json_decode($producerRecord->getPayload(), true); + $encodedMessage = 'binary_message'; - $schema = (new Schema())->parse($schemaTest, '123'); - $clientFactory = new ClientFactory(); - $schemaIdEncoder = m::mock(SchemaId::class, [$cachedSchemaRegistryClient])->makePartial(); + // Expectations + $clientFactory->expects() + ->make($avroSchemaConfigOptions) + ->andReturn($cachedSchemaRegistryClient); - $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder($schemaIdEncoder, $clientFactory, $configOptionsProducer); + $cachedSchemaRegistryClient->expects() + ->getBySubjectAndVersion('kafka-test-value', 'latest') + ->andReturn($schema); - //expect - $cachedSchemaRegistryClient->shouldReceive('getBySubjectAndVersion')->andReturn($schema); - $schemaIdEncoder->shouldReceive('encode')->andReturn('string'); + $schemaIdEncoder->expects() + ->encode($schema, $payload) + ->andReturn($encodedMessage); - //act - $avroSchemaMixedEncoder->process($record, $closure); + // Actions + $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder($schemaIdEncoder, $clientFactory, $producerConfigOptions); + $result = $avroSchemaMixedEncoder->process($producerRecord, $closure); - //assert - $this->assertInstanceOf(AvroSchemaMixedEncoder::class, $avroSchemaMixedEncoder); + // Assertions + $this->assertSame($record, $result->getOriginal()); + $this->assertSame($encodedMessage, $result->getPayload()); } - private function createProducer() + private function getRecord(AvroSchema $avroSchema): string { - $brokerOptions = new Broker('kafka:9092', new None()); - return new ProducerConfigOptions( - 'kafka-test', - $brokerOptions, - null, - new AvroSchema('http://url.teste', []), - [], - 20000, - false, - true, - 10, - 500 - ); + $defaultValues = [ + 'null' => null, + 'boolean' => true, + 'string' => 'abc', + 'int' => 1, + 'long' => 1.0, + 'float' => 1.0, + 'double' => 1.0, + 'array' => [], + ]; + + $result = []; + foreach ($avroSchema->fields() as $field) { + $result[$field->name()] = $defaultValues[$field->type->type]; + } + + return json_encode($result); } - private function getSchemaTest(): string + private function getSchemaFixture(): string { return file_get_contents(__DIR__.'/../fixtures/schemas/sales_price.avsc'); } diff --git a/tests/Unit/Producer/PollTest.php b/tests/Unit/Producer/PollTest.php index 8bda848d..3ef21c96 100644 --- a/tests/Unit/Producer/PollTest.php +++ b/tests/Unit/Producer/PollTest.php @@ -4,6 +4,10 @@ use Metamorphosis\Producer\Poll; use Metamorphosis\ProducerConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Mockery as m; use RdKafka\Producer as KafkaProducer; use RuntimeException; @@ -14,17 +18,21 @@ class PollTest extends LaravelTestCase public function testItShouldHandleMessageWithoutAcknowledgment(): void { // Set - $configManager = new ProducerConfigManager(); - $configManager->set([ - 'topic_id' => 'topic_name', - 'timeout' => 4000, - 'is_async' => true, - 'max_poll_records' => 500, - 'flush_attempts' => 10, - 'required_acknowledgment' => false, - ]); + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + 'topic_name', + $broker, + null, + new AvroSchemaConfigOptions('string', []), + [], + 4000, + true, + false, + 500, + 10 + ); $kafkaProducer = m::mock(KafkaProducer::class); - $poll = new Poll($kafkaProducer, $configManager); + $poll = new Poll($kafkaProducer, $producerConfigOptions); // Expectations $kafkaProducer->expects() @@ -51,8 +59,21 @@ public function testShouldThrowExceptionWhenFlushFailed(): void 'partition' => 0, ]); + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + 'topic_name', + $broker, + null, + new AvroSchemaConfigOptions('string', []), + [], + 100, + false, + true, + 500, + 10 + ); $kafkaProducer = m::mock(KafkaProducer::class); - $poll = new Poll($kafkaProducer, $configManager); + $poll = new Poll($kafkaProducer, $producerConfigOptions); // Expectations $kafkaProducer->expects() @@ -72,18 +93,22 @@ public function testShouldThrowExceptionWhenFlushFailed(): void public function testItShouldHandleResponseEveryTimeWhenAsyncModeIsTrue(): void { // Set - $configManager = new ProducerConfigManager(); - $configManager->set([ - 'topic_id' => 'topic_name', - 'timeout' => 4000, - 'is_async' => false, - 'max_poll_records' => 500, - 'flush_attempts' => 10, - 'required_acknowledgment' => true, - 'partition' => 0, - ]); + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + 'topic_name', + $broker, + null, + new AvroSchemaConfigOptions('string', []), + [], + 4000, + false, + true, + 10, + 500 + ); + $kafkaProducer = m::mock(KafkaProducer::class); - $poll = new Poll($kafkaProducer, $configManager); + $poll = new Poll($kafkaProducer, $producerConfigOptions); // Expectations $kafkaProducer->expects() diff --git a/tests/Unit/ProducerTest.php b/tests/Unit/ProducerTest.php index 31650aef..02808390 100644 --- a/tests/Unit/ProducerTest.php +++ b/tests/Unit/ProducerTest.php @@ -8,8 +8,8 @@ use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\Middlewares\Handler\Producer as ProducerMiddleware; use Metamorphosis\Producer; -use Metamorphosis\ProducerConfigManager; use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; use Metamorphosis\TopicHandler\ConfigOptions\Broker; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Metamorphosis\TopicHandler\Producer\AbstractProducer; @@ -31,38 +31,22 @@ public function testItShouldProduceRecordAsArrayThroughMiddlewareQueue(): void ); $config = m::mock(Config::class); $connector = m::mock(Connector::class); - $configManager = m::mock(ProducerConfigManager::class)->makePartial(); $producer = new Producer($config, $connector); $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class($record, $topic) extends AbstractProducer { + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker + ); + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $config->expects() - ->makeByTopic($topic) - ->andReturn($configManager); - - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topic); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - - $configManager->expects() - ->get('timeout') - ->andReturn(1000); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() @@ -86,36 +70,27 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void ProducerMiddleware::class, m::mock(ProducerMiddleware::class) ); + $config = m::mock(Config::class); $connector = m::mock(Connector::class); - $configManager = m::mock(ProducerConfigManager::class)->makePartial(); + + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker + ); + $producer = new Producer($config, $connector); $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class($record, $topic) extends AbstractProducer { + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $config->expects() - ->makeByTopic($topic) - ->andReturn($configManager); - - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topic); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() @@ -127,10 +102,7 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void ->withAnyArgs(); // Actions - $result = $producer->produce($producerHandler); - - // Assertions - $this->assertNull($result); + $producer->produce($producerHandler); } public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): void @@ -145,52 +117,29 @@ public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): vo $config = m::mock(Config::class); $connector = m::mock(Connector::class); $producer = new Producer($config, $connector); - $configManager = m::mock(ProducerConfigManager::class); + $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class($record, $topic) extends AbstractProducer { + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker, + 0, + new AvroSchemaConfigOptions('string'), + [], + 1000, + false, + true, + 500, + 1 + ); + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topic); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - - $configManager->expects() - ->get('max_poll_records') - ->andReturn(500); - - $configManager->expects() - ->get('required_acknowledgment') - ->andReturn(true); - - $configManager->expects() - ->get('flush_attempts') - ->andReturn(1); - - $configManager->expects() - ->get('timeout') - ->andReturn(1000); - - $configManager->expects() - ->get('is_async') - ->andReturn(false); - - $config->expects() - ->makeByTopic($topic) - ->andReturn($configManager); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() @@ -223,50 +172,27 @@ public function testShouldBuildDispatcher(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $configManager = m::mock(ProducerConfigManager::class); - $producerHandler = new class($record, $topic) extends AbstractProducer { + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker, + null, + new AvroSchemaConfigOptions('string'), + [], + 1000, + true, + true, + 500, + 1 + ); + + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $config->expects() - ->makeByTopic($topic) - ->andReturn($configManager); - - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topic); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - - $configManager->expects() - ->get('max_poll_records') - ->andReturn(500); - - $configManager->expects() - ->get('is_async') - ->andReturn(true); - - $configManager->expects() - ->get('required_acknowledgment') - ->andReturn(true); - - $configManager->expects() - ->get('flush_attempts') - ->andReturn(1); - - $configManager->expects() - ->get('timeout') - ->andReturn(1000); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() @@ -288,7 +214,7 @@ public function testShouldBuildDispatcherWithConfigOptions(): void { // Set $record = json_encode(['message' => 'some message']); - $topicId = 'TOPIC-ID'; + $topic = 'TOPIC-ID'; $config = m::mock(Config::class); $connector = m::mock(Connector::class); @@ -296,56 +222,30 @@ public function testShouldBuildDispatcherWithConfigOptions(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $configManager = m::mock(ProducerConfigManager::class); - $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); - $broker = new Broker($connections, new None()); - $configOptions = new ProducerConfigOptions($topicId, $broker); - $producerHandler = new class($record, $configOptions) extends AbstractProducer { + + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker, + null, + new AvroSchemaConfigOptions('string'), + [], + 1000, + true, + true, + 500, + 1 + ); + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $config->expects() - ->make($configOptions) - ->andReturn($configManager); - - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topicId); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - - $configManager->expects() - ->get('max_poll_records') - ->andReturn(500); - - $configManager->expects() - ->get('is_async') - ->andReturn(true); - - $configManager->expects() - ->get('required_acknowledgment') - ->andReturn(true); - - $configManager->expects() - ->get('flush_attempts') - ->andReturn(1); - - $configManager->expects() - ->get('timeout') - ->andReturn(1000); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() - ->newTopic($topicId) + ->newTopic($topic) ->andReturn($producerTopic); $kafkaProducer->expects() diff --git a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php index cb78b0f7..f5c07f80 100644 --- a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php +++ b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php @@ -33,6 +33,7 @@ public function testShouldMakeConfigOptionWithAvroSchema(): void ]; $topicData = [ 'topic_id' => 'kafka-test', + 'consumer_group' => 'test-consumer-group', 'consumer' => [ 'consumer_groups' => [ 'test-consumer-group' => [ From d33816e2ebb00fa1c7be5c0eea48356dc669c785 Mon Sep 17 00:00:00 2001 From: David Franca Date: Mon, 21 Mar 2022 16:47:14 -0300 Subject: [PATCH 020/152] fix: update consumer middlewares --- src/Connectors/Consumer/Factory.php | 11 ++++++++++- src/Console/ConsumerCommand.php | 5 +++++ src/Facades/Metamorphosis.php | 1 + src/Producer.php | 1 - tests/Integration/ProducerTest.php | 27 ++++++++++++++++++++------- 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index 6a7c61aa..0da13cac 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -3,6 +3,7 @@ namespace Metamorphosis\Connectors\Consumer; use Metamorphosis\Consumers\ConsumerInterface; +use Metamorphosis\Middlewares\Handler\Consumer as ConsumerMiddleware; use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; @@ -23,9 +24,17 @@ public static function make(ConsumerConfigOptions $configOptions): Manager $commitAsync = $configOptions->isCommitASync(); $consumer = self::getConsumer($autoCommit, $configOptions); + $handler = app($configOptions->getHandler()); - $dispatcher = self::getMiddlewareDispatcher($configOptions->getMiddlewares()); + $middlewares = $configOptions->getMiddlewares(); + foreach ($middlewares as &$middleware) { + $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $configOptions]) : $middleware; + } + + $middlewares[] = app(ConsumerMiddleware::class, ['consumerTopicHandler' => $handler]); + + $dispatcher = self::getMiddlewareDispatcher($middlewares); return new Manager($consumer, $handler, $dispatcher, $autoCommit, $commitAsync); } diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index 5c233db4..bd4272e1 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -40,6 +40,11 @@ public function handle(Config $config) { $consumer = $config->make($this->option(), $this->argument()); + $middlewares = $consumer->getMiddlewares(); + foreach ($middlewares as &$middleware) { + $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $consumer]) : $middleware; + } + $this->writeStartingConsumer($consumer); $manager = Factory::make($consumer); diff --git a/src/Facades/Metamorphosis.php b/src/Facades/Metamorphosis.php index 70f903c8..c009a5f5 100644 --- a/src/Facades/Metamorphosis.php +++ b/src/Facades/Metamorphosis.php @@ -3,6 +3,7 @@ namespace Metamorphosis\Facades; use Illuminate\Support\Facades\Facade; +use Metamorphosis\TopicHandler\Producer\HandlerInterface; /** * @method static void produce(HandlerInterface $producerHandler) diff --git a/src/Producer.php b/src/Producer.php index ee277fdf..4fa9ee5b 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -35,7 +35,6 @@ public function build(HandlerInterface $producerHandler): Dispatcher $producerConfigOptions = $producerHandler->getConfigOptions(); $middlewares = $producerConfigOptions->getMiddlewares(); - foreach ($middlewares as &$middleware) { $middleware = is_string($middleware) ? app($middleware, ['producerConfigOptions' => $producerConfigOptions]) : $middleware; } diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 9780d74c..ec43989b 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -77,6 +77,8 @@ protected function haveAConsumerPartitionConfigured(): void protected function runTheConsumer(): void { + config(['kafka.topics.default.topic_id' => $this->topicId]); + $this->artisan( 'kafka:consume', [ @@ -139,9 +141,13 @@ protected function runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingT private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $producerConfigOptions = $this->createProducerConfigOptions('kafka-test'); -// $producer = app(MessageProducer::class, ['record' => $this->highLevelMessage, 'producer'=> $producerConfigOptions]); - $producer = new MessageProducer($this->highLevelMessage, $producerConfigOptions, 'recordId123'); + $this->topicId = 'kafka-test-'.Str::random(5); + $producerConfigOptions = $this->createProducerConfigOptions(); + $producer = app(MessageProducer::class, [ + 'record' => $this->highLevelMessage, + 'producer' => $producerConfigOptions, + 'key' => 'recordId123', + ]); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -150,7 +156,11 @@ private function haveSomeRandomMessagesProduced(): void private function produceRecordMessage(string $record): string { $producerConfigOptions = $this->createProducerConfigOptions('low_level'); - $producer = new MessageProducer($record, $producerConfigOptions, 'recordId123'); + $producer = app(MessageProducer::class, [ + 'record' => $record, + 'producer' => $producerConfigOptions, + 'key' => 'recordId123' + ]); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -190,15 +200,18 @@ private function haveFourProducedMessages(): void $this->produceRecordMessage($this->secondLowLevelMessage); } - private function createProducerConfigOptions(string $topicId = 'kafka-test'): ProducerConfigOptions + private function createProducerConfigOptions(): ProducerConfigOptions { $broker = new Broker('kafka:9092', new None()); return new ProducerConfigOptions( - $topicId, + $this->topicId, $broker, null, null, - [] + [], + 2000, + false, + true ); } } From 7b7407baa56cd9d698e2ac83db521572cf73f71a Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:21:58 -0300 Subject: [PATCH 021/152] fix: get and set options from consumer command --- src/Connectors/Consumer/Config.php | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 4db8f76a..1b0b5c6e 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -56,7 +56,7 @@ public function make(array $options, array $arguments): Consumer $configName = $options['config_name'] ?? 'kafka'; $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); $consumerGroupId = $this->getConsumerGroup($topicConfig, $arguments['consumer_group']); - $consumerConfig = $this->getConsumerConfig($topicConfig, $arguments, $consumerGroupId); + $consumerConfig = $this->getConsumerConfig($topicConfig, $options, $consumerGroupId); $brokerConfig = $this->getBrokerConfig($configName, $topicConfig['broker']); $schemaConfig = $this->getSchemaConfig($configName, $arguments['topic']); $override = array_merge($this->filterValues($options), $this->filterValues($arguments)); @@ -68,8 +68,19 @@ public function make(array $options, array $arguments): Consumer ); $this->validate(array_merge($config, $override)); - if (isset($topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'])) { - $topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'] = $consumerConfig['partition']; + + if (isset($topicConfig['consumer']['consumer_groups'][$consumerGroupId])) { + if (isset($options['partition'])) { + $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['partition'] = $options['partition']; + } + + if (isset($options['offset'])) { + $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['offset'] = $options['offset']; + } + + if (isset($options['timeout'])) { + $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['timeout'] = $options['timeout']; + } } $topicConfig['consumer_group'] = $consumerGroupId; @@ -95,7 +106,7 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } - private function getConsumerConfig(array $topicConfig, array $arguments, string $consumerGroupId): array + private function getConsumerConfig(array $topicConfig, array $options, string $consumerGroupId): array { $consumerConfig = $topicConfig['consumer']['consumer_groups'][$consumerGroupId] ?? null; if (!$consumerConfig) { @@ -104,10 +115,6 @@ private function getConsumerConfig(array $topicConfig, array $arguments, string $consumerConfig['consumer_group'] = $consumerGroupId; - if (isset($arguments['partition'])) { - $consumerConfig['partition'] = $arguments['partition']; - } - return $consumerConfig; } From b8059b38d5f004627e53754a8933a8e2ea4a6946 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:23:16 -0300 Subject: [PATCH 022/152] chore: remove inutilized code --- src/Console/ConsumerCommand.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index bd4272e1..70f151e0 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -39,12 +39,6 @@ class ConsumerCommand extends BaseCommand public function handle(Config $config) { $consumer = $config->make($this->option(), $this->argument()); - - $middlewares = $consumer->getMiddlewares(); - foreach ($middlewares as &$middleware) { - $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $consumer]) : $middleware; - } - $this->writeStartingConsumer($consumer); $manager = Factory::make($consumer); From a8291f4332b6061788968bdec13fd0db40117738 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:26:34 -0300 Subject: [PATCH 023/152] chore: create public variable for message --- tests/Integration/ProducerTest.php | 34 ++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index ec43989b..5a456a71 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -18,7 +18,22 @@ class ProducerTest extends LaravelTestCase protected string $firstLowLevelMessage; - protected string $secondLowLevelMessage; + /** + * @var string + */ + protected $secondLowLevelMessage; + + /** + * @var string + */ + protected $topicId; + + protected function setUp(): void + { + parent::setUp(); + $this->withoutAuthentication(); + $this->topicId = 'kafka-test-'.Str::random(5); + } /** * @group runProducer @@ -51,13 +66,6 @@ public function testShouldRunAProducerAndReceiveMessagesWithALowLevelConsumer(): $this->runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingToTwoMessagesConsumed(); } - protected function setUp(): void - { - parent::setUp(); - - $this->withoutAuthentication(); - } - protected function withoutAuthentication(): void { config(['kafka.brokers.default.auth' => []]); @@ -141,8 +149,8 @@ protected function runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingT private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $this->topicId = 'kafka-test-'.Str::random(5); - $producerConfigOptions = $this->createProducerConfigOptions(); + + $producerConfigOptions = $this->createProducerConfigOptions($this->topicId); $producer = app(MessageProducer::class, [ 'record' => $this->highLevelMessage, 'producer' => $producerConfigOptions, @@ -159,7 +167,7 @@ private function produceRecordMessage(string $record): string $producer = app(MessageProducer::class, [ 'record' => $record, 'producer' => $producerConfigOptions, - 'key' => 'recordId123' + 'key' => 'recordId123', ]); Metamorphosis::produce($producer); @@ -200,11 +208,11 @@ private function haveFourProducedMessages(): void $this->produceRecordMessage($this->secondLowLevelMessage); } - private function createProducerConfigOptions(): ProducerConfigOptions + private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { $broker = new Broker('kafka:9092', new None()); return new ProducerConfigOptions( - $this->topicId, + $topicId, $broker, null, null, From 1d677b091eaa4f5ccd9adb79758ec9383c4033f2 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:34:21 -0300 Subject: [PATCH 024/152] chore: create public variable for message --- tests/Integration/ProducerWithAvroTest.php | 65 ++++++++++++++++++---- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index 449370da..014100ef 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -6,15 +6,30 @@ use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\HandlerStack; use GuzzleHttp\Psr7\Response; +use Illuminate\Support\Facades\Log; use Metamorphosis\Facades\Metamorphosis; use Metamorphosis\Middlewares\AvroSchemaDecoder; use Metamorphosis\Middlewares\AvroSchemaMixedEncoder; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Tests\Integration\Dummies\MessageConsumer; use Tests\Integration\Dummies\MessageProducer; use Tests\LaravelTestCase; class ProducerWithAvroTest extends LaravelTestCase { + /** + * @var string[] + */ + protected $records; + + public function setUp(): void + { + parent::setUp(); + $this->records = ['saleOrderId' => 'SALE_ORDER_ID', 'productId' => 'PRODUCT_ID']; + } + public function testShouldRunAProducerMessagesWithAAvroSchema(): void { // Given That I @@ -22,7 +37,9 @@ public function testShouldRunAProducerMessagesWithAAvroSchema(): void // When I $this->haveSomeRandomMessagesProduced(); - $this->expectNotToPerformAssertions(); + + // I expect that + $this->myMessagesHaveBeenLogged(); } protected function haveAHandlerConfigured(): void @@ -93,14 +110,17 @@ protected function haveAHandlerConfigured(): void private function haveSomeRandomMessagesProduced(): void { - $saleOrderProducer = app( - MessageProducer::class, - ['record' => ['saleOrderId' => 'SALE_ORDER_ID'], 'topic' => 'sale_order'] - ); - $productProducer = app( - MessageProducer::class, - ['record' => ['productId' => 'PRODUCT_ID'], 'topic' => 'product'] - ); + $producerConfigOptionsSale = $this->createProducerConfigOptions('sale_order'); + $producerConfigOptionsProduct = $this->createProducerConfigOptions('product'); + + $saleOrderProducer = app(MessageProducer::class, [ + 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], + 'producer' => $producerConfigOptionsSale, + ]); + $productProducer = app(MessageProducer::class, [ + 'record' => ['productId' => 'PRODUCT_ID'], + 'producer' => $producerConfigOptionsProduct, + ]); $saleOrderSchemaResponse = '{ "subject":"sale_order-value", @@ -129,7 +149,32 @@ private function haveSomeRandomMessagesProduced(): void $productDispatcher = Metamorphosis::build($productProducer); $productDispatcher->handle($productProducer->createRecord()); + } - $saleOrderDispatcher->handle($saleOrderProducer->createRecord()); + private function myMessagesHaveBeenLogged(): void + { + $this->setLogExpectationsFor($this->records['saleOrderId']); + $this->setLogExpectationsFor($this->records['productId']); + } + + private function setLogExpectationsFor(string $message): void + { + Log::shouldReceive('info') + ->with($message); + } + + private function createProducerConfigOptions(string $topicId): ProducerConfigOptions + { + $broker = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( + $topicId, + $broker, + null, + null, + [], + 2000, + false, + true + ); } } From 881960655c4b7374cbd16b990c24007d7a05c496 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:37:24 -0300 Subject: [PATCH 025/152] chore: create public variable for message --- tests/Integration/ProducerWithConfigOptionsTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 4e256a42..f4f58b94 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -88,7 +88,7 @@ private function haveSomeRandomMessageProduced(): void MessageProducerWithConfigOptions::class, [ 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], - 'configOptions' => $this->producerConfigOptions, + 'producer' => $this->producerConfigOptions, 'key' => 1, ] ); From 5b718ec10f3170dc888ac486770e0fb54df639ba Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:38:44 -0300 Subject: [PATCH 026/152] fix: change parameter order --- tests/Unit/Connectors/Consumer/FactoryTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Unit/Connectors/Consumer/FactoryTest.php b/tests/Unit/Connectors/Consumer/FactoryTest.php index 50a158f2..ae3dedf6 100644 --- a/tests/Unit/Connectors/Consumer/FactoryTest.php +++ b/tests/Unit/Connectors/Consumer/FactoryTest.php @@ -58,11 +58,11 @@ public function testItMakesManagerWithLowLevelConsumer(): void $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); } - public function testItMakesManagerWithLowLevelConsumerWhenPartitionIsNotValid(): void + public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid(): void { // Set $config = new Config(); - $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition', 'partition' => -1]); + $configConsumer = $config->make(['timeout' => 61, 'partition' => -1], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); $manager = Factory::make($configConsumer); // Assertions From 62a193847abb8c193e8ac9ce6b8cdfe94396b546 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 16:44:23 -0300 Subject: [PATCH 027/152] docs: add CHANGELOG file --- CHANGELOG.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..0ecf5eae --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,40 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- + +### Fixed + +- + +### Removed +- Remove deprecated class AbstractHandler + +## [4.1.0] - 2022-03-23 + +### Added +- Added AvroSchemaMixedEncoderTest +- Added AvroSchemaDecoderTest +- Added ProducerWithConfigOptionsTest +- Added ConfigOptionsCommand to run commands with ConfigOptions class +- Added pt_BR contributing section +- Added setup-dev script on composer +- Added grumphp commit validation + +### Fixed +- Fixed parameters and options override on Consumer\Config class +- Update instructions on contribute section +- Update project install section + +### Changed +- Updated class from ConfigManager to ConfigOptions on unit tests +- Updated class from ConfigManager to ConfigOptions where any config request was made +- Consumer and Producer middlewares resolution From d8bf597e650b2d6689a64809852edf707562857c Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 17:12:50 -0300 Subject: [PATCH 028/152] docs: add CHANGELOG file --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ecf5eae..32abee8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - +### Changed + +- + ### Removed - Remove deprecated class AbstractHandler From 8ddde21470997c2133e3f45990e18995c889b8f4 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 11:50:50 -0300 Subject: [PATCH 029/152] chore: rename parameter from 'producer' to 'configOptions' This would be a breaking change, so it's better keep as 'configOptions' by now. --- src/TopicHandler/Producer/AbstractProducer.php | 4 ++-- tests/Integration/ConsumerTest.php | 2 +- tests/Integration/ProducerTest.php | 4 ++-- tests/Integration/ProducerWithAvroTest.php | 4 ++-- tests/Integration/ProducerWithConfigOptionsTest.php | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/TopicHandler/Producer/AbstractProducer.php b/src/TopicHandler/Producer/AbstractProducer.php index 946db3ce..694aedd0 100644 --- a/src/TopicHandler/Producer/AbstractProducer.php +++ b/src/TopicHandler/Producer/AbstractProducer.php @@ -20,11 +20,11 @@ class AbstractProducer implements HandlerInterface */ protected $producer; - public function __construct($record, Producer $producer, string $key = null) + public function __construct($record, Producer $configOptions, string $key = null) { $this->record = $record; $this->key = $key; - $this->producer = $producer; + $this->producer = $configOptions; } public function getConfigOptions(): Producer diff --git a/tests/Integration/ConsumerTest.php b/tests/Integration/ConsumerTest.php index 7088e46b..1d4397e8 100644 --- a/tests/Integration/ConsumerTest.php +++ b/tests/Integration/ConsumerTest.php @@ -51,7 +51,7 @@ public function testItShouldSetup(): void MessageProducerWithConfigOptions::class, [ 'record' => ['id' => 'MESSAGE_ID'], - 'producer' => $producerConfigOptions, + 'configOptions' => $producerConfigOptions, 'key' => 1, ] ); diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 5a456a71..73852c4c 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -153,7 +153,7 @@ private function haveSomeRandomMessagesProduced(): void $producerConfigOptions = $this->createProducerConfigOptions($this->topicId); $producer = app(MessageProducer::class, [ 'record' => $this->highLevelMessage, - 'producer' => $producerConfigOptions, + 'configOptions' => $producerConfigOptions, 'key' => 'recordId123', ]); @@ -166,7 +166,7 @@ private function produceRecordMessage(string $record): string $producerConfigOptions = $this->createProducerConfigOptions('low_level'); $producer = app(MessageProducer::class, [ 'record' => $record, - 'producer' => $producerConfigOptions, + 'configOptions' => $producerConfigOptions, 'key' => 'recordId123', ]); diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index 014100ef..b0c801ad 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -115,11 +115,11 @@ private function haveSomeRandomMessagesProduced(): void $saleOrderProducer = app(MessageProducer::class, [ 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], - 'producer' => $producerConfigOptionsSale, + 'configOptions' => $producerConfigOptionsSale, ]); $productProducer = app(MessageProducer::class, [ 'record' => ['productId' => 'PRODUCT_ID'], - 'producer' => $producerConfigOptionsProduct, + 'configOptions' => $producerConfigOptionsProduct, ]); $saleOrderSchemaResponse = '{ diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index f4f58b94..4e256a42 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -88,7 +88,7 @@ private function haveSomeRandomMessageProduced(): void MessageProducerWithConfigOptions::class, [ 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], - 'producer' => $this->producerConfigOptions, + 'configOptions' => $this->producerConfigOptions, 'key' => 1, ] ); From f61dd2dacc5275888d233a6e537423e7e6e422a8 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 14:44:51 -0300 Subject: [PATCH 030/152] chore: remove annotation --- src/Facades/Metamorphosis.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Facades/Metamorphosis.php b/src/Facades/Metamorphosis.php index c009a5f5..b3c934ca 100644 --- a/src/Facades/Metamorphosis.php +++ b/src/Facades/Metamorphosis.php @@ -3,11 +3,7 @@ namespace Metamorphosis\Facades; use Illuminate\Support\Facades\Facade; -use Metamorphosis\TopicHandler\Producer\HandlerInterface; -/** - * @method static void produce(HandlerInterface $producerHandler) - */ class Metamorphosis extends Facade { protected static function getFacadeAccessor() From 4b83d91a7cfc091b6cda980ab1a1e481753cd55c Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 20:42:10 -0300 Subject: [PATCH 031/152] chore: update middleware resolution on consumerCommand --- src/Console/ConsumerCommand.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index 70f151e0..bd4272e1 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -39,6 +39,12 @@ class ConsumerCommand extends BaseCommand public function handle(Config $config) { $consumer = $config->make($this->option(), $this->argument()); + + $middlewares = $consumer->getMiddlewares(); + foreach ($middlewares as &$middleware) { + $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $consumer]) : $middleware; + } + $this->writeStartingConsumer($consumer); $manager = Factory::make($consumer); From 2a8149c508eda5b6bb5f45ccc948ddc0059e86e1 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 21:43:32 -0300 Subject: [PATCH 032/152] chore: remove ManageTest --- tests/Unit/ManagerTest.php | 66 -------------------------------------- 1 file changed, 66 deletions(-) diff --git a/tests/Unit/ManagerTest.php b/tests/Unit/ManagerTest.php index 846db3f5..e69de29b 100644 --- a/tests/Unit/ManagerTest.php +++ b/tests/Unit/ManagerTest.php @@ -1,66 +0,0 @@ -set([ - 'topic' => 'products', - 'middlewares' => [ - Log::class, - app(JsonDecode::class), - ], - 'other' => 'config', - ]); - - // Actions - $other = $manager->get('other'); - $middlewares = $manager->middlewares(); - - // Assertions - $this->assertSame('config', $other); - $this->assertInstanceOf(Log::class, $middlewares[0]); - $this->assertInstanceOf(JsonDecode::class, $middlewares[1]); - } - - public function testShouldRemoveOldMiddlewareBeforeAddOthers(): void - { - // Set - $manager = new ConsumerConfigManager(); - $firstConfig = [ - 'topic' => 'products', - 'middlewares' => [ - Log::class, - app(JsonDecode::class), - ], - 'other' => 'config', - ]; - $secondConfig = [ - 'topic' => 'products', - 'middlewares' => [ - JsonDecode::class, - ], - 'other' => 'config', - ]; - - // Actions - $manager->set($firstConfig); - $manager->set($secondConfig); - $other = $manager->get('other'); - $middlewares = $manager->middlewares(); - - // Assertions - $this->assertSame('config', $other); - $this->assertInstanceOf(JsonDecode::class, $middlewares[0]); - $this->assertCount(1, $middlewares); - } -} From 16118fd77c51f2b5f41b78312fcf39338f82c527 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 21:47:10 -0300 Subject: [PATCH 033/152] chore: remove occurrences of ConsumerConfigManager --- src/Consumer.php | 4 +--- tests/Unit/Connectors/Consumer/ManagerTest.php | 2 -- tests/Unit/Consumers/LowLevelTest.php | 3 --- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Consumer.php b/src/Consumer.php index 1d35a0c1..367556fa 100644 --- a/src/Consumer.php +++ b/src/Consumer.php @@ -15,10 +15,8 @@ class Consumer private Dispatcher $dispatcher; - public function __construct(ConsumerConfigManager $configManager, ConsumerConfigOptions $configOptions) + public function __construct(ConsumerConfigOptions $configOptions) { - $configManager->set($configOptions->toArray()); - $this->consumer = Factory::getConsumer(true, $configOptions); $this->dispatcher = new Dispatcher($configOptions->getMiddlewares()); } diff --git a/tests/Unit/Connectors/Consumer/ManagerTest.php b/tests/Unit/Connectors/Consumer/ManagerTest.php index d69d72eb..230513b2 100644 --- a/tests/Unit/Connectors/Consumer/ManagerTest.php +++ b/tests/Unit/Connectors/Consumer/ManagerTest.php @@ -4,7 +4,6 @@ use Exception; use Metamorphosis\Connectors\Consumer\Manager; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Exceptions\ResponseTimeoutException; use Metamorphosis\Exceptions\ResponseWarningException; @@ -14,7 +13,6 @@ use Mockery as m; use RdKafka\Message as KafkaMessage; use Tests\LaravelTestCase; -use Tests\Unit\Dummies\ConsumerHandlerDummy; class ManagerTest extends LaravelTestCase { diff --git a/tests/Unit/Consumers/LowLevelTest.php b/tests/Unit/Consumers/LowLevelTest.php index cc8bb988..b39ac2aa 100644 --- a/tests/Unit/Consumers/LowLevelTest.php +++ b/tests/Unit/Consumers/LowLevelTest.php @@ -2,7 +2,6 @@ namespace Tests\Unit\Consumers; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\LowLevel; use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; @@ -20,8 +19,6 @@ public function testItShouldConsume(): void // Set $timeout = 2; $partition = 3; - $configManager = new ConsumerConfigManager(); - $configManager->set(compact('timeout', 'partition')); $brokerOptions = new Broker('kafka:9092', new None()); $consumerConfigOptions = new ConsumerConfigOptions( From 3df64261c55e57aafc7f4904f936e4284f839097 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 21:47:56 -0300 Subject: [PATCH 034/152] chore: remove TODO comment --- src/Connectors/Consumer/Factory.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index 0da13cac..c82c4ad2 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -13,9 +13,6 @@ * means that we will use the low level approach. */ -/** - * TODO Rename this class to ConsumerFactory, it will improve semantics - */ class Factory { public static function make(ConsumerConfigOptions $configOptions): Manager From c0d23785958f153dd7fe90f606054d0327da736f Mon Sep 17 00:00:00 2001 From: hcdias Date: Fri, 25 Mar 2022 10:20:06 -0300 Subject: [PATCH 035/152] chore: remove ProducerConfigManager occurences --- src/ConsumerConfigManager.php | 54 ------------------- .../Unit/Middlewares/Handler/ProducerTest.php | 1 - tests/Unit/Producer/PollTest.php | 12 ----- 3 files changed, 67 deletions(-) delete mode 100644 src/ConsumerConfigManager.php diff --git a/src/ConsumerConfigManager.php b/src/ConsumerConfigManager.php deleted file mode 100644 index 88207c97..00000000 --- a/src/ConsumerConfigManager.php +++ /dev/null @@ -1,54 +0,0 @@ -setConfig($config, $consumerHandler); - $this->setCommandConfig($commandConfig); - - $middlewares = $this->get('middlewares', []); - $this->middlewares = []; - $this->remove('middlewares'); - - foreach ($middlewares as $middleware) { - $this->middlewares[] = is_string($middleware) ? app($middleware) : $middleware; - } - - if (!$consumerHandler) { - return; - } - - $this->middlewares[] = new ConsumerMiddleware($consumerHandler); - } - - private function setCommandConfig(?array $commandConfig): void - { - if (!$commandConfig) { - return; - } - - $this->setting = array_merge($this->setting, $commandConfig); - } - - private function setConfig(array $config, ?AbstractHandler $handler): void - { - if (!$handler || !$overrideConfig = $handler->getConfigOptions()) { - $this->setting = $config; - - return; - } - - $this->setting = $overrideConfig->toArray(); - } -} diff --git a/tests/Unit/Middlewares/Handler/ProducerTest.php b/tests/Unit/Middlewares/Handler/ProducerTest.php index 3a7bbc70..df2750e6 100644 --- a/tests/Unit/Middlewares/Handler/ProducerTest.php +++ b/tests/Unit/Middlewares/Handler/ProducerTest.php @@ -5,7 +5,6 @@ use Closure; use Metamorphosis\Middlewares\Handler\Producer; use Metamorphosis\Producer\Poll; -use Metamorphosis\ProducerConfigManager; use Metamorphosis\Record\ProducerRecord; use Mockery as m; use RdKafka\ProducerTopic as KafkaTopicProducer; diff --git a/tests/Unit/Producer/PollTest.php b/tests/Unit/Producer/PollTest.php index 3ef21c96..0576cd6c 100644 --- a/tests/Unit/Producer/PollTest.php +++ b/tests/Unit/Producer/PollTest.php @@ -3,7 +3,6 @@ namespace Tests\Unit\Producer; use Metamorphosis\Producer\Poll; -use Metamorphosis\ProducerConfigManager; use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; use Metamorphosis\TopicHandler\ConfigOptions\Broker; @@ -48,17 +47,6 @@ public function testItShouldHandleMessageWithoutAcknowledgment(): void public function testShouldThrowExceptionWhenFlushFailed(): void { // Set - $configManager = new ProducerConfigManager(); - $configManager->set([ - 'topic_id' => 'topic_name', - 'timeout' => 1000, - 'is_async' => false, - 'max_poll_records' => 500, - 'flush_attempts' => 3, - 'required_acknowledgment' => true, - 'partition' => 0, - ]); - $broker = new Broker('kafka:9092', new None()); $producerConfigOptions = new ProducerConfigOptions( 'topic_name', From 353a22852367c83267d8fe574c31e99ca33be26f Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 19 Apr 2022 16:10:10 -0300 Subject: [PATCH 036/152] chore: remove unecessary interface --- src/Avro/AvroClientInterface.php | 6 ------ src/Avro/CachedSchemaRegistryClientInterface.php | 7 ------- 2 files changed, 13 deletions(-) delete mode 100644 src/Avro/AvroClientInterface.php delete mode 100644 src/Avro/CachedSchemaRegistryClientInterface.php diff --git a/src/Avro/AvroClientInterface.php b/src/Avro/AvroClientInterface.php deleted file mode 100644 index d7617023..00000000 --- a/src/Avro/AvroClientInterface.php +++ /dev/null @@ -1,6 +0,0 @@ - Date: Tue, 19 Apr 2022 16:12:26 -0300 Subject: [PATCH 037/152] chore: remove unecessary code block --- src/Console/ConsumerCommand.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index bd4272e1..5c233db4 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -40,11 +40,6 @@ public function handle(Config $config) { $consumer = $config->make($this->option(), $this->argument()); - $middlewares = $consumer->getMiddlewares(); - foreach ($middlewares as &$middleware) { - $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $consumer]) : $middleware; - } - $this->writeStartingConsumer($consumer); $manager = Factory::make($consumer); From 9342da49c15b91e74560696be498be0873619228 Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 19 Apr 2022 16:13:45 -0300 Subject: [PATCH 038/152] chore: define constant visibility --- src/Avro/ClientFactory.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Avro/ClientFactory.php b/src/Avro/ClientFactory.php index b30d0c1b..388a8fed 100644 --- a/src/Avro/ClientFactory.php +++ b/src/Avro/ClientFactory.php @@ -7,7 +7,7 @@ class ClientFactory { - const REQUEST_TIMEOUT = 2000; + protected const REQUEST_TIMEOUT = 2000; public function make(AvroSchema $avroSchema): CachedSchemaRegistryClient { From 9fa3113faef7df4965c1a81edc47780f6c5ed038 Mon Sep 17 00:00:00 2001 From: hcdias Date: Fri, 1 Apr 2022 15:59:18 -0300 Subject: [PATCH 039/152] chore: change config file format --- config/kafka.php | 168 ++++-------------- config/service.php | 28 +++ src/Connectors/AbstractConfig.php | 24 +-- src/Connectors/Consumer/Config.php | 56 +----- src/Connectors/Producer/Config.php | 12 +- src/MetamorphosisServiceProvider.php | 6 +- .../Factories/ConsumerFactory.php | 6 +- 7 files changed, 86 insertions(+), 214 deletions(-) create mode 100644 config/service.php diff --git a/config/kafka.php b/config/kafka.php index 9f9ae370..9e74693b 100644 --- a/config/kafka.php +++ b/config/kafka.php @@ -1,124 +1,46 @@ [ - 'default' => [ - 'url' => '', - // Disable SSL verification on schema request. - 'ssl_verify' => true, - // This option will be put directly into a Guzzle http request - // Use this to do authorizations or send any headers you want. - // Here is a example of basic authentication on AVRO schema. - 'request_options' => [ - 'headers' => [ - 'Authorization' => [ - 'Basic ' . base64_encode( - env('AVRO_SCHEMA_USERNAME') - . ':' - . env('AVRO_SCHEMA_PASSWORD') - ), - ], - ], - ], - ], - ], - - /* - |-------------------------------------------------------------------------- - | Brokers - |-------------------------------------------------------------------------- - | - | Here you may specify the connections details for each broker configured - | on topic's broker key. - | - */ - - 'brokers' => [ - 'default' => [ - 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), - - // If your broker doest not have authentication, you can - // remove this configuration, or set as empty. - // The Authentication types may be "ssl" or "none" - 'auth' => [ - 'type' => 'ssl', // ssl and none - 'ca' => storage_path('ca.pem'), - 'certificate' => storage_path('kafka.cert'), - 'key' => storage_path('kafka.key'), - ], - ], - ], - 'topics' => [ - // This is your topic "keyword" where you will put all configurations needed - // on this specific topic. 'default' => [ - // The topic id is where you want to send or consume - // your messages from kafka. 'topic_id' => 'kafka-test', - - // Here you may point the key of the broker configured above. - 'broker' => 'default', - - // Configurations specific for consumer 'consumer' => [ - // You may define more than one consumer group per topic. - // If there is just one defined, it will be used by default, - // otherwise, you may pass which consumer group should be used - // when using the consumer command. - 'consumer_groups' => [ - 'test-consumer-group' => [ - - // Action to take when there is no initial - // offset in offset store or the desired offset is out of range. - // This config will be passed to 'auto.offset.reset'. - // The valid options are: smallest, earliest, beginning, largest, latest, end, error. - 'offset_reset' => 'earliest', - - // The offset at which to start consumption. This only applies if partition is set. - // You can use a positive integer or any of the constants: RD_KAFKA_OFFSET_BEGINNING, - // RD_KAFKA_OFFSET_END, RD_KAFKA_OFFSET_STORED. - 'offset' => 0, - - // The partition to consume. It can be null, - // if you don't wish do specify one. - 'partition' => 0, - - // A consumer class that implements ConsumerTopicHandler - 'handler' => '\App\Kafka\Consumers\ConsumerExample', - - // A Timeout to listen to a message. That means: how much - // time we need to wait until receiving a message? - 'timeout' => 20000, - - // Once you've enabled this, the Kafka consumer will commit the - // offset of the last message received in response to its poll() call - 'auto_commit' => true, - - // If commit_async is false process block until offsets are committed or the commit fails. - // Only works when auto_commit is false - 'commit_async' => false, - - // An array of middlewares applied only for this consumer_group - 'middlewares' => [], - ], - ], + 'consumer_group' => 'test-consumer-group', + // Action to take when there is no initial + // offset in offset store or the desired offset is out of range. + // This config will be passed to 'auto.offset.reset'. + // The valid options are: smallest, earliest, beginning, largest, latest, end, error. + 'offset_reset' => 'earliest', + + // The offset at which to start consumption. This only applies if partition is set. + // You can use a positive integer or any of the constants: RD_KAFKA_OFFSET_BEGINNING, + // RD_KAFKA_OFFSET_END, RD_KAFKA_OFFSET_STORED. + 'offset' => 0, + + // The partition to consume. It can be null, + // if you don't wish do specify one. + 'partition' => 0, + + // A consumer class that implements ConsumerTopicHandler + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + + // A Timeout to listen to a message. That means: how much + // time we need to wait until receiving a message? + 'timeout' => 20000, + + // Once you've enabled this, the Kafka consumer will commit the + // offset of the last message received in response to its poll() call + 'auto_commit' => true, + + // If commit_async is false process block until offsets are committed or the commit fails. + // Only works when auto_commit is false + 'commit_async' => false, + + // An array of middlewares applied only for this consumer_group + 'middlewares' => [], ], - // Configurations specific for producer 'producer' => [ - // Sets to true if you want to know if a message was successfully posted. 'required_acknowledgment' => true, @@ -149,28 +71,4 @@ ], ], ], - - /* - |-------------------------------------------------------------------------- - | Global Middlewares - |-------------------------------------------------------------------------- - | - | Here you may specify the global middlewares that will be applied for every - | consumed topic. Middlewares work between the received data from broker and - | before being passed into consumers. - | Available middlewares: log, avro-decode - | - */ - - 'middlewares' => [ - 'consumer' => [ - \Metamorphosis\Middlewares\Log::class, - ], - 'producer' => [ - \Metamorphosis\Middlewares\Log::class, - ], - 'global' => [ - \Metamorphosis\Middlewares\Log::class, - ], - ], ]; diff --git a/config/service.php b/config/service.php new file mode 100644 index 00000000..82cb1299 --- /dev/null +++ b/config/service.php @@ -0,0 +1,28 @@ + [ + 'url' => '', + 'request_options' => [ + 'headers' => [ + 'Authorization' => [ + 'Basic '.base64_encode( + env('AVRO_SCHEMA_USERNAME').':'.env('AVRO_SCHEMA_PASSWORD') + ), + ], + ], + ], + 'ssl_verify' => true, + 'username' => 'USERNAME', + 'password' => 'PASSWORD', + ], + 'broker' => [ + 'connections' => 'kafka:9092', + 'auth' => [ + 'type' => 'ssl', // ssl and none + 'ca' => storage_path('ca.pem'), + 'certificate' => storage_path('kafka.cert'), + 'key' => storage_path('kafka.key'), + ], + ], +]; diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index 9942dc2e..fa8b8ac1 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -7,20 +7,17 @@ abstract class AbstractConfig { - /** - * @var mixed[] - */ - protected array $rules = []; - - protected function getBrokerConfig(string $configName, string $brokerId): array + protected function getBrokerConfig(string $servicesFile): array { - if (!$brokerConfig = config("{$configName}.brokers.{$brokerId}")) { - throw new ConfigurationException( - "Broker '{$brokerId}' configuration not found" - ); + if (!$brokerConfig = config($servicesFile.'.broker')) { + throw new ConfigurationException("Broker configuration not found on '{$servicesFile}'"); } + return $brokerConfig; + } - return (array) $brokerConfig; + protected function getSchemaConfig(string $servicesFile): array + { + return config($servicesFile.'.avro_schema', []); } protected function validate(array $config): void @@ -31,9 +28,4 @@ protected function validate(array $config): void throw new ConfigurationException($validator->errors()->toJson()); } } - - protected function getSchemaConfig(string $configName, string $topicId): array - { - return config($configName . '.avro_schemas.' . $topicId, []); - } } diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 1b0b5c6e..aabcfd8b 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -54,37 +54,26 @@ public function makeWithConfigOptions(string $handlerClass, ?int $times = null): public function make(array $options, array $arguments): Consumer { $configName = $options['config_name'] ?? 'kafka'; - $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); - $consumerGroupId = $this->getConsumerGroup($topicConfig, $arguments['consumer_group']); - $consumerConfig = $this->getConsumerConfig($topicConfig, $options, $consumerGroupId); - $brokerConfig = $this->getBrokerConfig($configName, $topicConfig['broker']); - $schemaConfig = $this->getSchemaConfig($configName, $arguments['topic']); - $override = array_merge($this->filterValues($options), $this->filterValues($arguments)); - $config = array_merge( - $topicConfig, - $brokerConfig, - $consumerConfig, - $schemaConfig - ); + $service = $options['service'] ?? 'service'; - $this->validate(array_merge($config, $override)); + $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); + $brokerConfig = $this->getBrokerConfig($service); + $schemaConfig = $this->getSchemaConfig($service); - if (isset($topicConfig['consumer']['consumer_groups'][$consumerGroupId])) { + if (isset($topicConfig['consumer'])) { if (isset($options['partition'])) { - $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['partition'] = $options['partition']; + $topicConfig['consumer']['partition'] = $options['partition']; } if (isset($options['offset'])) { - $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['offset'] = $options['offset']; + $topicConfig['consumer']['offset'] = $options['offset']; } if (isset($options['timeout'])) { - $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['timeout'] = $options['timeout']; + $topicConfig['consumer']['timeout'] = $options['timeout']; } } - $topicConfig['consumer_group'] = $consumerGroupId; - return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); } @@ -106,35 +95,6 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } - private function getConsumerConfig(array $topicConfig, array $options, string $consumerGroupId): array - { - $consumerConfig = $topicConfig['consumer']['consumer_groups'][$consumerGroupId] ?? null; - if (!$consumerConfig) { - throw new ConfigurationException("Consumer group '{$consumerGroupId}' not found"); - } - - $consumerConfig['consumer_group'] = $consumerGroupId; - - return $consumerConfig; - } - - private function getConsumerGroup(array $topicConfig, ?string $consumerGroupId): string - { - if (!$consumerGroupId && 1 === count($topicConfig['consumer']['consumer_groups'])) { - $consumerGroupId = current(array_keys($topicConfig['consumer']['consumer_groups'])); - } - - return $consumerGroupId ?? 'default'; - } - - private function getMiddlewares(string $configName, array $topicConfig): array - { - return array_merge( - config($configName . '.middlewares.consumer', []), - $topicConfig['consumer']['middlewares'] ?? [] - ); - } - /** * Sometimes that user may pass `--partition=0` as argument. * So if we just use array_filter here, this option will diff --git a/src/Connectors/Producer/Config.php b/src/Connectors/Producer/Config.php index 90a9cd1d..e3cf8e80 100644 --- a/src/Connectors/Producer/Config.php +++ b/src/Connectors/Producer/Config.php @@ -50,15 +50,9 @@ public function make(ProducerConfigOptions $configOptions): AbstractConfigManage public function makeByTopic(string $topicId): AbstractConfigManager { $topicConfig = $this->getTopicConfig($topicId); - $topicConfig['middlewares'] = array_merge( - config('kafka.middlewares.producer', []), - $topicConfig['producer']['middlewares'] ?? [] - ); - $brokerConfig = $this->getBrokerConfig( - 'kafka', - $topicConfig['broker'] - ); - $schemaConfig = $this->getSchemaConfig('kafka', $topicId); + $topicConfig['middlewares'] = $topicConfig['producer']['middlewares'] ?? []; + $brokerConfig = $this->getBrokerConfig('service'); + $schemaConfig = $this->getSchemaConfig('service'); $config = array_merge($topicConfig, $brokerConfig, $schemaConfig); $this->validate($config); diff --git a/src/MetamorphosisServiceProvider.php b/src/MetamorphosisServiceProvider.php index 4938fbac..36dffdf5 100644 --- a/src/MetamorphosisServiceProvider.php +++ b/src/MetamorphosisServiceProvider.php @@ -14,10 +14,12 @@ class MetamorphosisServiceProvider extends ServiceProvider public function boot() { $this->publishes([ - __DIR__ . '/../config/kafka.php' => config_path('kafka.php'), + __DIR__.'/../config/kafka.php' => config_path('kafka.php'), + __DIR__.'/../config/service.php' => config_path('service.php'), ], 'config'); - $this->mergeConfigFrom(__DIR__ . '/../config/kafka.php', 'kafka'); + $this->mergeConfigFrom(__DIR__.'/../config/kafka.php', 'kafka'); + $this->mergeConfigFrom(__DIR__.'/../config/service.php', 'service'); } public function register() diff --git a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php index 7bd58c93..4dd62736 100644 --- a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php +++ b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php @@ -22,10 +22,8 @@ public static function make( private static function getConsumerGroupConfig(array $topicData): array { $topicData['topicId'] = $topicData['topic_id']; - $topicData['consumerGroup'] = $topicData['consumer_group']; - - $consumerGroup = $topicData['consumerGroup']; - $consumer = current($topicData['consumer'])[$consumerGroup]; + $consumer = $topicData['consumer']; + $topicData['consumerGroup'] = $consumer['consumer_group']; return array_merge_recursive($topicData, self::convertConfigAttributes($consumer)); } From ddc454c5391f7c2043610b4fd42f936f8bc3625c Mon Sep 17 00:00:00 2001 From: hcdias Date: Fri, 1 Apr 2022 16:09:05 -0300 Subject: [PATCH 040/152] chore: adjust tests to match new config file format --- tests/Integration/ProducerTest.php | 23 +++---- tests/Unit/Connectors/Consumer/ConfigTest.php | 7 ++- .../Unit/Connectors/Consumer/FactoryTest.php | 52 ++++++++++++++++ tests/Unit/Connectors/Producer/ConfigTest.php | 2 - tests/Unit/Console/ConsumerCommandTest.php | 61 ++++++++----------- .../Factories/ConsumerFactoryTest.php | 22 +++---- 6 files changed, 102 insertions(+), 65 deletions(-) diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 73852c4c..94ab5626 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -68,19 +68,17 @@ public function testShouldRunAProducerAndReceiveMessagesWithALowLevelConsumer(): protected function withoutAuthentication(): void { - config(['kafka.brokers.default.auth' => []]); + config(['service.broker.auth' => []]); } protected function haveAConsumerHandlerConfigured(): void { - config( - ['kafka.topics.default.consumer.consumer_groups.test-consumer-group.handler' => MessageConsumer::class] - ); + config(['kafka.topics.default.consumer.handler' => MessageConsumer::class]); } protected function haveAConsumerPartitionConfigured(): void { - config(['kafka.topics.default.consumer.consumer_groups.test-consumer-group.partition' => -1]); + config(['kafka.topics.default.consumer.partition' => -1]); } protected function runTheConsumer(): void @@ -107,15 +105,12 @@ protected function haveALowLevelConsumerConfigured(): void 'topic_id' => 'low_level', 'broker' => 'default', 'consumer' => [ - 'consumer_groups' => [ - 'test-consumer-group' => [ - 'offset_reset' => 'earliest', - 'offset' => 0, - 'handler' => MessageConsumer::class, - 'timeout' => 20000, - 'middlewares' => [], - ], - ], + 'consumer_group' => 'test-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'handler' => MessageConsumer::class, + 'timeout' => 20000, + 'middlewares' => [], ], 'producer' => [ 'required_acknowledgment' => true, diff --git a/tests/Unit/Connectors/Consumer/ConfigTest.php b/tests/Unit/Connectors/Consumer/ConfigTest.php index 539b6ea0..ffeaf5cd 100644 --- a/tests/Unit/Connectors/Consumer/ConfigTest.php +++ b/tests/Unit/Connectors/Consumer/ConfigTest.php @@ -8,6 +8,7 @@ use Mockery as m; use Tests\LaravelTestCase; use Tests\Unit\Dummies\ConsumerHandlerDummy; +use TypeError; class ConfigTest extends LaravelTestCase { @@ -104,8 +105,10 @@ public function testShouldNotSetRuntimeConfigWhenOptionsIsInvalid(): void 'consumer_group' => 'default', ]; + // Expectations + $this->expectException(TypeError::class); + // Actions - $this->expectException(ConfigurationException::class); $configManager = $config->make($options, $arguments); // Assertions @@ -115,7 +118,7 @@ public function testShouldNotSetRuntimeConfigWhenOptionsIsInvalid(): void public function testShouldNotSetRuntimeConfigWhenKafkaConfigIsInvalid(): void { // Set - config(['kafka.brokers.default.connections' => null]); + config(['service.broker' => null]); $config = new Config(); $options = [ 'partition' => 0, diff --git a/tests/Unit/Connectors/Consumer/FactoryTest.php b/tests/Unit/Connectors/Consumer/FactoryTest.php index ae3dedf6..2ccb9cd9 100644 --- a/tests/Unit/Connectors/Consumer/FactoryTest.php +++ b/tests/Unit/Connectors/Consumer/FactoryTest.php @@ -50,6 +50,8 @@ protected function setUp(): void public function testItMakesManagerWithLowLevelConsumer(): void { // Set + $this->haveAConsumerWithPartitionConfigured(); + $config = new Config(); $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); $manager = Factory::make($configConsumer); @@ -61,6 +63,7 @@ public function testItMakesManagerWithLowLevelConsumer(): void public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid(): void { // Set + $this->haveAConsumerWithoutPartitionConfigured(); $config = new Config(); $configConsumer = $config->make(['timeout' => 61, 'partition' => -1], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); $manager = Factory::make($configConsumer); @@ -72,6 +75,7 @@ public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid() public function testItMakesHighLevelClass(): void { // Set + $this->haveAConsumerWithoutPartitionConfigured(); $config = new Config(); $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'without-partition']); $manager = Factory::make($configConsumer); @@ -79,4 +83,52 @@ public function testItMakesHighLevelClass(): void // Assertions $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); } + + private function haveAConsumerWithPartitionConfigured() + { + config([ + 'kafka' => [ + 'topics' => [ + 'topic_key' => [ + 'topic_id' => 'topic_name', + 'consumer' => [ + 'consumer_group' => 'with-partition', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => ConsumerHandlerDummy::class, + ], + ], + ], + ], + 'service' => [ + 'broker' => [ + 'connections' => 'kafka:123', + ], + ], + ]); + } + + private function haveAConsumerWithoutPartitionConfigured() + { + config([ + 'kafka' => [ + 'topics' => [ + 'topic_key' => [ + 'topic_id' => 'topic_name', + 'consumer' => [ + 'consumer_group' => 'without-partition', + 'offset_reset' => 'earliest', + 'handler' => ConsumerHandlerDummy::class, + ], + ], + ], + ], + 'service' => [ + 'broker' => [ + 'connections' => 'kafka:123', + ], + ], + ]); + } } diff --git a/tests/Unit/Connectors/Producer/ConfigTest.php b/tests/Unit/Connectors/Producer/ConfigTest.php index 82adae08..d60567e1 100644 --- a/tests/Unit/Connectors/Producer/ConfigTest.php +++ b/tests/Unit/Connectors/Producer/ConfigTest.php @@ -24,7 +24,6 @@ public function testShouldValidateProducerConfig(): void 'max_poll_records' => 500, 'flush_attempts' => 10, 'partition' => -1, - 'broker' => 'default', 'topic' => 'default', 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), 'auth' => [ @@ -70,7 +69,6 @@ public function testShouldNotOverrideDefaultParametersWhenConfigIsSet(): void 'required_acknowledgment' => true, 'max_poll_records' => 3000, 'flush_attempts' => 10, - 'broker' => 'default', 'topic' => 'default', 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), 'auth' => [ diff --git a/tests/Unit/Console/ConsumerCommandTest.php b/tests/Unit/Console/ConsumerCommandTest.php index 26132745..5a5a8fcf 100644 --- a/tests/Unit/Console/ConsumerCommandTest.php +++ b/tests/Unit/Console/ConsumerCommandTest.php @@ -10,6 +10,33 @@ class ConsumerCommandTest extends LaravelTestCase { + protected function setUp(): void + { + parent::setUp(); + + config([ + 'kafka' => [ + 'topics' => [ + 'topic_key' => [ + 'topic_id' => 'topic_name', + 'consumer' => [ + 'consumer_group' => 'default', + 'offset_reset' => 'earliest', + 'handler' => ConsumerHandlerDummy::class, + 'timeout' => 123, + ], + ], + ], + ], + 'service' => [ + 'broker' => [ + 'connections' => 'test_kafka:6680', + 'auth' => [], + ], + ], + ]); + } + public function testItCallsCommandWithInvalidTopic(): void { // Set @@ -131,38 +158,4 @@ public function testItOverridesBrokerConnectionWhenCallingCommand(): void $this->artisan($command, $parameters); } - - protected function setUp(): void - { - parent::setUp(); - - config([ - 'kafka' => [ - 'brokers' => [ - 'default' => [ - 'connections' => env( - 'KAFKA_BROKER_CONNECTIONS', - 'kafka:9092' - ), - 'auth' => [], - ], - ], - 'topics' => [ - 'topic_key' => [ - 'topic_id' => 'topic_name', - 'broker' => 'default', - 'consumer' => [ - 'consumer_groups' => [ - 'default' => [ - 'offset_reset' => 'earliest', - 'handler' => ConsumerHandlerDummy::class, - 'timeout' => 123, - ], - ], - ], - ], - ], - ], - ]); - } } diff --git a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php index f5c07f80..02aa3368 100644 --- a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php +++ b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php @@ -33,20 +33,16 @@ public function testShouldMakeConfigOptionWithAvroSchema(): void ]; $topicData = [ 'topic_id' => 'kafka-test', - 'consumer_group' => 'test-consumer-group', 'consumer' => [ - 'consumer_groups' => [ - 'test-consumer-group' => [ - 'middlewares' => [], - 'auto_commit' => true, - 'commit_async' => true, - 'offset_reset' => 'earliest', - 'handler' => '\App\Kafka\Consumers\ConsumerExample', - 'partition' => 0, - 'offset' => 0, - 'timeout' => 20000, - ], - ], + 'consumer_group' => 'test-consumer-group', + 'middlewares' => [], + 'auto_commit' => true, + 'commit_async' => true, + 'offset_reset' => 'earliest', + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'partition' => 0, + 'offset' => 0, + 'timeout' => 20000, ], ]; $expected = [ From b060e90b555b5a2429baaa1109a21ed5fcdf3aa0 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 15:51:49 -0300 Subject: [PATCH 041/152] chore: update docs --- CHANGELOG.md | 25 ++----- docs/advanced.md | 35 ++++++--- docs/quick-usage.md | 179 +++++++++++++++++++++++--------------------- 3 files changed, 124 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32abee8f..009e3b39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,36 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- - -### Fixed - -- - -### Changed - -- - -### Removed -- Remove deprecated class AbstractHandler - -## [4.1.0] - 2022-03-23 - -### Added +- Added file `config/service.php` to configure broker and authentication - Added AvroSchemaMixedEncoderTest - Added AvroSchemaDecoderTest - Added ProducerWithConfigOptionsTest - Added ConfigOptionsCommand to run commands with ConfigOptions class - Added pt_BR contributing section - Added setup-dev script on composer -- Added grumphp commit validation +- Added grumphp commit validation + +### Fixed -### Fixed - Fixed parameters and options override on Consumer\Config class - Update instructions on contribute section - Update project install section ### Changed + - Updated class from ConfigManager to ConfigOptions on unit tests - Updated class from ConfigManager to ConfigOptions where any config request was made - Consumer and Producer middlewares resolution + +### Removed diff --git a/docs/advanced.md b/docs/advanced.md index f290b06c..acca8234 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -275,29 +275,42 @@ stdout_logfile=/var/log/default/kafka-consumer-price-update.log Although you can run this simple command, it provides some options you can pass to make it more flexible to your needs. -- `--broker=` - - Sometimes, you may want to change which broker the consumer should connect to (maybe for testing/debug purposes). - For that, you just nedd to call the `--broker` option with another broker connection key already set in the `config/kafka.php` file. - - `$ php artisan kafka:consume price-update --broker='some-other-broker'` - `--offset=` And if you need to start the consumption of a topic in a specific offset (it can be useful for debug purposes) you can pass the `--offset=` option, but for this, it will be required to specify the partition too. - `$ php artisan kafka:consume price-update --partition=2 --offset=34` + + $ php artisan kafka:consume price-update --partition=2 --offset=34 + - `--partition=` - If you wish do specify in which partition the consumer must be attached, you can set the option `--partition=`. + Set in which partition the consumer must be attached. + + + $ php artisan kafka:consume price-update --partition=2 --offset=34 - `$ php artisan kafka:consume price-update --partition=2 --offset=34` - `--timeout=` - You can specify what would be the timeout for the consumer, by using the `--timeout=` option, the time is in milliseconds. + Set the timeout for the consumer in milliseconds. + + + $ php artisan kafka:consume price-update --timeout=23000 + + +- `--config_name=` + + Specify from what file topics configuration should be read. + + + $ php artisan kafka:consume topic-name --config_name=config.file + + +- `--service_name=` - `$ php artisan kafka:consume price-update --timeout=23000` + Specify from what file services configurations should be read. + `$ php artisan kafka:consume price-update --service_name=config.file` diff --git a/docs/quick-usage.md b/docs/quick-usage.md index 3d003da4..7fd95905 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -8,76 +8,104 @@ - [Produce Message](#produce-message) -### Config file: `config/kafka.php` - -The config file holds all information about brokers, topics, consumer groups and middlewares. - -To quickly start using, we can focus in two sections: -- Brokers - - An array of brokers, with connection and authentication configurations: - - - `connections`: *required*. can be a `string` with multiple connections separated by comma or an `array` of connections (as `string`) - - - `auth`: *optional*. out of the box, the package can connect with SSL Authentication only or without any authentication - - ```php - 'brokers' => [ - 'price_brokers' => [ - 'connections' => 'localhost:8091,localhost:8092', - 'auth' => [ - 'type' => 'ssl', - 'ca' => storage_path('ca.pem'), - 'certificate' => storage_path('kafka.cert'), - 'key' => storage_path('kafka.key'), - ], - ], - 'stock_brokers' => [ - 'connections' => ['localhost:8091', 'localhost:8092'], - 'auth' => [], // can be an empty array or even don't have this key in the broker config - ], - ], - ``` - -- Topics - - An array of topics configuration, such as the topic name, which broker connection should use, consumer groups and middlewares. - - Here we can specify the group consumers, each topic can have multiple groups, - and each group holds the configuration for which consumer, offset_reset (for setting initial offset) and middleware it must use. - - ```php - 'topics' => [ - 'price_update' => [ - 'topic' => 'products.price.update', - 'broker' => 'price_brokers', - 'consumer_groups' => [ - 'default' => [ - 'offset_reset' => 'smallest', - 'handler' => '\App\Kafka\Consumers\PriceUpdateConsumer', - ], - ], - ], - ], - ``` +### Configure using files + +To get started using configuration files, at least two files are needed. A file to keep the topics +configuration and a file to keep the broker and schema configuration. In this example, we will use the files `config/kafka.php` and `config/service.php`. + +### File `config/kafka.php`: + +This file keeps configurations about topics, consumers and producers. +It should return an array of topics containing the topic name, topic_id, consumer, producer and the settings for each one of them: + + +```php + [ + 'this_is_your_topic_name' => [ + 'topic_id' => "this_is_your_topic_id", + 'consumer' => [ + 'consumer_group' => 'your-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'timeout' => 20000, + 'auto_commit' => true, + 'commit_async' => false, + 'middlewares' => [], + ], + + 'producer' => [ + 'required_acknowledgment' => true, + 'is_async' => true, + 'max_poll_records' => 500, + 'flush_attempts' => 10, + 'middlewares' => [], + 'timeout' => 10000, + 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1, + ], + ] + ], +]; +``` + +### File `config/service.php` + +This file keeps configurations about **broker** and **schema** utilized. + + +```php + [ + 'url' => '', + 'request_options' => [ + 'headers' => [ + 'Authorization' => [ + 'Basic ' . base64_encode( + env('AVRO_SCHEMA_USERNAME').':'.env('AVRO_SCHEMA_PASSWORD') + ), + ], + ], + ], + + 'ssl_verify' => true, + 'username' => 'USERNAME', + 'password' => 'PASSWORD', + ], + + 'broker' => [ + 'connections' => 'kafka:9092', + 'auth' => [ + 'type' => 'ssl', + 'ca' => storage_path('ca.pem'), + 'certificate' => storage_path('kafka.cert'), + 'key' => storage_path('kafka.key'), + ], + ], +]; +``` + ### Consumer -After setting up the required configs, you need to create the consumer, which will handle all records received -from the topic specified in the config. +After setting up the required configuration, you must create a consumer to handle records received +from the specified topic in your configuration. -#### Creating Consumer +#### Creating a Consumer -Creating the consumer is easy as running the following command: +To create a consumer run the following command: ```bash $ php artisan make:kafka-consumer PriceUpdateConsumer ``` -This will create a KafkaConsumer class inside the application, on the app/Kafka/Consumers/ directory - -There, you'll have a handler method, which will send all records from the topic to the Consumer, -also, methods will be available for handling exceptions +This will create a KafkaConsumer class on the app/Kafka/Consumers/ directory with the following +content: ```php use App\Kafka\Consumers\PriceUpdateConsumer; @@ -109,20 +137,19 @@ class PriceUpdateConsumer extends AbstractHandler ``` -#### Running consumer +#### Running the consumer -Now you just need to start consuming the topic. +To start consuming the topic, the simplest way to see it working is by running the kafka:consume command along with the topic name, topic configuration file and service configuration file: -The simplest way to see it working is by running the kafka:consume command along with the topic name -declared in the topics config key: ```bash -$ php artisan kafka:consume price-update +$ php artisan kafka:consume this_is_your_topic_name --config_name=config.file --service_name=service.file ``` This command will run in a `while true`, that means, it will never stop running. But, errors can happen, so we strongly advice you to run this command along with [supervisor](http://supervisord.org/running.html), like this example below: + ```bash [program:kafka-consumer-price-update] process_name=%(program_name)s_%(process_num)02d @@ -137,30 +164,10 @@ stdout_logfile=/var/log/default/kafka-consumer-price-update.log That's it. For more information about usage, middlewares, broker authentication, consumer groups and other advanced topics, please have a look at our [Advanced Usage Guide](advanced.md). - -### Producer - -Producer also required configs, which will produce all records using parameters specified in the config. - -```php - 'brokers' => [ - 'local-dev' => [ - 'connections' => 'kafka:9092', - ], - ], - 'topics' => [ - 'product-updated' => [ - 'topic_id' => 'product_updated', - 'broker' => 'local-dev', - ], - ], -``` ### Produce Message -Creating Producer handler. - -The Producer must extends AbstractHandler class and can be empty. +To create a producer handler, create a class that extends `Metamorphosis\TopicHandler\Producer\AbstractHandler` class: ```php Date: Mon, 4 Apr 2022 15:53:44 -0300 Subject: [PATCH 042/152] chore: update console parameters Added a new option to specify the service file --- src/Connectors/Consumer/Config.php | 2 +- src/Console/ConsumerCommand.php | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index aabcfd8b..81ba46a6 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -54,7 +54,7 @@ public function makeWithConfigOptions(string $handlerClass, ?int $times = null): public function make(array $options, array $arguments): Consumer { $configName = $options['config_name'] ?? 'kafka'; - $service = $options['service'] ?? 'service'; + $service = $options['service_name'] ?? 'service'; $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); $brokerConfig = $this->getBrokerConfig($service); diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index 5c233db4..93420418 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -34,7 +34,8 @@ class ConsumerCommand extends BaseCommand {--broker= : Override broker connection from config.} {--timeout= : Sets timeout for consumer.} {--times= : Amount of messages to be consumed.} - {--config_name= : Change default name for laravel config file.}'; + {--config_name= : Change default name for laravel config file.} + {--service_name= : Change default name for services config file.}'; public function handle(Config $config) { From ab85f7e5b271f8e19071e809f02c3f3305324968 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 16:10:57 -0300 Subject: [PATCH 043/152] chore: remove unused method --- src/Connectors/Consumer/Config.php | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 81ba46a6..33673cae 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -94,18 +94,4 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } - - /** - * Sometimes that user may pass `--partition=0` as argument. - * So if we just use array_filter here, this option will - * be removed. - * - * This code makes sure that only null values will be removed. - */ - private function filterValues(array $options = []): array - { - return array_filter($options, function ($value) { - return !is_null($value); - }); - } } From f25959d4523db6459966180086a5611304e9b267 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 16:46:50 -0300 Subject: [PATCH 044/152] chore: add how to use data objects --- docs/quick-usage.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/quick-usage.md b/docs/quick-usage.md index 7fd95905..f306cba2 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -1,6 +1,7 @@ ## Quick Usage Guide -- [Config file](#config) +- [Configure with files](#config) +- [Configure using data objects](#config-dto) - [Consumer](#consumer) - [Creating Consumer](#creating-consumer) - [Running](#running-consumer) @@ -90,7 +91,6 @@ return [ ]; ``` - ### Consumer @@ -144,7 +144,7 @@ To start consuming the topic, the simplest way to see it working is by running t ```bash $ php artisan kafka:consume this_is_your_topic_name --config_name=config.file --service_name=service.file -``` +``` This command will run in a `while true`, that means, it will never stop running. But, errors can happen, so we strongly advice you to run this command along with [supervisor](http://supervisord.org/running.html), @@ -162,6 +162,27 @@ redirect_stderr=true stdout_logfile=/var/log/default/kafka-consumer-price-update.log ``` +### Using data objects + +To configure and consume using classes: + +```php + use Metamorphosis\Consumer; + use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; + + $topic = config('yourConfig.topics.topic-id'); + $broker = config('yourService.broker'); + $avro = config('yourService.avro_schema'); + + $consumerConfiguration = ConsumerFactory::make($broker, $topic, $avro); + $consumer = app(Consumer::class, ['configOptions' => $consumerConfiguration]); + + $consumer->consume(); +``` + + + + That's it. For more information about usage, middlewares, broker authentication, consumer groups and other advanced topics, please have a look at our [Advanced Usage Guide](advanced.md). From eb023d5f1931b976bc45254c4df51afab9f763b8 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 16:48:54 -0300 Subject: [PATCH 045/152] chore: remove blank spaces --- docs/quick-usage.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/quick-usage.md b/docs/quick-usage.md index f306cba2..5178a312 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -180,9 +180,6 @@ To configure and consume using classes: $consumer->consume(); ``` - - - That's it. For more information about usage, middlewares, broker authentication, consumer groups and other advanced topics, please have a look at our [Advanced Usage Guide](advanced.md). From 708e14e1bce9b374b444f156a087291229b0a21a Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 16:56:07 -0300 Subject: [PATCH 046/152] chore: fix codacy warnings --- CHANGELOG.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 009e3b39..b282caa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added file `config/service.php` to configure broker and authentication -- Added AvroSchemaMixedEncoderTest -- Added AvroSchemaDecoderTest -- Added ProducerWithConfigOptionsTest -- Added ConfigOptionsCommand to run commands with ConfigOptions class -- Added pt_BR contributing section -- Added setup-dev script on composer -- Added grumphp commit validation +- Added file `config/service.php` to configure broker and authentication +- Added AvroSchemaMixedEncoderTest +- Added AvroSchemaDecoderTest +- Added ProducerWithConfigOptionsTest +- Added ConfigOptionsCommand to run commands with ConfigOptions class +- Added pt_BR contributing section +- Added setup-dev script on composer +- Added grumphp commit validation ### Fixed From 1b26ba4702dd546a3efb7c650a12a444bcb7ec8e Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 17:00:33 -0300 Subject: [PATCH 047/152] chore: fix codacy warnings --- CHANGELOG.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b282caa2..b237f25e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added file `config/service.php` to configure broker and authentication -- Added AvroSchemaMixedEncoderTest -- Added AvroSchemaDecoderTest -- Added ProducerWithConfigOptionsTest -- Added ConfigOptionsCommand to run commands with ConfigOptions class -- Added pt_BR contributing section -- Added setup-dev script on composer -- Added grumphp commit validation +- Added AvroSchemaMixedEncoderTest +- Added AvroSchemaDecoderTest +- Added ProducerWithConfigOptionsTest +- Added ConfigOptionsCommand to run commands with ConfigOptions class +- Added pt_BR contributing section +- Added setup-dev script on composer +- Added grumphp commit validation +- Added file `config/service.php` to configure broker and authentication ### Fixed From 84cda5b0b8255ccf81fee1acad76567cf25d85c9 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 17:39:32 -0300 Subject: [PATCH 048/152] docs: add commens explaining parameters --- config/kafka.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/kafka.php b/config/kafka.php index 9e74693b..3e7f0764 100644 --- a/config/kafka.php +++ b/config/kafka.php @@ -2,8 +2,14 @@ return [ 'topics' => [ + // This is your topic "keyword" where you will put all configurations needed + // on this specific topic. 'default' => [ + // The topic id is where you want to send or consume + // your messages from kafka. 'topic_id' => 'kafka-test', + + //your consumer configurations 'consumer' => [ 'consumer_group' => 'test-consumer-group', // Action to take when there is no initial From 4b618a58609d67de1a5c0f38c2674ca282d40a98 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 17:41:06 -0300 Subject: [PATCH 049/152] docs: document finished method --- docs/advanced.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index acca8234..14e40d1e 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -181,10 +181,17 @@ $ php artisan make:kafka-consumer PriceUpdateHandler This will create a KafkaConsumer class inside the application, on the `app/Kafka/Consumers/` directory. There, you'll have a `handler` method, which will send all records from the topic to the Consumer. -Methods will be available for handling exceptions: + +Available methods: + - `warning` method will be call whenever something not critical is received from the topic. Like a message informing that there's no more records to consume. - - `failure` method will be call whenever something critical happens, like an error to decode the record. + + + - `failure` will be call whenever something critical happens, like an error to decode the record. + + + - `finished` will be call when queue finishes ```php use App\Repository; @@ -230,6 +237,11 @@ class PriceUpdateHandler extends AbstractHandler { // handle failure exception } + + public function finished(): void + { + //handle queue end + } } ``` From 9822cf81c6f171cff6839c90204a93d260d2d2d7 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 17:41:52 -0300 Subject: [PATCH 050/152] docs: add anchor to section --- docs/quick-usage.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/quick-usage.md b/docs/quick-usage.md index 5178a312..b2cf9a92 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -162,7 +162,8 @@ redirect_stderr=true stdout_logfile=/var/log/default/kafka-consumer-price-update.log ``` -### Using data objects + +#### Using data objects To configure and consume using classes: From 2ba105064dace665a0644b844578455adbb5da82 Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 5 Apr 2022 14:34:32 -0300 Subject: [PATCH 051/152] chore: remove blank spaces on docs --- CHANGELOG.md | 2 +- docs/advanced.md | 10 ++++------ docs/quick-usage.md | 2 -- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b237f25e..47140732 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added ConfigOptionsCommand to run commands with ConfigOptions class - Added pt_BR contributing section - Added setup-dev script on composer -- Added grumphp commit validation +- Added grumphp commit validation - Added file `config/service.php` to configure broker and authentication ### Fixed diff --git a/docs/advanced.md b/docs/advanced.md index 14e40d1e..686d80ef 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -136,8 +136,7 @@ If you wish, you may set a middleware to run of a topic level or a consumer grou ], ``` -The order matters here, they'll be execute as queue, from the most global scope to the most specific (global scope > topic scope > group_consumers scope). - +The order matters here, they'll execute as queue, from the most global scope to the most specific (global scope > topic scope > group_consumers scope). ### Schemas @@ -245,7 +244,6 @@ class PriceUpdateHandler extends AbstractHandler } ``` - #### Creating Middleware You can create a middleware class, that works between the received data from broker and before being passed into consumers, using the follow command: @@ -287,7 +285,6 @@ stdout_logfile=/var/log/default/kafka-consumer-price-update.log Although you can run this simple command, it provides some options you can pass to make it more flexible to your needs. - - `--offset=` And if you need to start the consumption of a topic in a specific offset (it can be useful for debug purposes) @@ -323,6 +320,7 @@ Although you can run this simple command, it provides some options you can pass - `--service_name=` - Specify from what file services configurations should be read. + Specify from what file services configurations should be read. + - `$ php artisan kafka:consume price-update --service_name=config.file` + $ php artisan kafka:consume price-update --service_name=config.file diff --git a/docs/quick-usage.md b/docs/quick-usage.md index b2cf9a92..88488205 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -19,7 +19,6 @@ configuration and a file to keep the broker and schema configuration. In this ex This file keeps configurations about topics, consumers and producers. It should return an array of topics containing the topic name, topic_id, consumer, producer and the settings for each one of them: - ```php Date: Tue, 5 Apr 2022 14:50:42 -0300 Subject: [PATCH 052/152] chore: fix codacy --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47140732..101df70f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added pt_BR contributing section - Added setup-dev script on composer - Added grumphp commit validation -- Added file `config/service.php` to configure broker and authentication ### Fixed From c6e3f20e1279d3f21922d8d552f0b7efbb420783 Mon Sep 17 00:00:00 2001 From: Hugo Carvalho Date: Thu, 7 Apr 2022 11:03:20 -0300 Subject: [PATCH 053/152] Update docs/quick-usage.md Apply suggested changes Co-authored-by: Diego Felix --- docs/quick-usage.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/quick-usage.md b/docs/quick-usage.md index 88488205..d6a143c8 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -12,7 +12,8 @@ ### Configure using files To get started using configuration files, at least two files are needed. A file to keep the topics -configuration and a file to keep the broker and schema configuration. In this example, we will use the files `config/kafka.php` and `config/service.php`. +configuration and a file to keep the broker and schema configuration. In this example, we will use the files +`config/kafka.php` and `config/service.php`. ### File `config/kafka.php`: From e51273f110226abe955fdb6e965b513bea1e10d2 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 25 Apr 2022 11:10:53 -0300 Subject: [PATCH 054/152] fix(codacy): fix codacy warnings --- CHANGELOG.md | 30 +++-- docs/quick-usage.pt.md | 154 ++++++++++++++++---------- readme.md | 1 + src/Connectors/Consumer/Config.php | 2 +- src/Console/ConfigOptionsCommand.php | 3 +- src/Middlewares/AvroSchemaDecoder.php | 6 - 6 files changed, 109 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 101df70f..342b1ef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,28 +5,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] - ### Added -- Added AvroSchemaMixedEncoderTest -- Added AvroSchemaDecoderTest -- Added ProducerWithConfigOptionsTest -- Added ConfigOptionsCommand to run commands with ConfigOptions class -- Added pt_BR contributing section -- Added setup-dev script on composer -- Added grumphp commit validation +- Added AvroSchemaMixedEncoderTest +- Added AvroSchemaDecoderTest +- Added ProducerWithConfigOptionsTest +- Added ConfigOptionsCommand to run commands with ConfigOptions class +- Added pt_BR contributing section +- Added setup-dev script on composer +- Added grumphp commit validation ### Fixed -- Fixed parameters and options override on Consumer\Config class -- Update instructions on contribute section -- Update project install section +- Fixed parameters and options override on Consumer\Config class +- Update instructions on contribute section +- Update project install section ### Changed -- Updated class from ConfigManager to ConfigOptions on unit tests -- Updated class from ConfigManager to ConfigOptions where any config request was made -- Consumer and Producer middlewares resolution - -### Removed +- Updated class from ConfigManager to ConfigOptions on unit tests +- Updated class from ConfigManager to ConfigOptions where any config request was made +- Consumer and Producer middlewares resolution diff --git a/docs/quick-usage.pt.md b/docs/quick-usage.pt.md index daf4dbfd..c3e14753 100644 --- a/docs/quick-usage.pt.md +++ b/docs/quick-usage.pt.md @@ -1,64 +1,96 @@ -## Guia rápido +## Quick Usage Guide -- [Arquivo de configuração](#config) -- [Consumer](#consumer) - - [Criando um Consumer](#creating-consumer) - - [Rodando o Consumer](#running-consumer) +- [Configurar usando arquivos](#config) +- [Configurar usando objetos](#config-dto) +- [Consumidor](#consumer) + - [Criando um consumidor](#creating-consumer) + - [Executando um consumidor](#running-consumer) +- [Produtor](#producer) + - [Produzindo mensagens](#produce-message) -### Arquivo de configuração: `config/kafka.php` - -Esse arquivo contém todas as informações sobre *brokers*, tópicos, *consumer groups* e *middlewares*. - -Para começar a usar, podemos focar em duas seções: - -- Brokers - - Uma lista de *brokers*, com configurações de conexão e autenticação. - - - `connections`: *obrigatório*. Pode ser uma `string` com múltiplas conexões separadas por vírgula ou uma `array` de conexões. - - - `auth`: *opcional*. É possivel se conectar sem autenticação ou usando autenticação SSL. - - ```php - 'brokers' => [ - 'price_brokers' => [ - 'connections' => 'localhost:8091,localhost:8092', - 'auth' => [ - 'type' => 'ssl', - 'ca' => storage_path('ca.pem'), - 'certificate' => storage_path('kafka.cert'), - 'key' => storage_path('kafka.key'), - ], - ], - 'stock_brokers' => [ - 'connections' => ['localhost:8091', 'localhost:8092'], - 'auth' => [], // pode ser uma array vazia ou até mesmo não ter essa chave aqui. - ], - ], - ``` - -- Tópicos - - Uma lista de configuração de tópicos, como nome, qual *broker* usar, *consumer group* e *middlewares*. - - Aqui você pode especificar os *consumer groups*. Cada tópico pode ter vários grupos, - e cada grupo tem a sua configuração para cada *consumer*, *offset_reset* (para definir um *offset* inicial) e *middlewares* que devem ser usados. - - ```php - 'topics' => [ - 'price_update' => [ - 'topic' => 'products.price.update', - 'broker' => 'price_brokers', - 'consumer_groups' => [ - 'default' => [ - 'offset_reset' => 'smallest', - 'handler' => '\App\Kafka\Consumers\PriceUpdateConsumer', - ], - ], - ], - ], - ``` +### Configurar usando arquivos + +Para começar a usar arquivos de configuração, são necessários pelo menos dois arquivos. Um arquivo para manter os tópicos +configuração e um arquivo para manter a configuração do broker e do esquema. Neste exemplo, usaremos os arquivos +`config/kafka.php` e `config/service.php`. + + +### Arquivo `config/kafka.php`: + +Este arquivo mantém configurações sobre tópicos, consumidores e produtores. +Deve retornar um array de tópicos contendo o nome do tópico, topic_id, consumidor, produtor e as configurações de cada um deles: + + +```php + [ + 'this_is_your_topic_name' => [ + 'topic_id' => "this_is_your_topic_id", + 'consumer' => [ + 'consumer_group' => 'your-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'timeout' => 20000, + 'auto_commit' => true, + 'commit_async' => false, + 'middlewares' => [], + ], + + 'producer' => [ + 'required_acknowledgment' => true, + 'is_async' => true, + 'max_poll_records' => 500, + 'flush_attempts' => 10, + 'middlewares' => [], + 'timeout' => 10000, + 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1, + ], + ] + ], +]; +``` + +### File `config/service.php` + +Esse arquivo possui as configurações de **broker** e **schema** utilizados. + +```php + [ + 'url' => '', + 'request_options' => [ + 'headers' => [ + 'Authorization' => [ + 'Basic ' . base64_encode( + env('AVRO_SCHEMA_USERNAME').':'.env('AVRO_SCHEMA_PASSWORD') + ), + ], + ], + ], + + 'ssl_verify' => true, + 'username' => 'USERNAME', + 'password' => 'PASSWORD', + ], + + 'broker' => [ + 'connections' => 'kafka:9092', + 'auth' => [ + 'type' => 'ssl', + 'ca' => storage_path('ca.pem'), + 'certificate' => storage_path('kafka.cert'), + 'key' => storage_path('kafka.key'), + ], + ], +]; +``` ### Consumer @@ -113,11 +145,11 @@ class PriceUpdateConsumer extends AbstractHandler Agora é só consumir o tópico. -A forma mais simples de ver tudo isso funcionando é rodando o comando `kafka:consume` com o nome do tópico que foi configurado: +Para começar a consumir o tópico, a maneira mais simples de vê-lo funcionando é executando o comando kafka:consume junto com o nome do tópico, arquivo de configuração do tópico e arquivo de configuração do serviço: ```bash -$ php artisan kafka:consume price-update -``` +$ php artisan kafka:consume this_is_your_topic_name --config_name=config.file --service_name=service.file +``` Esse comando rodará em um laço infinito (while true), isso significa que ele nunca irá parar de rodar por conta própria. Mas erros podem acontecer, então, recomendamos fortemente que você execute este comando com o auxílio de um [supervisor](http://supervisord.org/running.html), como no exemplo abaixo: diff --git a/readme.md b/readme.md index 76b81013..9206ab91 100644 --- a/readme.md +++ b/readme.md @@ -15,6 +15,7 @@ - [Installation](#installation) - [Quick Usage Guide](docs/quick-usage.md) - [Advanced Usage Guide](docs/advanced.md) +- [Upgrade Guide](docs/upgrade.md) - [Contributing](docs/CONTRIBUTING.md) - [License](#license) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 33673cae..fd74f86c 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -40,7 +40,7 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; - public function makeWithConfigOptions(string $handlerClass, ?int $times = null): ?Consumer + public function makeWithConfigOptions(string $handlerClass): ?Consumer { $handler = app($handlerClass); $configOptions = $handler->getConfigOptions(); diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index eb4c1071..c65bf454 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Console; use Illuminate\Console\Command as BaseCommand; -use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; @@ -26,7 +25,7 @@ class ConfigOptionsCommand extends BaseCommand {handler : handler.} {--times= : Amount of messages to be consumed.}'; - public function handle(Config $config) + public function handle() { $consumerHandler = app($this->argument('handler')); diff --git a/src/Middlewares/AvroSchemaDecoder.php b/src/Middlewares/AvroSchemaDecoder.php index 1aeffcff..d7a39f90 100644 --- a/src/Middlewares/AvroSchemaDecoder.php +++ b/src/Middlewares/AvroSchemaDecoder.php @@ -7,18 +7,12 @@ use Metamorphosis\Avro\Serializer\MessageDecoder; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; -use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema; use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; class AvroSchemaDecoder implements MiddlewareInterface { private MessageDecoder $decoder; - /** - * @var AvroSchema - */ - private $avroSchema; - public function __construct(ClientFactory $factory, ConsumerConfigOptions $consumerConfigOptions) { if (!$consumerConfigOptions->getAvroSchema()->getUrl()) { From 5a3fcda7a8553afc9dde8f91a5142d85731d9a5c Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 3 May 2022 16:12:16 -0300 Subject: [PATCH 055/152] docs: add upgrade guide --- docs/upgrade.md | 76 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 docs/upgrade.md diff --git a/docs/upgrade.md b/docs/upgrade.md new file mode 100644 index 00000000..5c4ef6fe --- /dev/null +++ b/docs/upgrade.md @@ -0,0 +1,76 @@ + +## Upgrade guide + +To upgrade from version X.x to version X.y: + +Move your `avroschema` and `broker` section from old `config/kafka.php` file into a new file: + + +```php + [ + 'this_is_your_topic_name' => [ + 'topic_id' => "this_is_your_topic_id", + 'consumer' => [ + 'consumer_group' => 'your-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'timeout' => 20000, + 'auto_commit' => true, + 'commit_async' => false, + 'middlewares' => [], + ], + + 'producer' => [ + 'required_acknowledgment' => true, + 'is_async' => true, + 'max_poll_records' => 500, + 'flush_attempts' => 10, + 'middlewares' => [], + 'timeout' => 10000, + 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1, + ], + ] + ], +]; +``` + +Upgrade your topic configuration files: + +```php + [ + 'this_is_your_topic_name' => [ + 'topic_id' => "this_is_your_topic_id", + 'consumer' => [ + 'consumer_group' => 'your-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'timeout' => 20000, + 'auto_commit' => true, + 'commit_async' => false, + 'middlewares' => [], + ], + + 'producer' => [ + 'required_acknowledgment' => true, + 'is_async' => true, + 'max_poll_records' => 500, + 'flush_attempts' => 10, + 'middlewares' => [], + 'timeout' => 10000, + 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1, + ], + ] + ], +]; +``` + From 066df6da91bb5f35bb49854064be94ea525fc146 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 5 Apr 2023 18:48:45 -0300 Subject: [PATCH 056/152] fix: add handleMiddlewares method --- src/Connectors/Consumer/Config.php | 8 ++++++++ tests/Unit/AbstractConfigManagerTest.php | 0 2 files changed, 8 insertions(+) delete mode 100644 tests/Unit/AbstractConfigManagerTest.php diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index fd74f86c..f868a2cb 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -94,4 +94,12 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } + + private function getMiddlewares(string $configName, array $topicConfig): array + { + return array_merge( + config($configName . '.middlewares.consumer', []), + $topicConfig['consumer']['middlewares'] ?? [] + ); + } } diff --git a/tests/Unit/AbstractConfigManagerTest.php b/tests/Unit/AbstractConfigManagerTest.php deleted file mode 100644 index e69de29b..00000000 From 402a59da91e71df409b5859e60c9bd4743976688 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 14:01:28 -0300 Subject: [PATCH 057/152] style: fix coding standard violations Signed-off-by: hcdias --- config/service.php | 6 ++- src/Authentication/Factory.php | 10 ++++- src/Authentication/SASLAuthentication.php | 10 ++--- src/Authentication/SSLAuthentication.php | 10 ++--- src/Connectors/AbstractConfig.php | 9 +++-- src/Connectors/Consumer/Config.php | 6 ++- src/Connectors/Consumer/Factory.php | 39 +++++++++++++------ src/Connectors/Consumer/LowLevel.php | 15 +++++-- src/Connectors/Producer/Connector.php | 6 ++- src/Console/ConfigOptionsCommand.php | 7 ++-- src/Console/ConsumerCommand.php | 6 +-- src/MetamorphosisServiceProvider.php | 8 ++-- src/Middlewares/AvroSchemaDecoder.php | 8 +++- src/Middlewares/AvroSchemaMixedEncoder.php | 25 +++++++----- src/Producer.php | 28 ++++++++++--- .../Factories/ConsumerFactory.php | 5 ++- .../Producer/AbstractProducer.php | 2 +- 17 files changed, 137 insertions(+), 63 deletions(-) diff --git a/config/service.php b/config/service.php index 82cb1299..64f81077 100644 --- a/config/service.php +++ b/config/service.php @@ -6,8 +6,10 @@ 'request_options' => [ 'headers' => [ 'Authorization' => [ - 'Basic '.base64_encode( - env('AVRO_SCHEMA_USERNAME').':'.env('AVRO_SCHEMA_PASSWORD') + 'Basic' . base64_encode( + env('AVRO_SCHEMA_USERNAME') . ':' . env( + 'AVRO_SCHEMA_PASSWORD' + ) ), ], ], diff --git a/src/Authentication/Factory.php b/src/Authentication/Factory.php index c8a8f38a..55fb7053 100644 --- a/src/Authentication/Factory.php +++ b/src/Authentication/Factory.php @@ -24,11 +24,17 @@ public static function authenticate(Conf $conf, AuthInterface $configOptions): v break; case self::TYPE_SSL: - app(SSLAuthentication::class, compact('conf', 'configOptions')); + app( + SSLAuthentication::class, + compact('conf', 'configOptions') + ); break; case self::TYPE_SASL_SSL: - app(SASLAuthentication::class, compact('conf', 'configOptions')); + app( + SASLAuthentication::class, + compact('conf', 'configOptions') + ); break; default: diff --git a/src/Authentication/SASLAuthentication.php b/src/Authentication/SASLAuthentication.php index 3918558d..f4a0e3b0 100644 --- a/src/Authentication/SASLAuthentication.php +++ b/src/Authentication/SASLAuthentication.php @@ -9,10 +9,7 @@ class SASLAuthentication implements AuthenticationInterface { private Conf $conf; - /** - * @var SaslSsl - */ - private $configOptions; + private SaslSsl $configOptions; public function __construct(Conf $conf, SaslSsl $configOptions) { @@ -29,7 +26,10 @@ private function authenticate(): void // The mechanisms key is optional when configuring this kind of authentication // If the user does not specify the mechanism, the default will be 'PLAIN'. // But, to make config more clear, we are asking the user every time. - $this->conf->set('sasl.mechanisms', $this->configOptions->getMechanisms()); + $this->conf->set( + 'sasl.mechanisms', + $this->configOptions->getMechanisms() + ); $this->conf->set('sasl.username', $this->configOptions->getUsername()); $this->conf->set('sasl.password', $this->configOptions->getPassword()); } diff --git a/src/Authentication/SSLAuthentication.php b/src/Authentication/SSLAuthentication.php index 5dfc06e4..e698dfe5 100644 --- a/src/Authentication/SSLAuthentication.php +++ b/src/Authentication/SSLAuthentication.php @@ -9,10 +9,7 @@ class SSLAuthentication implements AuthenticationInterface { private Conf $conf; - /** - * @var Ssl - */ - private $configOptions; + private Ssl $configOptions; public function __construct(Conf $conf, Ssl $configOptions) { @@ -26,7 +23,10 @@ private function authenticate(): void { $this->conf->set('security.protocol', $this->configOptions->getType()); $this->conf->set('ssl.ca.location', $this->configOptions->getCa()); - $this->conf->set('ssl.certificate.location', $this->configOptions->getCertificate()); + $this->conf->set( + 'ssl.certificate.location', + $this->configOptions->getCertificate() + ); $this->conf->set('ssl.key.location', $this->configOptions->getKey()); } } diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index fa8b8ac1..b71c683c 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -9,15 +9,18 @@ abstract class AbstractConfig { protected function getBrokerConfig(string $servicesFile): array { - if (!$brokerConfig = config($servicesFile.'.broker')) { - throw new ConfigurationException("Broker configuration not found on '{$servicesFile}'"); + if (!$brokerConfig = config($servicesFile . '.broker')) { + throw new ConfigurationException( + "Broker configuration not found on '{$servicesFile}'" + ); } + return $brokerConfig; } protected function getSchemaConfig(string $servicesFile): array { - return config($servicesFile.'.avro_schema', []); + return config($servicesFile . '.avro_schema', []); } protected function validate(array $config): void diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index f868a2cb..d13976be 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -74,7 +74,11 @@ public function make(array $options, array $arguments): Consumer } } - return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); + return ConsumerFactory::make( + $brokerConfig, + $topicConfig, + $schemaConfig + ); } /** diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index c82c4ad2..923fb817 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -26,32 +26,49 @@ public static function make(ConsumerConfigOptions $configOptions): Manager $middlewares = $configOptions->getMiddlewares(); foreach ($middlewares as &$middleware) { - $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $configOptions]) : $middleware; + $middleware = is_string($middleware) + ? app( + $middleware, + ['consumerConfigOptions' => $configOptions] + ) + : $middleware; } - $middlewares[] = app(ConsumerMiddleware::class, ['consumerTopicHandler' => $handler]); + $middlewares[] = app( + ConsumerMiddleware::class, + ['consumerTopicHandler' => $handler] + ); $dispatcher = self::getMiddlewareDispatcher($middlewares); - return new Manager($consumer, $handler, $dispatcher, $autoCommit, $commitAsync); - } - - protected static function requiresPartition(ConsumerConfigOptions $configOptions): bool - { - $partition = $configOptions->getPartition(); - - return !is_null($partition) && $partition >= 0; + return new Manager( + $consumer, + $handler, + $dispatcher, + $autoCommit, + $commitAsync + ); } public static function getConsumer(bool $autoCommit, ConsumerConfigOptions $configOptions): ConsumerInterface { if (self::requiresPartition($configOptions)) { - return app(LowLevel::class)->getConsumer($autoCommit, $configOptions); + return app(LowLevel::class)->getConsumer( + $autoCommit, + $configOptions + ); } return app(HighLevel::class)->getConsumer($autoCommit, $configOptions); } + protected static function requiresPartition(ConsumerConfigOptions $configOptions): bool + { + $partition = $configOptions->getPartition(); + + return !is_null($partition) && $partition >= 0; + } + private static function getMiddlewareDispatcher(array $middlewares): Dispatcher { return new Dispatcher($middlewares); diff --git a/src/Connectors/Consumer/LowLevel.php b/src/Connectors/Consumer/LowLevel.php index e897d467..33d0c1de 100644 --- a/src/Connectors/Consumer/LowLevel.php +++ b/src/Connectors/Consumer/LowLevel.php @@ -27,9 +27,15 @@ public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): Con $consumer->addBrokers($broker->getConnections()); $topicConf = $this->getTopicConfigs($configOptions); - $topicConsumer = $consumer->newTopic($configOptions->getTopicId(), $topicConf); + $topicConsumer = $consumer->newTopic( + $configOptions->getTopicId(), + $topicConf + ); - $topicConsumer->consumeStart($configOptions->getPartition(), $configOptions->getOffset()); + $topicConsumer->consumeStart( + $configOptions->getPartition(), + $configOptions->getOffset() + ); return new LowLevelConsumer($topicConsumer, $configOptions); } @@ -41,7 +47,10 @@ protected function getTopicConfigs(ConfigOptions $configOptions) // Set where to start consuming messages when there is no initial offset in // offset store or the desired offset is out of range. // 'smallest': start from the beginning - $topicConfig->set('auto.offset.reset', $configOptions->getOffsetReset()); + $topicConfig->set( + 'auto.offset.reset', + $configOptions->getOffsetReset() + ); return $topicConfig; } diff --git a/src/Connectors/Producer/Connector.php b/src/Connectors/Producer/Connector.php index 1109a598..3d427afc 100644 --- a/src/Connectors/Producer/Connector.php +++ b/src/Connectors/Producer/Connector.php @@ -12,8 +12,10 @@ class Connector { - public function getProducerTopic(HandlerInterface $handler, ProducerConfigOptions $producerConfigOptions): KafkaProducer - { + public function getProducerTopic( + HandlerInterface $handler, + ProducerConfigOptions $producerConfigOptions + ): KafkaProducer { $conf = resolve(Conf::class); if ($this->canHandleResponse($handler)) { diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index c65bf454..d592a8b1 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -1,4 +1,5 @@ getTopicId().PHP_EOL; - $text .= ' on consumer group: '.$configOptions->getConsumerGroup().PHP_EOL; - $text .= 'Connecting in '.$configOptions->getBroker()->getConnections().PHP_EOL; + $text = 'Starting consumer for topic: ' . $configOptions->getTopicId() . PHP_EOL; + $text .= ' on consumer group: ' . $configOptions->getConsumerGroup() . PHP_EOL; + $text .= 'Connecting in ' . $configOptions->getBroker()->getConnections() . PHP_EOL; $text .= 'Running consumer..'; $this->output->writeln($text); diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index 93420418..ded31d7f 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -51,9 +51,9 @@ public function handle(Config $config) private function writeStartingConsumer(Consumer $consumer) { - $text = 'Starting consumer for topic: '.$consumer->getTopicId().PHP_EOL; - $text .= ' on consumer group: '.$consumer->getConsumerGroup().PHP_EOL; - $text .= 'Connecting in '.$consumer->getBroker()->getConnections().PHP_EOL; + $text = 'Starting consumer for topic: ' . $consumer->getTopicId() . PHP_EOL; + $text .= ' on consumer group: ' . $consumer->getConsumerGroup() . PHP_EOL; + $text .= 'Connecting in ' . $consumer->getBroker()->getConnections() . PHP_EOL; $text .= 'Running consumer..'; $this->output->writeln($text); diff --git a/src/MetamorphosisServiceProvider.php b/src/MetamorphosisServiceProvider.php index 36dffdf5..6a64d8c4 100644 --- a/src/MetamorphosisServiceProvider.php +++ b/src/MetamorphosisServiceProvider.php @@ -14,12 +14,12 @@ class MetamorphosisServiceProvider extends ServiceProvider public function boot() { $this->publishes([ - __DIR__.'/../config/kafka.php' => config_path('kafka.php'), - __DIR__.'/../config/service.php' => config_path('service.php'), + __DIR__ . '/../config/kafka.php' => config_path('kafka.php'), + __DIR__ . '/../config/service.php' => config_path('service.php'), ], 'config'); - $this->mergeConfigFrom(__DIR__.'/../config/kafka.php', 'kafka'); - $this->mergeConfigFrom(__DIR__.'/../config/service.php', 'service'); + $this->mergeConfigFrom(__DIR__ . '/../config/kafka.php', 'kafka'); + $this->mergeConfigFrom(__DIR__ . '/../config/service.php', 'service'); } public function register() diff --git a/src/Middlewares/AvroSchemaDecoder.php b/src/Middlewares/AvroSchemaDecoder.php index d7a39f90..07f2193d 100644 --- a/src/Middlewares/AvroSchemaDecoder.php +++ b/src/Middlewares/AvroSchemaDecoder.php @@ -16,10 +16,14 @@ class AvroSchemaDecoder implements MiddlewareInterface public function __construct(ClientFactory $factory, ConsumerConfigOptions $consumerConfigOptions) { if (!$consumerConfigOptions->getAvroSchema()->getUrl()) { - throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaDecoder Middleware"); + throw new ConfigurationException( + "Avro schema url not found, it's required to use AvroSchemaDecoder Middleware" + ); } - $this->decoder = new MessageDecoder($factory->make($consumerConfigOptions->getAvroSchema())); + $this->decoder = new MessageDecoder( + $factory->make($consumerConfigOptions->getAvroSchema()) + ); } public function process(RecordInterface $record, Closure $next) diff --git a/src/Middlewares/AvroSchemaMixedEncoder.php b/src/Middlewares/AvroSchemaMixedEncoder.php index 6969a80e..67c09076 100644 --- a/src/Middlewares/AvroSchemaMixedEncoder.php +++ b/src/Middlewares/AvroSchemaMixedEncoder.php @@ -21,18 +21,22 @@ class AvroSchemaMixedEncoder implements MiddlewareInterface private CachedSchemaRegistryClient $schemaRegistry; - /** - * @var ProducerConfigOptions - */ - private $producerConfigOptions; + private ProducerConfigOptions $producerConfigOptions; - public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, ProducerConfigOptions $producerConfigOptions) - { + public function __construct( + SchemaId $schemaIdEncoder, + ClientFactory $factory, + ProducerConfigOptions $producerConfigOptions + ) { if (!$producerConfigOptions->getAvroSchema()->getUrl()) { - throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaEncoder Middleware"); + throw new ConfigurationException( + "Avro schema url not found, it's required to use AvroSchemaEncoder Middleware" + ); } - $schemaRegistry = $factory->make($producerConfigOptions->getAvroSchema()); + $schemaRegistry = $factory->make( + $producerConfigOptions->getAvroSchema() + ); $this->schemaIdEncoder = $schemaIdEncoder; $this->schemaRegistry = $schemaRegistry; $this->producerConfigOptions = $producerConfigOptions; @@ -41,7 +45,10 @@ public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, P public function process(RecordInterface $record, Closure $next) { $topic = $this->producerConfigOptions->getTopicId(); - $schema = $this->schemaRegistry->getBySubjectAndVersion("{$topic}-value", 'latest'); + $schema = $this->schemaRegistry->getBySubjectAndVersion( + "{$topic}-value", + 'latest' + ); $arrayPayload = json_decode($record->getPayload(), true); $encodedPayload = $this->schemaIdEncoder->encode( $schema, diff --git a/src/Producer.php b/src/Producer.php index 4fa9ee5b..b3211f7c 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -36,20 +36,36 @@ public function build(HandlerInterface $producerHandler): Dispatcher $middlewares = $producerConfigOptions->getMiddlewares(); foreach ($middlewares as &$middleware) { - $middleware = is_string($middleware) ? app($middleware, ['producerConfigOptions' => $producerConfigOptions]) : $middleware; + $middleware = is_string($middleware) + ? app( + $middleware, + ['producerConfigOptions' => $producerConfigOptions] + ) + : $middleware; } - $middlewares[] = $this->getProducerMiddleware($producerHandler, $producerConfigOptions); + $middlewares[] = $this->getProducerMiddleware( + $producerHandler, + $producerConfigOptions + ); return new Dispatcher($middlewares); } - public function getProducerMiddleware(HandlerInterface $producerHandler, ProducerConfigOptions $producerConfigOptions): ProducerMiddleware - { - $producer = $this->connector->getProducerTopic($producerHandler, $producerConfigOptions); + public function getProducerMiddleware( + HandlerInterface $producerHandler, + ProducerConfigOptions $producerConfigOptions + ): ProducerMiddleware { + $producer = $this->connector->getProducerTopic( + $producerHandler, + $producerConfigOptions + ); $topic = $producer->newTopic($producerConfigOptions->getTopicId()); - $poll = app(Poll::class, ['producer' => $producer, 'producerConfigOptions' => $producerConfigOptions]); + $poll = app( + Poll::class, + ['producer' => $producer, 'producerConfigOptions' => $producerConfigOptions] + ); $partition = $producerConfigOptions->getPartition(); return app( diff --git a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php index 4dd62736..2d28a28c 100644 --- a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php +++ b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php @@ -25,7 +25,10 @@ private static function getConsumerGroupConfig(array $topicData): array $consumer = $topicData['consumer']; $topicData['consumerGroup'] = $consumer['consumer_group']; - return array_merge_recursive($topicData, self::convertConfigAttributes($consumer)); + return array_merge_recursive( + $topicData, + self::convertConfigAttributes($consumer) + ); } private static function convertConfigAttributes(array $consumerConfig): array diff --git a/src/TopicHandler/Producer/AbstractProducer.php b/src/TopicHandler/Producer/AbstractProducer.php index 694aedd0..ecab7c93 100644 --- a/src/TopicHandler/Producer/AbstractProducer.php +++ b/src/TopicHandler/Producer/AbstractProducer.php @@ -20,7 +20,7 @@ class AbstractProducer implements HandlerInterface */ protected $producer; - public function __construct($record, Producer $configOptions, string $key = null) + public function __construct($record, Producer $configOptions, ?string $key = null) { $this->record = $record; $this->key = $key; From d886eb4af17c2c30a1e21babcd128099b9ac715f Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 14:15:25 -0300 Subject: [PATCH 058/152] style: fix coding standard violations Signed-off-by: hcdias --- tests/Integration/ConsumerTest.php | 5 +- tests/Integration/ProducerTest.php | 38 +++++---- tests/Integration/ProducerWithAvroTest.php | 12 ++- tests/Unit/Authentication/FactoryTest.php | 6 +- .../Authentication/SASLAuthenticationTest.php | 6 +- .../Authentication/SSLAuthenticationTest.php | 6 +- tests/Unit/Connectors/Consumer/ConfigTest.php | 4 +- .../Unit/Connectors/Consumer/FactoryTest.php | 83 ++++++++++--------- .../Connectors/Producer/ConnectorTest.php | 14 +++- tests/Unit/Console/ConsumerCommandTest.php | 54 ++++++------ tests/Unit/Consumers/LowLevelTest.php | 5 +- .../Middlewares/AvroSchemaDecoderTest.php | 10 ++- .../AvroSchemaMixedEncoderTest.php | 32 +++++-- tests/Unit/ProducerTest.php | 10 +-- 14 files changed, 177 insertions(+), 108 deletions(-) diff --git a/tests/Integration/ConsumerTest.php b/tests/Integration/ConsumerTest.php index 1d4397e8..2fb7b1bf 100644 --- a/tests/Integration/ConsumerTest.php +++ b/tests/Integration/ConsumerTest.php @@ -59,7 +59,10 @@ public function testItShouldSetup(): void $saleOrderDispatcher = Metamorphosis::build($messageProducer); $saleOrderDispatcher->handle($messageProducer->createRecord()); - $consumer = $this->app->make(Consumer::class, ['configOptions' => $consumerConfigOptions]); + $consumer = $this->app->make( + Consumer::class, + ['configOptions' => $consumerConfigOptions] + ); $expected = '{"id":"MESSAGE_ID"}'; // Actions diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 94ab5626..8fbe5305 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -18,22 +18,9 @@ class ProducerTest extends LaravelTestCase protected string $firstLowLevelMessage; - /** - * @var string - */ - protected $secondLowLevelMessage; + protected string $secondLowLevelMessage; - /** - * @var string - */ - protected $topicId; - - protected function setUp(): void - { - parent::setUp(); - $this->withoutAuthentication(); - $this->topicId = 'kafka-test-'.Str::random(5); - } + protected string $topicId; /** * @group runProducer @@ -66,6 +53,14 @@ public function testShouldRunAProducerAndReceiveMessagesWithALowLevelConsumer(): $this->runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingToTwoMessagesConsumed(); } + protected function setUp(): void + { + parent::setUp(); + + $this->withoutAuthentication(); + $this->topicId = 'kafka-test-' . Str::random(5); + } + protected function withoutAuthentication(): void { config(['service.broker.auth' => []]); @@ -73,7 +68,9 @@ protected function withoutAuthentication(): void protected function haveAConsumerHandlerConfigured(): void { - config(['kafka.topics.default.consumer.handler' => MessageConsumer::class]); + config( + ['kafka.topics.default.consumer.handler' => MessageConsumer::class] + ); } protected function haveAConsumerPartitionConfigured(): void @@ -145,7 +142,9 @@ private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $producerConfigOptions = $this->createProducerConfigOptions($this->topicId); + $producerConfigOptions = $this->createProducerConfigOptions( + $this->topicId + ); $producer = app(MessageProducer::class, [ 'record' => $this->highLevelMessage, 'configOptions' => $producerConfigOptions, @@ -158,7 +157,9 @@ private function haveSomeRandomMessagesProduced(): void private function produceRecordMessage(string $record): string { - $producerConfigOptions = $this->createProducerConfigOptions('low_level'); + $producerConfigOptions = $this->createProducerConfigOptions( + 'low_level' + ); $producer = app(MessageProducer::class, [ 'record' => $record, 'configOptions' => $producerConfigOptions, @@ -206,6 +207,7 @@ private function haveFourProducedMessages(): void private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { $broker = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( $topicId, $broker, diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index b0c801ad..54793d98 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -22,11 +22,12 @@ class ProducerWithAvroTest extends LaravelTestCase /** * @var string[] */ - protected $records; + protected array $records; public function setUp(): void { parent::setUp(); + $this->records = ['saleOrderId' => 'SALE_ORDER_ID', 'productId' => 'PRODUCT_ID']; } @@ -110,8 +111,12 @@ protected function haveAHandlerConfigured(): void private function haveSomeRandomMessagesProduced(): void { - $producerConfigOptionsSale = $this->createProducerConfigOptions('sale_order'); - $producerConfigOptionsProduct = $this->createProducerConfigOptions('product'); + $producerConfigOptionsSale = $this->createProducerConfigOptions( + 'sale_order' + ); + $producerConfigOptionsProduct = $this->createProducerConfigOptions( + 'product' + ); $saleOrderProducer = app(MessageProducer::class, [ 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], @@ -166,6 +171,7 @@ private function setLogExpectationsFor(string $message): void private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { $broker = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( $topicId, $broker, diff --git a/tests/Unit/Authentication/FactoryTest.php b/tests/Unit/Authentication/FactoryTest.php index 717abe16..f13b2e46 100644 --- a/tests/Unit/Authentication/FactoryTest.php +++ b/tests/Unit/Authentication/FactoryTest.php @@ -16,7 +16,11 @@ class FactoryTest extends LaravelTestCase public function testItMakesSslAuthenticationClass(): void { // Set - $configOptionsSsl = new Ssl('path/to/ca', 'path/to/certificate', 'path/to/key'); + $configOptionsSsl = new Ssl( + 'path/to/ca', + 'path/to/certificate', + 'path/to/key' + ); $conf = new Conf(); $expected = [ 'security.protocol' => 'ssl', diff --git a/tests/Unit/Authentication/SASLAuthenticationTest.php b/tests/Unit/Authentication/SASLAuthenticationTest.php index f9e9a474..45a4dc3a 100644 --- a/tests/Unit/Authentication/SASLAuthenticationTest.php +++ b/tests/Unit/Authentication/SASLAuthenticationTest.php @@ -12,7 +12,11 @@ class SASLAuthenticationTest extends LaravelTestCase public function testItShouldValidateAuthenticationConfigurations(): void { // Set - $configSaslSsl = new SaslSsl('PLAIN', 'some-username', 'some-password'); + $configSaslSsl = new SaslSsl( + 'PLAIN', + 'some-username', + 'some-password' + ); $conf = new Conf(); $expected = [ diff --git a/tests/Unit/Authentication/SSLAuthenticationTest.php b/tests/Unit/Authentication/SSLAuthenticationTest.php index 35a9168e..9ea6c6b0 100644 --- a/tests/Unit/Authentication/SSLAuthenticationTest.php +++ b/tests/Unit/Authentication/SSLAuthenticationTest.php @@ -13,7 +13,11 @@ public function testItShouldValidateAuthenticationConfigurations(): void { // Set $conf = new Conf(); - $configSsl = new Ssl('path/to/ca', 'path/to/certificate', 'path/to/key'); + $configSsl = new Ssl( + 'path/to/ca', + 'path/to/certificate', + 'path/to/key' + ); $expected = [ 'security.protocol' => 'ssl', 'ssl.ca.location' => 'path/to/ca', diff --git a/tests/Unit/Connectors/Consumer/ConfigTest.php b/tests/Unit/Connectors/Consumer/ConfigTest.php index ffeaf5cd..224f5666 100644 --- a/tests/Unit/Connectors/Consumer/ConfigTest.php +++ b/tests/Unit/Connectors/Consumer/ConfigTest.php @@ -85,7 +85,9 @@ public function testShouldValidateConsumerConfig(): void ]); // Actions - $configManager = $config->makeWithConfigOptions(ConsumerHandlerDummy::class); + $configManager = $config->makeWithConfigOptions( + ConsumerHandlerDummy::class + ); // Assertions $this->assertArraySubset($expected, $configManager->toArray()); diff --git a/tests/Unit/Connectors/Consumer/FactoryTest.php b/tests/Unit/Connectors/Consumer/FactoryTest.php index 2ccb9cd9..fcd1a6d3 100644 --- a/tests/Unit/Connectors/Consumer/FactoryTest.php +++ b/tests/Unit/Connectors/Consumer/FactoryTest.php @@ -11,6 +11,52 @@ class FactoryTest extends LaravelTestCase { + public function testItMakesManagerWithLowLevelConsumer(): void + { + // Set + $this->haveAConsumerWithPartitionConfigured(); + + $config = new Config(); + $configConsumer = $config->make( + ['timeout' => 61], + ['topic' => 'topic_key', 'consumer_group' => 'with-partition'] + ); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); + } + + public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid(): void + { + // Set + $this->haveAConsumerWithoutPartitionConfigured(); + $config = new Config(); + $configConsumer = $config->make( + ['timeout' => 61, 'partition' => -1], + ['topic' => 'topic_key', 'consumer_group' => 'with-partition'] + ); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); + } + + public function testItMakesHighLevelClass(): void + { + // Set + $this->haveAConsumerWithoutPartitionConfigured(); + $config = new Config(); + $configConsumer = $config->make( + ['timeout' => 61], + ['topic' => 'topic_key', 'consumer_group' => 'without-partition'] + ); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); + } + protected function setUp(): void { parent::setUp(); @@ -47,43 +93,6 @@ protected function setUp(): void ]); } - public function testItMakesManagerWithLowLevelConsumer(): void - { - // Set - $this->haveAConsumerWithPartitionConfigured(); - - $config = new Config(); - $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); - $manager = Factory::make($configConsumer); - - // Assertions - $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); - } - - public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid(): void - { - // Set - $this->haveAConsumerWithoutPartitionConfigured(); - $config = new Config(); - $configConsumer = $config->make(['timeout' => 61, 'partition' => -1], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); - $manager = Factory::make($configConsumer); - - // Assertions - $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); - } - - public function testItMakesHighLevelClass(): void - { - // Set - $this->haveAConsumerWithoutPartitionConfigured(); - $config = new Config(); - $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'without-partition']); - $manager = Factory::make($configConsumer); - - // Assertions - $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); - } - private function haveAConsumerWithPartitionConfigured() { config([ diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index f456dfb7..2869318b 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -33,7 +33,7 @@ public function testItShouldMakeSetup(): void $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); - $handler = new class('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface { + $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface { public function success(Message $message): void { } @@ -57,7 +57,10 @@ public function failed(Message $message): void ->andReturn($broker); // Actions - $result = $connector->getProducerTopic($handler, $producerConfigOptions); + $result = $connector->getProducerTopic( + $handler, + $producerConfigOptions + ); // Assertions $this->assertInstanceOf(KafkaProducer::class, $result); @@ -79,7 +82,7 @@ public function testItShouldMakeSetupWithoutHandleResponse(): void $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); - $handler = new class('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface { + $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface { public function success(Message $message): void { } @@ -102,7 +105,10 @@ public function failed(Message $message): void ->andReturn($broker); // Actions - $result = $connector->getProducerTopic($handler, $producerConfigOptions); + $result = $connector->getProducerTopic( + $handler, + $producerConfigOptions + ); // Assertions $this->assertInstanceOf(KafkaProducer::class, $result); diff --git a/tests/Unit/Console/ConsumerCommandTest.php b/tests/Unit/Console/ConsumerCommandTest.php index 5a5a8fcf..c4737533 100644 --- a/tests/Unit/Console/ConsumerCommandTest.php +++ b/tests/Unit/Console/ConsumerCommandTest.php @@ -10,33 +10,6 @@ class ConsumerCommandTest extends LaravelTestCase { - protected function setUp(): void - { - parent::setUp(); - - config([ - 'kafka' => [ - 'topics' => [ - 'topic_key' => [ - 'topic_id' => 'topic_name', - 'consumer' => [ - 'consumer_group' => 'default', - 'offset_reset' => 'earliest', - 'handler' => ConsumerHandlerDummy::class, - 'timeout' => 123, - ], - ], - ], - ], - 'service' => [ - 'broker' => [ - 'connections' => 'test_kafka:6680', - 'auth' => [], - ], - ], - ]); - } - public function testItCallsCommandWithInvalidTopic(): void { // Set @@ -158,4 +131,31 @@ public function testItOverridesBrokerConnectionWhenCallingCommand(): void $this->artisan($command, $parameters); } + + protected function setUp(): void + { + parent::setUp(); + + config([ + 'kafka' => [ + 'topics' => [ + 'topic_key' => [ + 'topic_id' => 'topic_name', + 'consumer' => [ + 'consumer_group' => 'default', + 'offset_reset' => 'earliest', + 'handler' => ConsumerHandlerDummy::class, + 'timeout' => 123, + ], + ], + ], + ], + 'service' => [ + 'broker' => [ + 'connections' => 'test_kafka:6680', + 'auth' => [], + ], + ], + ]); + } } diff --git a/tests/Unit/Consumers/LowLevelTest.php b/tests/Unit/Consumers/LowLevelTest.php index b39ac2aa..e2347b60 100644 --- a/tests/Unit/Consumers/LowLevelTest.php +++ b/tests/Unit/Consumers/LowLevelTest.php @@ -36,7 +36,10 @@ public function testItShouldConsume(): void $consumerTopic = m::mock(ConsumerTopic::class); $message = new Message(); - $lowLevelConsumer = new LowLevel($consumerTopic, $consumerConfigOptions); + $lowLevelConsumer = new LowLevel( + $consumerTopic, + $consumerConfigOptions + ); // Expectations $consumerTopic->expects() diff --git a/tests/Unit/Middlewares/AvroSchemaDecoderTest.php b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php index 6c8d31b0..ba9edd29 100644 --- a/tests/Unit/Middlewares/AvroSchemaDecoderTest.php +++ b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php @@ -1,4 +1,5 @@ getAvroSchema() ->andReturn($avroSchema); - $avroSchemaDecoder = new AvroSchemaDecoder($clientFactory, $consumerConfigOptions); + $avroSchemaDecoder = new AvroSchemaDecoder( + $clientFactory, + $consumerConfigOptions + ); $result = $avroSchemaDecoder->process($consumerRecord, $closure); diff --git a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php index fcff2d6c..7fc33c0f 100644 --- a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php +++ b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php @@ -1,4 +1,5 @@ getAvroSchema(); $clientFactory = m::mock(ClientFactory::class); - $cachedSchemaRegistryClient = m::mock(CachedSchemaRegistryClient::class); - $schemaIdEncoder = m::mock(SchemaId::class, [$cachedSchemaRegistryClient]); + $cachedSchemaRegistryClient = m::mock( + CachedSchemaRegistryClient::class + ); + $schemaIdEncoder = m::mock( + SchemaId::class, + [$cachedSchemaRegistryClient] + ); $schema = new Schema(); - $parsedSchema = $schema->parse($avroSchema, '123', 'kafka-test-value', 'latest'); + $parsedSchema = $schema->parse( + $avroSchema, + '123', + 'kafka-test-value', + 'latest' + ); $record = $this->getRecord($parsedSchema->getAvroSchema()); $producerRecord = new ProducerRecord($record, 'kafka-test'); @@ -62,7 +76,11 @@ public function testShouldEncodeRecord() ->andReturn($encodedMessage); // Actions - $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder($schemaIdEncoder, $clientFactory, $producerConfigOptions); + $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder( + $schemaIdEncoder, + $clientFactory, + $producerConfigOptions + ); $result = $avroSchemaMixedEncoder->process($producerRecord, $closure); // Assertions @@ -93,6 +111,8 @@ private function getRecord(AvroSchema $avroSchema): string private function getSchemaFixture(): string { - return file_get_contents(__DIR__.'/../fixtures/schemas/sales_price.avsc'); + return file_get_contents( + __DIR__ . '/../fixtures/schemas/sales_price.avsc' + ); } } diff --git a/tests/Unit/ProducerTest.php b/tests/Unit/ProducerTest.php index 02808390..61f43155 100644 --- a/tests/Unit/ProducerTest.php +++ b/tests/Unit/ProducerTest.php @@ -41,7 +41,7 @@ public function testItShouldProduceRecordAsArrayThroughMiddlewareQueue(): void $topic, $broker ); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations @@ -85,7 +85,7 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations @@ -134,7 +134,7 @@ public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): vo 500, 1 ); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations @@ -187,7 +187,7 @@ public function testShouldBuildDispatcher(): void 1 ); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations @@ -236,7 +236,7 @@ public function testShouldBuildDispatcherWithConfigOptions(): void 500, 1 ); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations From f6b6af4781d77c570a13387247bea960307b2632 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 14:21:32 -0300 Subject: [PATCH 059/152] style: supress phpcs typehing warning Signed-off-by: hcdias --- src/Console/ConfigOptionsCommand.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index d592a8b1..52d62dde 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -11,16 +11,19 @@ class ConfigOptionsCommand extends BaseCommand { /** * @var {inheritdoc} + * @phpcsSuppress SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingAnyTypeHint */ protected $name = 'kafka:consume-config-class'; /** * @var {inheritdoc} + * @phpcsSuppress SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingAnyTypeHint */ protected $description = 'Consumes something with a based class config'; /** * @var {inheritdoc} + * @phpcsSuppress SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingAnyTypeHint */ protected $signature = 'kafka:consume-config-class {handler : handler.} From 5afadef6f5459a4255dcba00d45f1edfd527e34a Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 14:39:10 -0300 Subject: [PATCH 060/152] style: supress phpcs typehing warning Signed-off-by: hcdias --- src/Connectors/AbstractConfig.php | 7 ++++++- src/Connectors/Consumer/Config.php | 2 +- tests/Unit/Connectors/Producer/ConnectorTest.php | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index b71c683c..5b084abc 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -7,6 +7,11 @@ abstract class AbstractConfig { + /** + * @var string[] + */ + protected array $rules; + protected function getBrokerConfig(string $servicesFile): array { if (!$brokerConfig = config($servicesFile . '.broker')) { @@ -15,7 +20,7 @@ protected function getBrokerConfig(string $servicesFile): array ); } - return $brokerConfig; + return $brokerConfig->all(); } protected function getSchemaConfig(string $servicesFile): array diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index d13976be..1e870981 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -19,7 +19,7 @@ class Config extends AbstractConfig { /** - * @var array + * @var string[] */ protected array $rules = [ 'topic' => 'required', diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index 2869318b..650dbd02 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -34,6 +34,7 @@ public function testItShouldMakeSetup(): void $connector = new Connector(); $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface { + /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */ public function success(Message $message): void { } @@ -83,6 +84,7 @@ public function testItShouldMakeSetupWithoutHandleResponse(): void $connector = new Connector(); $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface { + /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */ public function success(Message $message): void { } From f9a312a88e8d39ffb47991f95516c69a956d47cb Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 15:23:31 -0300 Subject: [PATCH 061/152] chore: remove call to inexistent method --- src/Connectors/AbstractConfig.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index 5b084abc..3ed44502 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -20,7 +20,7 @@ protected function getBrokerConfig(string $servicesFile): array ); } - return $brokerConfig->all(); + return $brokerConfig; } protected function getSchemaConfig(string $servicesFile): array From c7ffc005392370f5d9e55633d6fb36f20c8abac7 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 15:30:05 -0300 Subject: [PATCH 062/152] chore: supress psalm alert --- src/Connectors/AbstractConfig.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index 3ed44502..ef26cbf5 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -12,6 +12,9 @@ abstract class AbstractConfig */ protected array $rules; + /** + * @psalm-suppress InvalidReturnStatement + */ protected function getBrokerConfig(string $servicesFile): array { if (!$brokerConfig = config($servicesFile . '.broker')) { From 6d2fedb6fc3ada641cd6610c348cf5e9d81edc5f Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 15:55:36 -0300 Subject: [PATCH 063/152] tests: remove configManager related tests --- tests/Unit/Connectors/Consumer/ManagerTest.php | 18 ------------------ .../Unit/Middlewares/Handler/ProducerTest.php | 16 ---------------- tests/Unit/Producer/PollTest.php | 4 ++-- 3 files changed, 2 insertions(+), 36 deletions(-) diff --git a/tests/Unit/Connectors/Consumer/ManagerTest.php b/tests/Unit/Connectors/Consumer/ManagerTest.php index 230513b2..52e3bcc1 100644 --- a/tests/Unit/Connectors/Consumer/ManagerTest.php +++ b/tests/Unit/Connectors/Consumer/ManagerTest.php @@ -184,22 +184,4 @@ function () use ($messages, &$count, $exception) { $runner->handleMessage(); $runner->handleMessage(); } - - protected function setUp(): void - { - parent::setUp(); - - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'connections' => 'kafka:2019', - 'topic' => 'topic_key', - 'broker' => 'default', - 'offset_reset' => 'earliest', - 'offset' => 0, - 'timeout' => 30, - 'handler' => ConsumerHandlerDummy::class, - 'middlewares' => [], - 'consumer_group' => 'consumer-id', - ]); - } } diff --git a/tests/Unit/Middlewares/Handler/ProducerTest.php b/tests/Unit/Middlewares/Handler/ProducerTest.php index df2750e6..561a7c78 100644 --- a/tests/Unit/Middlewares/Handler/ProducerTest.php +++ b/tests/Unit/Middlewares/Handler/ProducerTest.php @@ -37,20 +37,4 @@ public function testItShouldSendMessageToKafkaBroker(): void $producerHandler = new Producer($producerTopic, $poll, 1); $producerHandler->process($record, $closure); } - - protected function setUp(): void - { - parent::setUp(); - - $configManager = new ProducerConfigManager(); - $configManager->set([ - 'topic_id' => 'topic_name', - 'timeout' => 4000, - 'is_async' => true, - 'max_poll_records' => 500, - 'flush_attempts' => 10, - 'required_acknowledgment' => true, - 'partition' => 0, - ]); - } } diff --git a/tests/Unit/Producer/PollTest.php b/tests/Unit/Producer/PollTest.php index 0576cd6c..ba216928 100644 --- a/tests/Unit/Producer/PollTest.php +++ b/tests/Unit/Producer/PollTest.php @@ -68,8 +68,8 @@ public function testShouldThrowExceptionWhenFlushFailed(): void ->poll(0); $kafkaProducer->expects() - ->flush(1000) - ->times(3) + ->flush(100) + ->times(10) ->andReturn(1); $this->expectException(RuntimeException::class); From 42ad7dceb7fd652191d9df3f62be4a874160daa3 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 16:14:30 -0300 Subject: [PATCH 064/152] gha: update kafka broker connection --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 59efe8d6..aa9333b8 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -18,7 +18,7 @@ jobs: - '8.0' - '8.1' env: - KAFKA_BROKER_CONNECTIONS: 'localhost:9092' + KAFKA_BROKER_CONNECTIONS: 'kafka:9092' services: zookeeper: image: bitnami/zookeeper From 96e072c534032d48855d5c57746347575ebdf687 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 18:06:43 -0300 Subject: [PATCH 065/152] gha: undo update kafka broker connection --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index aa9333b8..59efe8d6 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -18,7 +18,7 @@ jobs: - '8.0' - '8.1' env: - KAFKA_BROKER_CONNECTIONS: 'kafka:9092' + KAFKA_BROKER_CONNECTIONS: 'localhost:9092' services: zookeeper: image: bitnami/zookeeper From 33556c5c127de5b5b1f8ab2be46449ef3e518c31 Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 10 Oct 2023 11:02:30 -0300 Subject: [PATCH 066/152] gha: remove kafka broker variable --- .github/workflows/continuous-integration.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 59efe8d6..7a8052b7 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -17,8 +17,6 @@ jobs: - '7.4' - '8.0' - '8.1' - env: - KAFKA_BROKER_CONNECTIONS: 'localhost:9092' services: zookeeper: image: bitnami/zookeeper From a827976de1814fd5d2397584127c1575093bffed Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 10 Oct 2023 11:30:48 -0300 Subject: [PATCH 067/152] gha: get back kafka broker variable --- .github/workflows/continuous-integration.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 7a8052b7..59efe8d6 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -17,6 +17,8 @@ jobs: - '7.4' - '8.0' - '8.1' + env: + KAFKA_BROKER_CONNECTIONS: 'localhost:9092' services: zookeeper: image: bitnami/zookeeper From be49acff0d71ca32669d8cc2e88adf9a4183c6dc Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 11:49:20 -0300 Subject: [PATCH 068/152] fix(tests): get connection by env --- tests/Integration/ProducerTest.php | 3 ++- tests/Integration/ProducerWithAvroTest.php | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 8fbe5305..39cedbfd 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -206,7 +206,8 @@ private function haveFourProducedMessages(): void private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { - $broker = new Broker('kafka:9092', new None()); + $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); + $broker = new Broker($connections, new None()); return new ProducerConfigOptions( $topicId, diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index 54793d98..2a83cca3 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -170,7 +170,8 @@ private function setLogExpectationsFor(string $message): void private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { - $broker = new Broker('kafka:9092', new None()); + $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); + $broker = new Broker($connections, new None()); return new ProducerConfigOptions( $topicId, From 0e528157f11cf1d1b9c345d3ea079ab44a4d6f22 Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 11:56:13 -0300 Subject: [PATCH 069/152] test(producer): add expect not to perform assertion --- tests/Integration/ProducerWithAvroTest.php | 1 + tests/Integration/ProducerWithConfigOptionsTest.php | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index 2a83cca3..09237bfc 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -41,6 +41,7 @@ public function testShouldRunAProducerMessagesWithAAvroSchema(): void // I expect that $this->myMessagesHaveBeenLogged(); + $this->expectNotToPerformAssertions(); } protected function haveAHandlerConfigured(): void diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 4e256a42..ec37a620 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -31,6 +31,7 @@ public function testShouldRunAProducerMessagesWithConfigOptions(): void // I Expect That $this->myMessagesHaveBeenLogged(); + $this->expectNotToPerformAssertions(); // When I $this->runTheConsumer(); From f44e2f93f50e0f19283de8b6a5aa3767061873c2 Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 12:02:25 -0300 Subject: [PATCH 070/152] fix(tests): enable to get broker connection from env --- config/service.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/service.php b/config/service.php index 64f81077..7eacd41f 100644 --- a/config/service.php +++ b/config/service.php @@ -19,7 +19,7 @@ 'password' => 'PASSWORD', ], 'broker' => [ - 'connections' => 'kafka:9092', + 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), 'auth' => [ 'type' => 'ssl', // ssl and none 'ca' => storage_path('ca.pem'), From a3551b50b1ebdb5f7d06107ddf37cf3ad92d1a97 Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 12:11:43 -0300 Subject: [PATCH 071/152] fix(tests): enable to get broker connection from env --- tests/Unit/Connectors/Producer/ConnectorTest.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index 650dbd02..dc2dfe96 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -29,7 +29,8 @@ public function testItShouldMakeSetup(): void m::mock(KafkaProducer::class) ); - $broker = new Broker('kafka:9092', new None()); + $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); + $broker = new Broker($connections, new None()); $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); @@ -79,7 +80,8 @@ public function testItShouldMakeSetupWithoutHandleResponse(): void m::mock(KafkaProducer::class) ); - $broker = new Broker('kafka:9092', new None()); + $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); + $broker = new Broker($connections, new None()); $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); From 40173c0b288ff195ed483f4c06a83f418b836012 Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 12:17:05 -0300 Subject: [PATCH 072/152] fix(tests): adjust mock expectations --- tests/Unit/Connectors/Producer/ConnectorTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index dc2dfe96..6652a0e7 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -52,7 +52,7 @@ public function failed(Message $message): void ->withAnyArgs(); $conf->expects() - ->set('metadata.broker.list', 0); + ->set('metadata.broker.list', $connections); $producerConfigOptions->expects() ->getBroker() @@ -102,7 +102,7 @@ public function failed(Message $message): void ->never(); $conf->expects() - ->set('metadata.broker.list', 0); + ->set('metadata.broker.list', $connections); $producerConfigOptions->expects() ->getBroker() From 7e50037a50206a220e7c89310ebeeb8a610ec924 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Mon, 31 Jan 2022 17:13:51 -0300 Subject: [PATCH 073/152] chore(configOptions): add ConfigOptions command to ServiceProvider --- src/MetamorphosisServiceProvider.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/MetamorphosisServiceProvider.php b/src/MetamorphosisServiceProvider.php index 687c739a..4938fbac 100644 --- a/src/MetamorphosisServiceProvider.php +++ b/src/MetamorphosisServiceProvider.php @@ -3,6 +3,7 @@ namespace Metamorphosis; use Illuminate\Support\ServiceProvider; +use Metamorphosis\Console\ConfigOptionsCommand; use Metamorphosis\Console\ConsumerCommand; use Metamorphosis\Console\ConsumerMakeCommand; use Metamorphosis\Console\MiddlewareMakeCommand; @@ -26,6 +27,7 @@ public function register() ConsumerMakeCommand::class, MiddlewareMakeCommand::class, ProducerMakeCommand::class, + ConfigOptionsCommand::class, ]); $this->app->bind('metamorphosis', function ($app) { From ab06105c475c185a3eded288df63193a91e15ac0 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Mon, 31 Jan 2022 17:16:15 -0300 Subject: [PATCH 074/152] chore(configOptions): call new configOptions artisan command --- .../ProducerWithConfigOptionsTest.php | 41 +++---------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 3382b590..5f5017a0 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -24,14 +24,13 @@ public function testShouldRunAProducerMessagesWithConfigOptions(): void $this->haveAHandlerConfigured(); // I Expect That - $this->myMessagesHaveBeenProduced(); - $this->expectNotToPerformAssertions(); + //$this->myMessagesHaveBeenProduced(); // When I - $this->haveSomeRandomMessageProduced(); + //$this->haveSomeRandomMessageProduced(); // I Expect That - $this->myMessagesHaveBeenLogged(); + //$this->myMessagesHaveBeenLogged(); // When I $this->runTheConsumer(); @@ -41,39 +40,11 @@ protected function runTheConsumer(): void { $dummy = new MessageConsumer($this->consumerConfigOptions); $this->instance('\App\Kafka\Consumers\ConsumerOverride', $dummy); - config([ - 'kafka_new_config' => [ - 'brokers' => [ - 'override' => [ - 'connections' => env( - 'KAFKA_BROKER_CONNECTIONS', - 'kafka:9092' - ), - ], - ], - 'topics' => [ - 'default' => [ - 'broker' => 'override', - 'consumer' => [ - 'consumer_groups' => [ - 'test-consumer-group' => [ - 'handler' => '\App\Kafka\Consumers\ConsumerOverride', - 'offset_reset' => 'earliest', - ], - ], - ], - ], - ], - ], - ]); + $this->artisan( - 'kafka:consume', + 'kafka:consume-config-class', [ - 'topic' => 'default', - 'consumer_group' => 'test-consumer-group', - '--timeout' => 20000, - '--times' => 2, - '--config_name' => 'kafka_new_config', + 'handler' => '\\App\\Kafka\\Consumers\\ConsumerOverride', ] ); } From 571eeb0d5fa192769878fcc4bedeae0e1cf27d46 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Mon, 31 Jan 2022 17:19:05 -0300 Subject: [PATCH 075/152] chore(configOptions): add ConfigOptionsCommand --- src/Console/ConfigOptionsCommand.php | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 src/Console/ConfigOptionsCommand.php diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php new file mode 100644 index 00000000..ccd8bb37 --- /dev/null +++ b/src/Console/ConfigOptionsCommand.php @@ -0,0 +1,49 @@ +make($this->option(), $this->argument()); + +// $this->writeStartingConsumer($configManager); + + $config = $this->argument()['handler']; + + $manager = Factory::make($this->argument()['handler']); + + $runner = app(Runner::class, compact('manager')); + //$runner->run($manager->get('times')); + } + + private function writeStartingConsumer(AbstractConfigManager $configManager) + { + $text = 'Starting consumer for topic: '.$configManager->get('topic').PHP_EOL; + $text .= ' on consumer group: '.$configManager->get('consumer_group').PHP_EOL; + $text .= 'Connecting in '.$configManager->get('connections').PHP_EOL; + $text .= 'Running consumer..'; + + $this->output->writeln($text); + } +} From 76f4ae3284009be7726250ea6d75175e60c7004e Mon Sep 17 00:00:00 2001 From: David Franca Date: Mon, 31 Jan 2022 17:55:13 -0300 Subject: [PATCH 076/152] chore: make config manager with config options --- src/Connectors/Consumer/Config.php | 8 ++++++++ src/Console/ConfigOptionsCommand.php | 13 ++++++------- tests/Integration/ProducerWithConfigOptionsTest.php | 6 +++--- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 306600aa..57b8b92d 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -39,6 +39,14 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; + public function makeWithConfigOptions(string $handlerClass): AbstractConfigManager + { + $configManager = app(ConsumerConfigManager::class); + $configManager->set(['handler' => $handlerClass]); + + return $configManager; + } + public function make(array $options, array $arguments): AbstractConfigManager { $configName = $options['config_name'] ?? 'kafka'; diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index ccd8bb37..37dab14c 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -3,6 +3,7 @@ use Illuminate\Console\Command as BaseCommand; use Metamorphosis\AbstractConfigManager; +use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; @@ -23,18 +24,16 @@ class ConfigOptionsCommand extends BaseCommand */ protected $signature = 'kafka:consume-config-class {handler : handler.}'; - public function handle() + public function handle(Config $config) { -// $configManager = $config->make($this->option(), $this->argument()); + $configManager = $config->makeWithConfigOptions($this->argument()['handler']); -// $this->writeStartingConsumer($configManager); + $this->writeStartingConsumer($configManager); - $config = $this->argument()['handler']; - - $manager = Factory::make($this->argument()['handler']); + $manager = Factory::make($configManager); $runner = app(Runner::class, compact('manager')); - //$runner->run($manager->get('times')); + $runner->run(2); } private function writeStartingConsumer(AbstractConfigManager $configManager) diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 5f5017a0..4a45d8e0 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -24,13 +24,13 @@ public function testShouldRunAProducerMessagesWithConfigOptions(): void $this->haveAHandlerConfigured(); // I Expect That - //$this->myMessagesHaveBeenProduced(); + $this->myMessagesHaveBeenProduced(); // When I - //$this->haveSomeRandomMessageProduced(); + $this->haveSomeRandomMessageProduced(); // I Expect That - //$this->myMessagesHaveBeenLogged(); + $this->myMessagesHaveBeenLogged(); // When I $this->runTheConsumer(); From 64345304b870d74690f242fa797b0647f684095d Mon Sep 17 00:00:00 2001 From: David Franca Date: Mon, 31 Jan 2022 18:44:46 -0300 Subject: [PATCH 077/152] chore: replace config manager to config options --- src/Authentication/Factory.php | 16 +++---- src/Authentication/SASLAuthentication.php | 26 +++------- src/Authentication/SSLAuthentication.php | 26 +++------- src/Connectors/Consumer/Config.php | 4 +- .../Consumer/ConnectorInterface.php | 4 +- src/Connectors/Consumer/Factory.php | 48 ++++++++----------- src/Connectors/Consumer/HighLevel.php | 31 ++++++------ src/Connectors/Consumer/LowLevel.php | 35 +++++--------- src/Connectors/Producer/Connector.php | 9 ++-- src/Console/ConfigOptionsCommand.php | 23 +++++---- src/Consumers/LowLevel.php | 8 ++-- src/Producer.php | 30 ++++-------- src/Producer/Poll.php | 15 +++--- .../ProducerWithConfigOptionsTest.php | 1 + 14 files changed, 112 insertions(+), 164 deletions(-) diff --git a/src/Authentication/Factory.php b/src/Authentication/Factory.php index 67e3e1fa..c679209a 100644 --- a/src/Authentication/Factory.php +++ b/src/Authentication/Factory.php @@ -4,6 +4,8 @@ use Metamorphosis\AbstractConfigManager; use Metamorphosis\Exceptions\AuthenticationException; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\AuthInterface; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\Conf; class Factory @@ -14,9 +16,9 @@ class Factory private const TYPE_NONE = 'none'; - public static function authenticate(Conf $conf, AbstractConfigManager $configManager): void + public static function authenticate(Conf $conf, AuthInterface $configOptions): void { - $type = $configManager->get('auth.type'); + $type = $configOptions->getType(); switch ($type) { case null: case self::TYPE_NONE: @@ -24,17 +26,11 @@ public static function authenticate(Conf $conf, AbstractConfigManager $configMan break; case self::TYPE_SSL: - app( - SSLAuthentication::class, - compact('conf', 'configManager') - ); + app(SSLAuthentication::class, compact('conf', 'configOptions')); break; case self::TYPE_SASL_SSL: - app( - SASLAuthentication::class, - compact('conf', 'configManager') - ); + app(SASLAuthentication::class, compact('conf', 'configOptions')); break; default: diff --git a/src/Authentication/SASLAuthentication.php b/src/Authentication/SASLAuthentication.php index c0ebb42e..9b2fb0d0 100644 --- a/src/Authentication/SASLAuthentication.php +++ b/src/Authentication/SASLAuthentication.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Authentication; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\SaslSsl; use RdKafka\Conf; class SASLAuthentication implements AuthenticationInterface @@ -11,35 +11,23 @@ class SASLAuthentication implements AuthenticationInterface private AbstractConfigManager $configManager; - public function __construct(Conf $conf, AbstractConfigManager $configManager) + public function __construct(Conf $conf, SaslSsl $config) { $this->conf = $conf; - $this->configManager = $configManager; + $this->config = $config; $this->authenticate(); } private function authenticate(): void { - $this->conf->set( - 'security.protocol', - $this->configManager->get('auth.type') - ); + $this->conf->set('security.protocol', $this->config->getType()); // The mechanisms key is optional when configuring this kind of authentication // If the user does not specify the mechanism, the default will be 'PLAIN'. // But, to make config more clear, we are asking the user every time. - $this->conf->set( - 'sasl.mechanisms', - $this->configManager->get('auth.mechanisms') - ); - $this->conf->set( - 'sasl.username', - $this->configManager->get('auth.username') - ); - $this->conf->set( - 'sasl.password', - $this->configManager->get('auth.password') - ); + $this->conf->set('sasl.mechanisms', $this->config->getMechanisms()); + $this->conf->set('sasl.username', $this->config->getUsername()); + $this->conf->set('sasl.password', $this->config->getPassword()); } } diff --git a/src/Authentication/SSLAuthentication.php b/src/Authentication/SSLAuthentication.php index 246ab5e7..750552e8 100644 --- a/src/Authentication/SSLAuthentication.php +++ b/src/Authentication/SSLAuthentication.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Authentication; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\Ssl; use RdKafka\Conf; class SSLAuthentication implements AuthenticationInterface @@ -11,31 +11,19 @@ class SSLAuthentication implements AuthenticationInterface private AbstractConfigManager $configManager; - public function __construct(Conf $conf, AbstractConfigManager $configManager) + public function __construct(Conf $conf, Ssl $configSsl) { $this->conf = $conf; - $this->configManager = $configManager; + $this->configSsl = $configSsl; $this->authenticate(); } private function authenticate(): void { - $this->conf->set( - 'security.protocol', - $this->configManager->get('auth.type') - ); - $this->conf->set( - 'ssl.ca.location', - $this->configManager->get('auth.ca') - ); - $this->conf->set( - 'ssl.certificate.location', - $this->configManager->get('auth.certificate') - ); - $this->conf->set( - 'ssl.key.location', - $this->configManager->get('auth.key') - ); + $this->conf->set('security.protocol', $this->configSsl->getType()); + $this->conf->set('ssl.ca.location', $this->configSsl->getCa()); + $this->conf->set('ssl.certificate.location', $this->configSsl->getCertificate()); + $this->conf->set('ssl.key.location', $this->configSsl->getKey()); } } diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 57b8b92d..9216cf18 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -39,10 +39,10 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; - public function makeWithConfigOptions(string $handlerClass): AbstractConfigManager + public function makeWithConfigOptions(string $handlerClass, ?int $times = null): AbstractConfigManager { $configManager = app(ConsumerConfigManager::class); - $configManager->set(['handler' => $handlerClass]); + $configManager->set(['handler' => $handlerClass], ['times' => $times]); return $configManager; } diff --git a/src/Connectors/Consumer/ConnectorInterface.php b/src/Connectors/Consumer/ConnectorInterface.php index 3163fca6..738a6d0e 100644 --- a/src/Connectors/Consumer/ConnectorInterface.php +++ b/src/Connectors/Consumer/ConnectorInterface.php @@ -2,10 +2,10 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use Metamorphosis\Consumers\ConsumerInterface; interface ConnectorInterface { - public function getConsumer(bool $autoCommit, AbstractConfigManager $configManager): ConsumerInterface; + public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface; } diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index bc7258e9..f48fe56d 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -5,6 +5,7 @@ use Metamorphosis\AbstractConfigManager; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Middlewares\Handler\Dispatcher; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; /** * This factory will determine what kind of connector will be used. @@ -13,44 +14,33 @@ */ class Factory { - public static function make(AbstractConfigManager $configManager): Manager + public static function make(ConfigOptions $configOptions): Manager { - $autoCommit = $configManager->get('auto_commit', true); - $commitAsync = $configManager->get('commit_async', true); - - $consumer = self::getConsumer($autoCommit, $configManager); - $handler = app($configManager->get('handler')); - - $dispatcher = self::getMiddlewareDispatcher( - $configManager->middlewares() - ); - - return new Manager( - $consumer, - $handler, - $dispatcher, - $autoCommit, - $commitAsync - ); + $autoCommit = $configOptions->isAutoCommit(); + $commitAsync = $configOptions->isCommitASync(); + + $consumer = self::getConsumer($autoCommit, $configOptions); + $handler = app($configOptions->getHandler()); + + $dispatcher = self::getMiddlewareDispatcher($configOptions->getMiddlewares()); + + return new Manager($consumer, $handler, $dispatcher, $autoCommit, $commitAsync); } - public static function getConsumer(bool $autoCommit, AbstractConfigManager $configManager): ConsumerInterface + protected static function requiresPartition(ConfigOptions $configOptions): bool { - if (self::requiresPartition($configManager)) { - return app(LowLevel::class)->getConsumer( - $autoCommit, - $configManager - ); - } + $partition = $configOptions->getPartition(); - return app(HighLevel::class)->getConsumer($autoCommit, $configManager); + return !is_null($partition) && $partition >= 0; } - protected static function requiresPartition(AbstractConfigManager $configManager): bool + public static function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface { - $partition = $configManager->get('partition'); + if (self::requiresPartition($configOptions)) { + return app(LowLevel::class)->getConsumer($autoCommit, $configOptions); + } - return !is_null($partition) && $partition >= 0; + return app(HighLevel::class)->getConsumer($autoCommit, $configOptions); } private static function getMiddlewareDispatcher(array $middlewares): Dispatcher diff --git a/src/Connectors/Consumer/HighLevel.php b/src/Connectors/Consumer/HighLevel.php index 27d5f3a3..9c832ae0 100644 --- a/src/Connectors/Consumer/HighLevel.php +++ b/src/Connectors/Consumer/HighLevel.php @@ -2,44 +2,43 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Authentication\Factory; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Consumers\HighLevel as HighLevelConsumer; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\Conf; use RdKafka\KafkaConsumer; class HighLevel implements ConnectorInterface { - public function getConsumer(bool $autoCommit, AbstractConfigManager $configManager): ConsumerInterface + public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface { - $conf = $this->getConf($configManager); - $maxPollIntervalMs = (int) $configManager->get('max_poll_interval_ms'); - - $conf->set('group.id', $configManager->get('consumer_group')); - $conf->set('auto.offset.reset', $configManager->get('offset_reset')); + $conf = $this->getConf($configOptions); + $maxPollIntervalMs = (int) $configOptions->getTimeout(); + $conf->set('group.id', $configOptions->getConsumerGroup()); + $conf->set('auto.offset.reset', $configOptions->getOffsetReset()); + if (!$autoCommit) { + $conf->set('enable.auto.commit', 'false'); + } $conf->set( 'max.poll.interval.ms', $maxPollIntervalMs ?: 300000 ); - if (!$autoCommit) { - $conf->set('enable.auto.commit', 'false'); - } - $consumer = app(KafkaConsumer::class, ['conf' => $conf]); - $consumer->subscribe([$configManager->get('topic_id')]); - $timeout = $configManager->get('timeout'); + $consumer->subscribe([$configOptions->getTopicId()]); + $timeout = $configOptions->getTimeout(); return app(HighLevelConsumer::class, compact('consumer', 'timeout')); } - protected function getConf(AbstractConfigManager $configManager): Conf + protected function getConf(ConfigOptions $configOptions): Conf { $conf = resolve(Conf::class); - Factory::authenticate($conf, $configManager); + $broker = $configOptions->getBroker(); + Factory::authenticate($conf, $broker->getAuth()); - $conf->set('metadata.broker.list', $configManager->get('connections')); + $conf->set('metadata.broker.list', $broker->getConnections()); return $conf; } diff --git a/src/Connectors/Consumer/LowLevel.php b/src/Connectors/Consumer/LowLevel.php index c270fc74..09033a71 100644 --- a/src/Connectors/Consumer/LowLevel.php +++ b/src/Connectors/Consumer/LowLevel.php @@ -6,57 +6,48 @@ use Metamorphosis\Authentication\Factory; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Consumers\LowLevel as LowLevelConsumer; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\Conf; use RdKafka\Consumer; use RdKafka\TopicConf; class LowLevel implements ConnectorInterface { - public function getConsumer(bool $autoCommit, AbstractConfigManager $configManager): ConsumerInterface + public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface { $conf = $this->getConf(); - $maxPollIntervalMs = (int) $configManager->get('max_poll_interval_ms'); - - $conf->set('group.id', $configManager->get('consumer_group')); + $maxPollIntervalMs = (int) $configOptions->getTimeout(); $conf->set( 'max.poll.interval.ms', $maxPollIntervalMs ?: 300000 ); - + $conf->set('group.id', $configOptions->getConsumerGroup()); if (!$autoCommit) { $conf->set('enable.auto.commit', 'false'); } - Factory::authenticate($conf, $configManager); + $broker = $configOptions->getBroker(); + Factory::authenticate($conf, $broker->getAuth()); $consumer = new Consumer($conf); - $consumer->addBrokers($configManager->get('connections')); + $consumer->addBrokers($broker->getConnections()); - $topicConf = $this->getTopicConfigs($configManager); - $topicConsumer = $consumer->newTopic( - $configManager->get('topic_id'), - $topicConf - ); + $topicConf = $this->getTopicConfigs($configOptions); + $topicConsumer = $consumer->newTopic($configOptions->getTopicId(), $topicConf); - $topicConsumer->consumeStart( - $configManager->get('partition'), - $configManager->get('offset') - ); + $topicConsumer->consumeStart($configOptions->getPartition(), $configOptions->getOffset()); - return new LowLevelConsumer($topicConsumer, $configManager); + return new LowLevelConsumer($topicConsumer, $configOptions); } - protected function getTopicConfigs(AbstractConfigManager $configManager) + protected function getTopicConfigs(ConfigOptions $configOptions) { $topicConfig = new TopicConf(); // Set where to start consuming messages when there is no initial offset in // offset store or the desired offset is out of range. // 'smallest': start from the beginning - $topicConfig->set( - 'auto.offset.reset', - $configManager->get('offset_reset') - ); + $topicConfig->set('auto.offset.reset', $configOptions->getOffsetReset()); return $topicConfig; } diff --git a/src/Connectors/Producer/Connector.php b/src/Connectors/Producer/Connector.php index ebe696ea..68eafb07 100644 --- a/src/Connectors/Producer/Connector.php +++ b/src/Connectors/Producer/Connector.php @@ -2,8 +2,8 @@ namespace Metamorphosis\Connectors\Producer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Authentication\Factory; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; use Metamorphosis\TopicHandler\Producer\HandleableResponseInterface; use Metamorphosis\TopicHandler\Producer\HandlerInterface; use RdKafka\Conf; @@ -12,7 +12,7 @@ class Connector { - public function getProducerTopic(HandlerInterface $handler, AbstractConfigManager $configManager): KafkaProducer + public function getProducerTopic(HandlerInterface $handler, ConfigOptions $configOptions): KafkaProducer { $conf = resolve(Conf::class); @@ -29,9 +29,10 @@ function ($kafka, Message $message) use ($handler) { ); } - $conf->set('metadata.broker.list', $configManager->get('connections')); + $broker = $configOptions->getBroker(); + $conf->set('metadata.broker.list', $broker->getConnections()); - Factory::authenticate($conf, $configManager); + Factory::authenticate($conf, $broker->getAuth()); return app(KafkaProducer::class, compact('conf')); } diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index 37dab14c..4d37240b 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -6,6 +6,7 @@ use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; class ConfigOptionsCommand extends BaseCommand { @@ -22,25 +23,29 @@ class ConfigOptionsCommand extends BaseCommand /** * @var {inheritdoc} */ - protected $signature = 'kafka:consume-config-class {handler : handler.}'; + protected $signature = 'kafka:consume-config-class + {handler : handler.} + {--times= : Amount of messages to be consumed.}'; public function handle(Config $config) { - $configManager = $config->makeWithConfigOptions($this->argument()['handler']); + $consumerHandler = app($this->argument('handler')); - $this->writeStartingConsumer($configManager); + $configOptions = $consumerHandler->getConfigOptions(); - $manager = Factory::make($configManager); + $this->writeStartingConsumer($configOptions); + + $manager = Factory::make($configOptions); $runner = app(Runner::class, compact('manager')); - $runner->run(2); + $runner->run($this->option('times')); } - private function writeStartingConsumer(AbstractConfigManager $configManager) + private function writeStartingConsumer(ConfigOptions $configOptions) { - $text = 'Starting consumer for topic: '.$configManager->get('topic').PHP_EOL; - $text .= ' on consumer group: '.$configManager->get('consumer_group').PHP_EOL; - $text .= 'Connecting in '.$configManager->get('connections').PHP_EOL; + $text = 'Starting consumer for topic: '.$configOptions->getTopicId().PHP_EOL; + $text .= ' on consumer group: '.$configOptions->getConsumerGroup().PHP_EOL; + $text .= 'Connecting in '.$configOptions->getBroker()->getConnections().PHP_EOL; $text .= 'Running consumer..'; $this->output->writeln($text); diff --git a/src/Consumers/LowLevel.php b/src/Consumers/LowLevel.php index 0a1d762b..e4a31ea0 100644 --- a/src/Consumers/LowLevel.php +++ b/src/Consumers/LowLevel.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Consumers; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\ConsumerTopic; use RdKafka\Message; @@ -14,12 +14,12 @@ class LowLevel implements ConsumerInterface private ?int $timeout; - public function __construct(ConsumerTopic $consumer, AbstractConfigManager $configManager) + public function __construct(ConsumerTopic $consumer, ConfigOptions $configOptions) { $this->consumer = $consumer; - $this->partition = $configManager->get('partition'); - $this->timeout = $configManager->get('timeout'); + $this->partition = $configOptions->getPartition(); + $this->timeout = $configOptions->getTimeout(); } public function consume(): ?Message diff --git a/src/Producer.php b/src/Producer.php index 61c155a3..e1187cf0 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -7,6 +7,7 @@ use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\Middlewares\Handler\Producer as ProducerMiddleware; use Metamorphosis\Producer\Poll; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Metamorphosis\TopicHandler\Producer\HandlerInterface; @@ -31,32 +32,21 @@ public function produce(HandlerInterface $producerHandler): void public function build(HandlerInterface $producerHandler): Dispatcher { - $configManager = $this->getConfigManager($producerHandler); + $configOptions = $producerHandler->getConfigOptions(); - $middlewares = $configManager->middlewares(); - $middlewares[] = $this->getProducerMiddleware( - $producerHandler, - $configManager - ); + $middlewares = $configOptions->getMiddlewares(); + $middlewares[] = $this->getProducerMiddleware($producerHandler, $configOptions); return new Dispatcher($middlewares); } - public function getProducerMiddleware( - HandlerInterface $producerHandler, - AbstractConfigManager $configManager - ): ProducerMiddleware { - $producer = $this->connector->getProducerTopic( - $producerHandler, - $configManager - ); + public function getProducerMiddleware(HandlerInterface $producerHandler, ConfigOptions $configOptions): ProducerMiddleware + { + $producer = $this->connector->getProducerTopic($producerHandler, $configOptions); - $topic = $producer->newTopic($configManager->get('topic_id')); - $poll = app( - Poll::class, - ['producer' => $producer, 'configManager' => $configManager] - ); - $partition = $configManager->get('partition'); + $topic = $producer->newTopic($configOptions->getTopicId()); + $poll = app(Poll::class, ['producer' => $producer, 'configOptions' => $configOptions]); + $partition = $configOptions->getPartition(); return app( ProducerMiddleware::class, diff --git a/src/Producer/Poll.php b/src/Producer/Poll.php index afb8d081..d6a64626 100644 --- a/src/Producer/Poll.php +++ b/src/Producer/Poll.php @@ -3,6 +3,7 @@ namespace Metamorphosis\Producer; use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; use RdKafka\Producer; use RuntimeException; @@ -24,15 +25,13 @@ class Poll private Producer $producer; - public function __construct(Producer $producer, AbstractConfigManager $configManager) + public function __construct(Producer $producer, ConfigOptions $configOptions) { - $this->isAsync = $configManager->get('is_async'); - $this->maxPollRecords = $configManager->get('max_poll_records'); - $this->requiredAcknowledgment = $configManager->get( - 'required_acknowledgment' - ); - $this->maxFlushAttempts = $configManager->get('flush_attempts'); - $this->timeout = $configManager->get('timeout'); + $this->isAsync = $configOptions->isAsync(); + $this->maxPollRecords = $configOptions->getMaxPollRecords(); + $this->requiredAcknowledgment = $configOptions->isRequiredAcknowledgment(); + $this->maxFlushAttempts = $configOptions->getFlushAttempts(); + $this->timeout = $configOptions->getTimeout(); $this->producer = $producer; } diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 4a45d8e0..4e256a42 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -45,6 +45,7 @@ protected function runTheConsumer(): void 'kafka:consume-config-class', [ 'handler' => '\\App\\Kafka\\Consumers\\ConsumerOverride', + '--times' => 2, ] ); } From cc761f3137233117a17302347ef4a3e0881ea2e7 Mon Sep 17 00:00:00 2001 From: David Franca Date: Tue, 1 Feb 2022 09:46:24 -0300 Subject: [PATCH 078/152] chore: migrate Avro to use ConfigOptions\AvroSchema --- src/Avro/ClientFactory.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Avro/ClientFactory.php b/src/Avro/ClientFactory.php index 0e537e4c..b30d0c1b 100644 --- a/src/Avro/ClientFactory.php +++ b/src/Avro/ClientFactory.php @@ -3,29 +3,31 @@ namespace Metamorphosis\Avro; use GuzzleHttp\Client as GuzzleClient; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema; class ClientFactory { - public function make(AbstractConfigManager $configManager): CachedSchemaRegistryClient + const REQUEST_TIMEOUT = 2000; + + public function make(AvroSchema $avroSchema): CachedSchemaRegistryClient { - $guzzleHttp = $this->getGuzzleHttpClient($configManager); + $guzzleHttp = $this->getGuzzleHttpClient($avroSchema); $client = app(Client::class, ['client' => $guzzleHttp]); return app(CachedSchemaRegistryClient::class, compact('client')); } - private function getGuzzleHttpClient(AbstractConfigManager $configManager): GuzzleClient + private function getGuzzleHttpClient(AvroSchema $avroSchema): GuzzleClient { - $config = $configManager->get('request_options') ?: []; - $config['timeout'] = $configManager->get('timeout'); - $config['base_uri'] = $configManager->get('url'); + $config = $avroSchema->getRequestOptions(); + $config['timeout'] = self::REQUEST_TIMEOUT; + $config['base_uri'] = $avroSchema->getUrl(); $config['headers'] = array_merge( $this->getDefaultHeaders(), $config['headers'] ?? [] ); - $config['verify'] = $configManager->get('ssl_verify') ?? false; + $config['verify'] = $avroSchema->getRequestOptions()['ssl_verify'] ?? false; return app(GuzzleClient::class, compact('config')); } From de4331cd91881dc6b770c33599f9cdae457d7ff4 Mon Sep 17 00:00:00 2001 From: David Franca Date: Tue, 1 Feb 2022 10:00:06 -0300 Subject: [PATCH 079/152] chore: migrate Aith factory to use config options --- src/Authentication/Factory.php | 2 -- src/Authentication/SASLAuthentication.php | 17 +++++---- src/Authentication/SSLAuthentication.php | 17 +++++---- src/Connectors/Consumer/Config.php | 5 ++- src/Console/ConsumerCommand.php | 1 + src/Middlewares/AvroSchemaDecoder.php | 19 +++++----- tests/Unit/Authentication/FactoryTest.php | 43 ++++++++++------------- 7 files changed, 52 insertions(+), 52 deletions(-) diff --git a/src/Authentication/Factory.php b/src/Authentication/Factory.php index c679209a..c8a8f38a 100644 --- a/src/Authentication/Factory.php +++ b/src/Authentication/Factory.php @@ -2,10 +2,8 @@ namespace Metamorphosis\Authentication; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Exceptions\AuthenticationException; use Metamorphosis\TopicHandler\ConfigOptions\Auth\AuthInterface; -use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\Conf; class Factory diff --git a/src/Authentication/SASLAuthentication.php b/src/Authentication/SASLAuthentication.php index 9b2fb0d0..3918558d 100644 --- a/src/Authentication/SASLAuthentication.php +++ b/src/Authentication/SASLAuthentication.php @@ -9,25 +9,28 @@ class SASLAuthentication implements AuthenticationInterface { private Conf $conf; - private AbstractConfigManager $configManager; + /** + * @var SaslSsl + */ + private $configOptions; - public function __construct(Conf $conf, SaslSsl $config) + public function __construct(Conf $conf, SaslSsl $configOptions) { $this->conf = $conf; - $this->config = $config; + $this->configOptions = $configOptions; $this->authenticate(); } private function authenticate(): void { - $this->conf->set('security.protocol', $this->config->getType()); + $this->conf->set('security.protocol', $this->configOptions->getType()); // The mechanisms key is optional when configuring this kind of authentication // If the user does not specify the mechanism, the default will be 'PLAIN'. // But, to make config more clear, we are asking the user every time. - $this->conf->set('sasl.mechanisms', $this->config->getMechanisms()); - $this->conf->set('sasl.username', $this->config->getUsername()); - $this->conf->set('sasl.password', $this->config->getPassword()); + $this->conf->set('sasl.mechanisms', $this->configOptions->getMechanisms()); + $this->conf->set('sasl.username', $this->configOptions->getUsername()); + $this->conf->set('sasl.password', $this->configOptions->getPassword()); } } diff --git a/src/Authentication/SSLAuthentication.php b/src/Authentication/SSLAuthentication.php index 750552e8..5dfc06e4 100644 --- a/src/Authentication/SSLAuthentication.php +++ b/src/Authentication/SSLAuthentication.php @@ -9,21 +9,24 @@ class SSLAuthentication implements AuthenticationInterface { private Conf $conf; - private AbstractConfigManager $configManager; + /** + * @var Ssl + */ + private $configOptions; - public function __construct(Conf $conf, Ssl $configSsl) + public function __construct(Conf $conf, Ssl $configOptions) { $this->conf = $conf; - $this->configSsl = $configSsl; + $this->configOptions = $configOptions; $this->authenticate(); } private function authenticate(): void { - $this->conf->set('security.protocol', $this->configSsl->getType()); - $this->conf->set('ssl.ca.location', $this->configSsl->getCa()); - $this->conf->set('ssl.certificate.location', $this->configSsl->getCertificate()); - $this->conf->set('ssl.key.location', $this->configSsl->getKey()); + $this->conf->set('security.protocol', $this->configOptions->getType()); + $this->conf->set('ssl.ca.location', $this->configOptions->getCa()); + $this->conf->set('ssl.certificate.location', $this->configOptions->getCertificate()); + $this->conf->set('ssl.key.location', $this->configOptions->getKey()); } } diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 9216cf18..fa6d1398 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -6,6 +6,7 @@ use Metamorphosis\Connectors\AbstractConfig; use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\ConfigurationException; +use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; /** * This class is responsible for handling all configuration made on the @@ -75,10 +76,8 @@ public function make(array $options, array $arguments): AbstractConfigManager ); $this->validate(array_merge($config, $override)); - $configManager = app(ConsumerConfigManager::class); - $configManager->set($config, $override); - return $configManager; + return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); } /** diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index c4bdab76..bd80175e 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -7,6 +7,7 @@ use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; +use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; class ConsumerCommand extends BaseCommand { diff --git a/src/Middlewares/AvroSchemaDecoder.php b/src/Middlewares/AvroSchemaDecoder.php index 4ae626c9..8d185fdd 100644 --- a/src/Middlewares/AvroSchemaDecoder.php +++ b/src/Middlewares/AvroSchemaDecoder.php @@ -3,28 +3,29 @@ namespace Metamorphosis\Middlewares; use Closure; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Avro\ClientFactory; use Metamorphosis\Avro\Serializer\MessageDecoder; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema; class AvroSchemaDecoder implements MiddlewareInterface { private MessageDecoder $decoder; - private AbstractConfigManager $configManager; + /** + * @var AvroSchema + */ + private $avroSchema; - public function __construct(AbstractConfigManager $configManager, ClientFactory $factory) + public function __construct(AvroSchema $avroSchema, ClientFactory $factory) { - $this->configManager = $configManager; - if (!$this->configManager->get('url')) { - throw new ConfigurationException( - "Avro schema url not found, it's required to use AvroSchemaDecoder Middleware" - ); + $this->avroSchema = $avroSchema; + if (!$this->avroSchema->getUrl()) { + throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaDecoder Middleware"); } - $this->decoder = new MessageDecoder($factory->make($configManager)); + $this->decoder = new MessageDecoder($factory->make($avroSchema)); } public function process(RecordInterface $record, Closure $next) diff --git a/tests/Unit/Authentication/FactoryTest.php b/tests/Unit/Authentication/FactoryTest.php index 53999f8a..717abe16 100644 --- a/tests/Unit/Authentication/FactoryTest.php +++ b/tests/Unit/Authentication/FactoryTest.php @@ -3,8 +3,11 @@ namespace Tests\Unit\Authentication; use Metamorphosis\Authentication\Factory; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\AuthenticationException; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\AuthInterface; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\SaslSsl; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\Ssl; +use Mockery as m; use RdKafka\Conf; use Tests\LaravelTestCase; @@ -13,15 +16,7 @@ class FactoryTest extends LaravelTestCase public function testItMakesSslAuthenticationClass(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'auth' => [ - 'type' => 'ssl', - 'ca' => 'path/to/ca', - 'certificate' => 'path/to/certificate', - 'key' => 'path/to/key', - ], - ]); + $configOptionsSsl = new Ssl('path/to/ca', 'path/to/certificate', 'path/to/key'); $conf = new Conf(); $expected = [ 'security.protocol' => 'ssl', @@ -31,7 +26,7 @@ public function testItMakesSslAuthenticationClass(): void ]; // Actions - Factory::authenticate($conf, $configManager); + Factory::authenticate($conf, $configOptionsSsl); // Assertions $this->assertArraySubset($expected, $conf->dump()); @@ -40,15 +35,11 @@ public function testItMakesSslAuthenticationClass(): void public function testItMakesSASLAuthenticationClass(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'auth' => [ - 'type' => 'sasl_ssl', - 'mechanisms' => 'PLAIN', - 'username' => 'some-username', - 'password' => 'some-password', - ], - ]); + $configOptionsSaslSsl = new SaslSsl( + 'PLAIN', + 'some-username', + 'some-password' + ); $conf = new Conf(); $expected = [ 'security.protocol' => 'sasl_ssl', @@ -58,7 +49,7 @@ public function testItMakesSASLAuthenticationClass(): void ]; // Actions - Factory::authenticate($conf, $configManager); + Factory::authenticate($conf, $configOptionsSaslSsl); // Assertions $this->assertArraySubset($expected, $conf->dump()); @@ -67,13 +58,17 @@ public function testItMakesSASLAuthenticationClass(): void public function testItThrowsExceptionWhenInvalidProtocolIsPassed(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set(['auth' => ['type' => 'some-invalid-type']]); + $invalidAuth = m::mock(AuthInterface::class); $conf = new Conf(); + // Expectations + $invalidAuth->expects() + ->getType() + ->andReturn('some-invalid-type'); + $this->expectException(AuthenticationException::class); // Actions - Factory::authenticate($conf, $configManager); + Factory::authenticate($conf, $invalidAuth); } } From 4147c57152dc582d64e26e19a57dbdfa5e6cab74 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Tue, 1 Feb 2022 10:28:02 -0300 Subject: [PATCH 080/152] chore: remove configManager and add Sasl config class --- .../Authentication/SASLAuthenticationTest.php | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/Unit/Authentication/SASLAuthenticationTest.php b/tests/Unit/Authentication/SASLAuthenticationTest.php index ce1078b3..f9e9a474 100644 --- a/tests/Unit/Authentication/SASLAuthenticationTest.php +++ b/tests/Unit/Authentication/SASLAuthenticationTest.php @@ -3,7 +3,7 @@ namespace Tests\Unit\Authentication; use Metamorphosis\Authentication\SASLAuthentication; -use Metamorphosis\ConsumerConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\SaslSsl; use RdKafka\Conf; use Tests\LaravelTestCase; @@ -12,16 +12,9 @@ class SASLAuthenticationTest extends LaravelTestCase public function testItShouldValidateAuthenticationConfigurations(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'auth' => [ - 'type' => 'sasl_ssl', - 'mechanisms' => 'PLAIN', - 'username' => 'some-username', - 'password' => 'some-password', - ], - ]); + $configSaslSsl = new SaslSsl('PLAIN', 'some-username', 'some-password'); $conf = new Conf(); + $expected = [ 'security.protocol' => 'sasl_ssl', 'sasl.username' => 'some-username', @@ -30,7 +23,7 @@ public function testItShouldValidateAuthenticationConfigurations(): void ]; // Actions - new SASLAuthentication($conf, $configManager); + new SASLAuthentication($conf, $configSaslSsl); // Assertions $this->assertArraySubset($expected, $conf->dump()); From 2ff2aa20ad5acd6a028a03164a6d035d6d3eeb77 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Tue, 1 Feb 2022 10:31:05 -0300 Subject: [PATCH 081/152] chore: remove configManager and add SSL config class --- .../Unit/Authentication/SSLAuthenticationTest.php | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/Unit/Authentication/SSLAuthenticationTest.php b/tests/Unit/Authentication/SSLAuthenticationTest.php index 11c93691..35a9168e 100644 --- a/tests/Unit/Authentication/SSLAuthenticationTest.php +++ b/tests/Unit/Authentication/SSLAuthenticationTest.php @@ -3,7 +3,7 @@ namespace Tests\Unit\Authentication; use Metamorphosis\Authentication\SSLAuthentication; -use Metamorphosis\ConsumerConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\Ssl; use RdKafka\Conf; use Tests\LaravelTestCase; @@ -12,16 +12,8 @@ class SSLAuthenticationTest extends LaravelTestCase public function testItShouldValidateAuthenticationConfigurations(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'auth' => [ - 'type' => 'ssl', - 'ca' => 'path/to/ca', - 'certificate' => 'path/to/certificate', - 'key' => 'path/to/key', - ], - ]); $conf = new Conf(); + $configSsl = new Ssl('path/to/ca', 'path/to/certificate', 'path/to/key'); $expected = [ 'security.protocol' => 'ssl', 'ssl.ca.location' => 'path/to/ca', @@ -30,7 +22,7 @@ public function testItShouldValidateAuthenticationConfigurations(): void ]; // Actions - new SSLAuthentication($conf, $configManager); + new SSLAuthentication($conf, $configSsl); // Assertions $this->assertArraySubset($expected, $conf->dump()); From 0628b306c079a7feef679c77bc23d2165e9cec24 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Tue, 1 Feb 2022 12:04:03 -0300 Subject: [PATCH 082/152] chore: fix code standard with phpcbf --- src/Connectors/Consumer/ConnectorInterface.php | 2 +- src/Connectors/Consumer/Factory.php | 1 - src/Connectors/Consumer/LowLevel.php | 1 - src/Console/ConfigOptionsCommand.php | 1 - src/Console/ConsumerCommand.php | 1 - src/Producer/Poll.php | 1 - 6 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Connectors/Consumer/ConnectorInterface.php b/src/Connectors/Consumer/ConnectorInterface.php index 738a6d0e..f9a7934d 100644 --- a/src/Connectors/Consumer/ConnectorInterface.php +++ b/src/Connectors/Consumer/ConnectorInterface.php @@ -2,8 +2,8 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use Metamorphosis\Consumers\ConsumerInterface; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; interface ConnectorInterface { diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index f48fe56d..893e9030 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; diff --git a/src/Connectors/Consumer/LowLevel.php b/src/Connectors/Consumer/LowLevel.php index 09033a71..7c7395ac 100644 --- a/src/Connectors/Consumer/LowLevel.php +++ b/src/Connectors/Consumer/LowLevel.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Authentication\Factory; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Consumers\LowLevel as LowLevelConsumer; diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index 4d37240b..eb4c1071 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Console; use Illuminate\Console\Command as BaseCommand; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index bd80175e..c4bdab76 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -7,7 +7,6 @@ use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; -use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; class ConsumerCommand extends BaseCommand { diff --git a/src/Producer/Poll.php b/src/Producer/Poll.php index d6a64626..4c3b9c81 100644 --- a/src/Producer/Poll.php +++ b/src/Producer/Poll.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Producer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; use RdKafka\Producer; use RuntimeException; From 602bcbd65dd5cc1d82adb2b26e168b7b4fad934b Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 7 Mar 2022 18:11:01 -0300 Subject: [PATCH 083/152] chore: debug config options --- src/Connectors/Consumer/Config.php | 3 +- src/Console/ConsumerCommand.php | 24 ++++------- src/Consumer.php | 4 +- .../Producer/AbstractProducer.php | 15 ++++--- tests/Integration/ConsumerTest.php | 11 ++--- tests/Integration/Dummies/MessageProducer.php | 14 +------ tests/Integration/ProducerTest.php | 41 ++++++++++++------- tests/Unit/ProducerTest.php | 16 ++++---- 8 files changed, 64 insertions(+), 64 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index fa6d1398..d14770fb 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -6,6 +6,7 @@ use Metamorphosis\Connectors\AbstractConfig; use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\ConfigurationException; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer; use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; /** @@ -48,7 +49,7 @@ public function makeWithConfigOptions(string $handlerClass, ?int $times = null): return $configManager; } - public function make(array $options, array $arguments): AbstractConfigManager + public function make(array $options, array $arguments): Consumer { $configName = $options['config_name'] ?? 'kafka'; $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index c4bdab76..5c233db4 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -3,10 +3,10 @@ namespace Metamorphosis\Console; use Illuminate\Console\Command as BaseCommand; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer; class ConsumerCommand extends BaseCommand { @@ -38,27 +38,21 @@ class ConsumerCommand extends BaseCommand public function handle(Config $config) { - $configManager = $config->make($this->option(), $this->argument()); + $consumer = $config->make($this->option(), $this->argument()); - $this->writeStartingConsumer($configManager); + $this->writeStartingConsumer($consumer); - $manager = Factory::make($configManager); + $manager = Factory::make($consumer); $runner = app(Runner::class, compact('manager')); - $runner->run($configManager->get('times')); + $runner->run($this->option('times')); } - private function writeStartingConsumer(AbstractConfigManager $configManager) + private function writeStartingConsumer(Consumer $consumer) { - $text = 'Starting consumer for topic: ' . $configManager->get( - 'topic' - ) . PHP_EOL; - $text .= ' on consumer group: ' . $configManager->get( - 'consumer_group' - ) . PHP_EOL; - $text .= 'Connecting in ' . $configManager->get( - 'connections' - ) . PHP_EOL; + $text = 'Starting consumer for topic: '.$consumer->getTopicId().PHP_EOL; + $text .= ' on consumer group: '.$consumer->getConsumerGroup().PHP_EOL; + $text .= 'Connecting in '.$consumer->getBroker()->getConnections().PHP_EOL; $text .= 'Running consumer..'; $this->output->writeln($text); diff --git a/src/Consumer.php b/src/Consumer.php index ba04c783..1d35a0c1 100644 --- a/src/Consumer.php +++ b/src/Consumer.php @@ -19,8 +19,8 @@ public function __construct(ConsumerConfigManager $configManager, ConsumerConfig { $configManager->set($configOptions->toArray()); - $this->consumer = Factory::getConsumer(true, $configManager); - $this->dispatcher = new Dispatcher($configManager->middlewares()); + $this->consumer = Factory::getConsumer(true, $configOptions); + $this->dispatcher = new Dispatcher($configOptions->getMiddlewares()); } public function consume(): ?RecordInterface diff --git a/src/TopicHandler/Producer/AbstractProducer.php b/src/TopicHandler/Producer/AbstractProducer.php index e95a6f63..946db3ce 100644 --- a/src/TopicHandler/Producer/AbstractProducer.php +++ b/src/TopicHandler/Producer/AbstractProducer.php @@ -4,7 +4,7 @@ use Metamorphosis\Exceptions\JsonException; use Metamorphosis\Record\ProducerRecord; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptionsProducer; +use Metamorphosis\TopicHandler\ConfigOptions\Producer; class AbstractProducer implements HandlerInterface { @@ -15,18 +15,21 @@ class AbstractProducer implements HandlerInterface protected ?string $key; - private ConfigOptionsProducer $configOptions; + /** + * @var Producer + */ + protected $producer; - public function __construct($record, ConfigOptionsProducer $configOptions, ?string $key = null) + public function __construct($record, Producer $producer, string $key = null) { $this->record = $record; $this->key = $key; - $this->configOptions = $configOptions; + $this->producer = $producer; } - public function getConfigOptions(): ConfigOptionsProducer + public function getConfigOptions(): Producer { - return $this->configOptions; + return $this->producer; } public function getRecord() diff --git a/tests/Integration/ConsumerTest.php b/tests/Integration/ConsumerTest.php index 8c107499..1d4397e8 100644 --- a/tests/Integration/ConsumerTest.php +++ b/tests/Integration/ConsumerTest.php @@ -59,16 +59,13 @@ public function testItShouldSetup(): void $saleOrderDispatcher = Metamorphosis::build($messageProducer); $saleOrderDispatcher->handle($messageProducer->createRecord()); - $consumer = $this->app->make( - Consumer::class, - ['configOptions' => $consumerConfigOptions] - ); - $expected = ['id' => 'MESSAGE_ID']; + $consumer = $this->app->make(Consumer::class, ['configOptions' => $consumerConfigOptions]); + $expected = '{"id":"MESSAGE_ID"}'; // Actions - $result = $consumer->consume(); + $result = $consumer->consume()->getPayload(); // Assertions - $this->assertSame($expected, $result->getPayload()); + $this->assertSame($expected, $result); } } diff --git a/tests/Integration/Dummies/MessageProducer.php b/tests/Integration/Dummies/MessageProducer.php index 22aac6f8..38fb8090 100644 --- a/tests/Integration/Dummies/MessageProducer.php +++ b/tests/Integration/Dummies/MessageProducer.php @@ -3,22 +3,12 @@ namespace Tests\Integration\Dummies; use Illuminate\Support\Facades\Log; -use Metamorphosis\TopicHandler\Producer\AbstractHandler; +use Metamorphosis\TopicHandler\Producer\AbstractProducer; use RdKafka\Message; use RuntimeException; -class MessageProducer extends AbstractHandler +class MessageProducer extends AbstractProducer { - public string $topic = 'default'; - - public function __construct($record, string $topic, ?string $key = null, ?int $partition = null) - { - $this->record = $record; - $this->topic = $topic; - $this->key = $key ?? 'recordId123'; - $this->partition = $partition; - } - public function success(Message $message): void { Log::info('Record successfully sent to broker.', [ diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index f7d13966..1689fe87 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -5,6 +5,9 @@ use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; use Metamorphosis\Facades\Metamorphosis; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Tests\Integration\Dummies\MessageConsumer; use Tests\Integration\Dummies\MessageProducer; use Tests\LaravelTestCase; @@ -17,11 +20,13 @@ class ProducerTest extends LaravelTestCase protected string $secondLowLevelMessage; + /** + * @group runProducer + */ public function testShouldRunAProducerAndReceiveMessagesWithAHighLevelConsumer(): void { // Given That I $this->haveAConsumerHandlerConfigured(); - $this->haveNoPartitionConfigured(); $this->haveSomeRandomMessagesProduced(); // I Expect That @@ -128,13 +133,9 @@ protected function runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingT private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $producer = app( - MessageProducer::class, - [ - 'record' => $this->highLevelMessage, - 'topic' => 'default', - ] - ); + $configOptionsProducer = $this->createConfigOptionsProducer('kafka-test'); + //$producer = app(MessageProducer::class, ['record' => $this->highLevelMessage, 'configOptions'=> $configOptionsProducer]); + $producer = new MessageProducer($this->highLevelMessage, $configOptionsProducer, 'recordId123'); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -142,8 +143,10 @@ private function haveSomeRandomMessagesProduced(): void private function produceRecordMessage(string $record): string { - $topic = 'low_level'; - $producer = app(MessageProducer::class, compact('record', 'topic')); + $configOptionsProducer = $this->createConfigOptionsProducer('low_level'); + $producer = new MessageProducer($record, $configOptionsProducer, 'recordId123'); + //$producer = app(MessageProducer::class, ['record'=>$record, 'configOptions' => $a]); + //$producer->topic = 'low_level'; Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -183,10 +186,20 @@ private function haveFourProducedMessages(): void $this->produceRecordMessage($this->secondLowLevelMessage); } - private function haveNoPartitionConfigured(): void - { - config( - ['kafka.topics.default.consumer.consumer_groups.test-consumer-group.partition' => -1] + private function createConfigOptionsProducer(string $topicId = 'kafka-test'): ProducerConfigOptions + { + $brokerOptions = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( + $topicId, + $brokerOptions, + null, + null, + [], + 20000, + false, + true, + 10, + 500 ); } } diff --git a/tests/Unit/ProducerTest.php b/tests/Unit/ProducerTest.php index 5a93dc8c..31650aef 100644 --- a/tests/Unit/ProducerTest.php +++ b/tests/Unit/ProducerTest.php @@ -12,7 +12,6 @@ use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; use Metamorphosis\TopicHandler\ConfigOptions\Broker; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; -use Metamorphosis\TopicHandler\Producer\AbstractHandler; use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Mockery as m; use RdKafka\Producer as KafkaProducer; @@ -38,7 +37,7 @@ public function testItShouldProduceRecordAsArrayThroughMiddlewareQueue(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class ($record, $topic) extends AbstractHandler { + $producerHandler = new class($record, $topic) extends AbstractProducer { }; // Expectations @@ -95,7 +94,7 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class ($record, $topic) extends AbstractHandler { + $producerHandler = new class($record, $topic) extends AbstractProducer { }; // Expectations @@ -128,7 +127,10 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void ->withAnyArgs(); // Actions - $producer->produce($producerHandler); + $result = $producer->produce($producerHandler); + + // Assertions + $this->assertNull($result); } public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): void @@ -147,7 +149,7 @@ public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): vo $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class ($record, $topic) extends AbstractHandler { + $producerHandler = new class($record, $topic) extends AbstractProducer { }; // Expectations @@ -223,7 +225,7 @@ public function testShouldBuildDispatcher(): void $producerTopic = m::mock(ProducerTopic::class); $configManager = m::mock(ProducerConfigManager::class); - $producerHandler = new class ($record, $topic) extends AbstractHandler { + $producerHandler = new class($record, $topic) extends AbstractProducer { }; // Expectations @@ -298,7 +300,7 @@ public function testShouldBuildDispatcherWithConfigOptions(): void $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); $broker = new Broker($connections, new None()); $configOptions = new ProducerConfigOptions($topicId, $broker); - $producerHandler = new class ($record, $configOptions) extends AbstractProducer { + $producerHandler = new class($record, $configOptions) extends AbstractProducer { }; // Expectations From 56f721bfc6bd5ae4005742d7f5b55120ea260aff Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 9 Mar 2022 09:45:27 -0300 Subject: [PATCH 084/152] chore: debug config options --- src/Connectors/Consumer/Config.php | 19 ++++-- src/ConsumerConfigManager.php | 7 +-- tests/Unit/AbstractConfigManagerTest.php | 75 ---------------------- tests/Unit/ConsumerConfigManagerTest.php | 79 ------------------------ 4 files changed, 16 insertions(+), 164 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index d14770fb..d2c97dd3 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -2,12 +2,14 @@ namespace Metamorphosis\Connectors\Consumer; +use InvalidArgumentException; use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\AbstractConfig; use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\TopicHandler\ConfigOptions\Consumer; use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; +use Metamorphosis\TopicHandler\Consumer\AbstractHandler; /** * This class is responsible for handling all configuration made on the @@ -41,12 +43,21 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; - public function makeWithConfigOptions(string $handlerClass, ?int $times = null): AbstractConfigManager + /** + * @param string $handlerClass + * @param int|null $times + * @return Consumer + */ + public function makeWithConfigOptions(string $handlerClass, ?int $times = null): ?Consumer { - $configManager = app(ConsumerConfigManager::class); - $configManager->set(['handler' => $handlerClass], ['times' => $times]); + /** @var AbstractHandler */ + $handler = app($handlerClass); + $configOptions = $handler->getConfigOptions(); + if(is_null($configOptions)){ + throw new InvalidArgumentException('Handler class cannot be null'); + } - return $configManager; + return $configOptions; } public function make(array $options, array $arguments): Consumer diff --git a/src/ConsumerConfigManager.php b/src/ConsumerConfigManager.php index 36d31c40..88207c97 100644 --- a/src/ConsumerConfigManager.php +++ b/src/ConsumerConfigManager.php @@ -22,12 +22,7 @@ public function set(array $config, ?array $commandConfig = null): void $this->remove('middlewares'); foreach ($middlewares as $middleware) { - $this->middlewares[] = is_string($middleware) - ? app( - $middleware, - ['configManager' => $this] - ) - : $middleware; + $this->middlewares[] = is_string($middleware) ? app($middleware) : $middleware; } if (!$consumerHandler) { diff --git a/tests/Unit/AbstractConfigManagerTest.php b/tests/Unit/AbstractConfigManagerTest.php index 7cf86812..e69de29b 100644 --- a/tests/Unit/AbstractConfigManagerTest.php +++ b/tests/Unit/AbstractConfigManagerTest.php @@ -1,75 +0,0 @@ -instance( - AbstractHandler::class, - m::mock(AbstractHandler::class) - ); - $config = [ - 'middlewares' => [], - 'handler' => AbstractHandler::class, - 'broker' => [ - 'default' => [ - 'connections' => 'kafka:9092', - ], - ], - 'topic_id' => 'kafka-test', - ]; - $broker = new Broker('kafka:9092', new None()); - $configOptions = new ConsumerConfigOptions( - 'kafka-override', - $broker, - '\App\Kafka\Consumers\ConsumerExample', - null, - null, - 'default', - null, - [MiddlewareDummy::class], - 200, - false, - true - ); - - $expected = [ - 'topic_id' => 'kafka-override', - 'connections' => 'kafka:9092', - 'auth' => null, - 'timeout' => 200, - 'handler' => '\App\Kafka\Consumers\ConsumerExample', - 'partition' => -1, - 'offset' => null, - 'consumer_group' => 'default', - 'auto_commit' => false, - 'commit_async' => true, - 'offset_reset' => 'smallest', - ]; - - $configManager = new ConsumerConfigManager(); - - // Expectations - $handler->expects() - ->getConfigOptions() - ->andReturn($configOptions); - - // Actions - $configManager->set($config); - - // Expectations - $this->assertEquals($expected, $configManager->get()); - } -} diff --git a/tests/Unit/ConsumerConfigManagerTest.php b/tests/Unit/ConsumerConfigManagerTest.php index 347aea6c..e69de29b 100644 --- a/tests/Unit/ConsumerConfigManagerTest.php +++ b/tests/Unit/ConsumerConfigManagerTest.php @@ -1,79 +0,0 @@ -instance( - AbstractHandler::class, - m::mock(AbstractHandler::class) - ); - $config = [ - 'middlewares' => [], - 'handler' => AbstractHandler::class, - 'broker' => [ - 'default' => [ - 'connections' => 'kafka:9092', - ], - ], - 'topic_id' => 'kafka-test', - ]; - $broker = new Broker('kafka:9092', new None()); - $configOptions = new ConsumerConfigOptions( - 'kafka-override', - $broker, - '\App\Kafka\Consumers\ConsumerExample', - null, - null, - 'default', - null, - [MiddlewareDummy::class], - 200, - false - ); - - $expected = [ - 'topic_id' => 'kafka-override', - 'connections' => 'kafka:9092', - 'auth' => null, - 'timeout' => 1000, - 'handler' => '\App\Kafka\Consumers\ConsumerExample', - 'partition' => -1, - 'offset' => null, - 'consumer_group' => 'default', - 'auto_commit' => false, - 'commit_async' => true, - 'offset_reset' => 'smallest', - 'times' => 2, - ]; - - $configManager = new ConsumerConfigManager(); - - $commandConfig = [ - 'timeout' => 1000, - 'times' => 2, - ]; - - // Expectations - $handler->expects() - ->getConfigOptions() - ->andReturn($configOptions); - - // Actions - $configManager->set($config, $commandConfig); - - // Expectations - $this->assertEquals($expected, $configManager->get()); - } -} From e90d589d05e8a4a4888c57c1b72e7cf59a4f1525 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 10 Mar 2022 17:23:57 -0300 Subject: [PATCH 085/152] chore: create interfaces for classes --- src/Avro/CachedSchemaRegistryClient.php | 4 ++-- src/Avro/Client.php | 2 +- src/Avro/Serializer/Encoders/SchemaId.php | 8 +++++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Avro/CachedSchemaRegistryClient.php b/src/Avro/CachedSchemaRegistryClient.php index c91e643c..97706cd5 100644 --- a/src/Avro/CachedSchemaRegistryClient.php +++ b/src/Avro/CachedSchemaRegistryClient.php @@ -5,7 +5,7 @@ use AvroSchemaParseException; use RuntimeException; -class CachedSchemaRegistryClient +class CachedSchemaRegistryClient implements CachedSchemaRegistryClientInterface { private Client $client; @@ -20,7 +20,7 @@ class CachedSchemaRegistryClient */ private array $subjectVersionToSchema = []; - public function __construct(Client $client) + public function __construct(AvroClientInterface $client) { $this->client = $client; } diff --git a/src/Avro/Client.php b/src/Avro/Client.php index e0b714ac..02516b70 100644 --- a/src/Avro/Client.php +++ b/src/Avro/Client.php @@ -5,7 +5,7 @@ use GuzzleHttp\Client as GuzzleHttp; use Psr\Http\Message\ResponseInterface; -class Client +class Client implements AvroClientInterface { private GuzzleHttp $client; diff --git a/src/Avro/Serializer/Encoders/SchemaId.php b/src/Avro/Serializer/Encoders/SchemaId.php index ea428472..fd796333 100644 --- a/src/Avro/Serializer/Encoders/SchemaId.php +++ b/src/Avro/Serializer/Encoders/SchemaId.php @@ -6,6 +6,7 @@ use AvroIODatumWriter; use AvroStringIO; use Metamorphosis\Avro\CachedSchemaRegistryClient; +use Metamorphosis\Avro\CachedSchemaRegistryClientInterface; use Metamorphosis\Avro\Schema; use Metamorphosis\Avro\Serializer\SchemaFormats; @@ -13,7 +14,7 @@ class SchemaId implements EncoderInterface { private CachedSchemaRegistryClient $registry; - public function __construct(CachedSchemaRegistryClient $registry) + public function __construct(CachedSchemaRegistryClientInterface $registry) { $this->registry = $registry; } @@ -41,4 +42,9 @@ public function encode(Schema $schema, $message): string return $io->string(); } + + public function getRegistry() + { + return $this->registry; + } } From 5922eeee7e7abb4f3b59c7e9a09cfa8ce1b366ca Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 10 Mar 2022 17:25:28 -0300 Subject: [PATCH 086/152] chore: update calls from configManager to configOptions --- src/Middlewares/AvroSchemaMixedEncoder.php | 29 ++++++++++------------ 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/Middlewares/AvroSchemaMixedEncoder.php b/src/Middlewares/AvroSchemaMixedEncoder.php index 4df0beb7..11534359 100644 --- a/src/Middlewares/AvroSchemaMixedEncoder.php +++ b/src/Middlewares/AvroSchemaMixedEncoder.php @@ -3,12 +3,12 @@ namespace Metamorphosis\Middlewares; use Closure; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Avro\CachedSchemaRegistryClient; use Metamorphosis\Avro\ClientFactory; use Metamorphosis\Avro\Serializer\Encoders\SchemaId; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptionsProducer; /** * Fetches a schema for a topic by subject and version (currently only 'latest') @@ -21,30 +21,27 @@ class AvroSchemaMixedEncoder implements MiddlewareInterface private CachedSchemaRegistryClient $schemaRegistry; - private AbstractConfigManager $configManager; + /** + * @var ConfigOptionsProducer + */ + private $configOptionsProducer; - public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, AbstractConfigManager $configManager) + public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, ConfigOptionsProducer $configOptionsProducer) { - if (!$configManager->get('url')) { - throw new ConfigurationException( - "Avro schema url not found, it's required to use AvroSchemaEncoder Middleware" - ); + if (!$configOptionsProducer->getAvroSchema()->getUrl()) { + throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaEncoder Middleware"); } - $schemaRegistry = $factory->make($configManager); +// $schemaRegistry = $factory->make($configOptionsProducer->getAvroSchema()); $this->schemaIdEncoder = $schemaIdEncoder; - $this->schemaRegistry = $schemaRegistry; - $this->configManager = $configManager; + $this->schemaRegistry = $schemaIdEncoder->getRegistry(); + $this->configOptionsProducer = $configOptionsProducer; } public function process(RecordInterface $record, Closure $next) { - $topic = $this->configManager->get('topic_id'); - $schema = $this->schemaRegistry->getBySubjectAndVersion( - "{$topic}-value", - 'latest' - ); - + $topic = $this->configOptionsProducer->getTopicId(); + $schema = $this->schemaRegistry->getBySubjectAndVersion("{$topic}-value", 'latest'); $arrayPayload = json_decode($record->getPayload(), true); $encodedPayload = $this->schemaIdEncoder->encode( $schema, From 65efb9808c9307d01eba2961a21e7ac717385da1 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 10 Mar 2022 17:26:51 -0300 Subject: [PATCH 087/152] chore: add interfaces --- src/Avro/AvroClientInterface.php | 6 ++++++ src/Avro/CachedSchemaRegistryClientInterface.php | 7 +++++++ 2 files changed, 13 insertions(+) create mode 100644 src/Avro/AvroClientInterface.php create mode 100644 src/Avro/CachedSchemaRegistryClientInterface.php diff --git a/src/Avro/AvroClientInterface.php b/src/Avro/AvroClientInterface.php new file mode 100644 index 00000000..d7617023 --- /dev/null +++ b/src/Avro/AvroClientInterface.php @@ -0,0 +1,6 @@ + Date: Thu, 10 Mar 2022 17:30:37 -0300 Subject: [PATCH 088/152] chore: add AvroSchemaMixedEncoderTest --- .../AvroSchemaMixedEncoderTest.php | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php diff --git a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php new file mode 100644 index 00000000..9412bd4b --- /dev/null +++ b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php @@ -0,0 +1,69 @@ +getSchemaTest(); + $configOptionsProducer = $this->createProducer(); + + $record = new ProducerRecord($schemaTest, 'kafka-test'); + $cachedSchemaRegistryClient = m::mock(CachedSchemaRegistryClient::class); + + $schema = (new Schema())->parse($schemaTest, '123'); + $clientFactory = new ClientFactory(); + $schemaIdEncoder = m::mock(SchemaId::class, [$cachedSchemaRegistryClient])->makePartial(); + + $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder($schemaIdEncoder, $clientFactory, $configOptionsProducer); + + //expect + $cachedSchemaRegistryClient->shouldReceive('getBySubjectAndVersion')->andReturn($schema); + $schemaIdEncoder->shouldReceive('encode')->andReturn('string'); + + //act + $avroSchemaMixedEncoder->process($record, $closure); + + //assert + $this->assertInstanceOf(AvroSchemaMixedEncoder::class, $avroSchemaMixedEncoder); + } + + private function createProducer() + { + $brokerOptions = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( + 'kafka-test', + $brokerOptions, + null, + new AvroSchema('http://url.teste', []), + [], + 20000, + false, + true, + 10, + 500 + ); + } + + private function getSchemaTest(): string + { + return file_get_contents(__DIR__.'/../fixtures/schemas/sales_price.avsc'); + } +} From b91b3873226ee6ebc08caaea7cc697916b4c3240 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 21 Mar 2022 15:25:33 -0300 Subject: [PATCH 089/152] chore: remove inutilized interfaces --- src/Avro/CachedSchemaRegistryClient.php | 4 ++-- src/Avro/Client.php | 2 +- src/Avro/Serializer/Encoders/SchemaId.php | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Avro/CachedSchemaRegistryClient.php b/src/Avro/CachedSchemaRegistryClient.php index 97706cd5..c91e643c 100644 --- a/src/Avro/CachedSchemaRegistryClient.php +++ b/src/Avro/CachedSchemaRegistryClient.php @@ -5,7 +5,7 @@ use AvroSchemaParseException; use RuntimeException; -class CachedSchemaRegistryClient implements CachedSchemaRegistryClientInterface +class CachedSchemaRegistryClient { private Client $client; @@ -20,7 +20,7 @@ class CachedSchemaRegistryClient implements CachedSchemaRegistryClientInterface */ private array $subjectVersionToSchema = []; - public function __construct(AvroClientInterface $client) + public function __construct(Client $client) { $this->client = $client; } diff --git a/src/Avro/Client.php b/src/Avro/Client.php index 02516b70..e0b714ac 100644 --- a/src/Avro/Client.php +++ b/src/Avro/Client.php @@ -5,7 +5,7 @@ use GuzzleHttp\Client as GuzzleHttp; use Psr\Http\Message\ResponseInterface; -class Client implements AvroClientInterface +class Client { private GuzzleHttp $client; diff --git a/src/Avro/Serializer/Encoders/SchemaId.php b/src/Avro/Serializer/Encoders/SchemaId.php index fd796333..75d08216 100644 --- a/src/Avro/Serializer/Encoders/SchemaId.php +++ b/src/Avro/Serializer/Encoders/SchemaId.php @@ -6,7 +6,6 @@ use AvroIODatumWriter; use AvroStringIO; use Metamorphosis\Avro\CachedSchemaRegistryClient; -use Metamorphosis\Avro\CachedSchemaRegistryClientInterface; use Metamorphosis\Avro\Schema; use Metamorphosis\Avro\Serializer\SchemaFormats; @@ -14,7 +13,7 @@ class SchemaId implements EncoderInterface { private CachedSchemaRegistryClient $registry; - public function __construct(CachedSchemaRegistryClientInterface $registry) + public function __construct(CachedSchemaRegistryClient $registry) { $this->registry = $registry; } From 8ac1a92efd305b501e725cc4d241e109b97c5266 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 21 Mar 2022 15:37:01 -0300 Subject: [PATCH 090/152] chore: add method to get consumerGroupId --- src/Connectors/Consumer/Config.php | 54 +++++++++++++----------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index d2c97dd3..aace08a9 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -64,22 +64,11 @@ public function make(array $options, array $arguments): Consumer { $configName = $options['config_name'] ?? 'kafka'; $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); - $consumerConfig = $this->getConsumerConfig( - $topicConfig, - $arguments['consumer_group'] - ); - $brokerConfig = $this->getBrokerConfig( - $configName, - $topicConfig['broker'] - ); - $schemaConfig = $this->getSchemaConfig( - $configName, - $arguments['topic'] - ); - $override = array_merge( - $this->filterValues($options), - $this->filterValues($arguments) - ); + $consumerGroupId = $this->getConsumerGroup($topicConfig, $arguments['consumer_group']); + $consumerConfig = $this->getConsumerConfig($topicConfig, $arguments, $consumerGroupId); + $brokerConfig = $this->getBrokerConfig($configName, $topicConfig['broker']); + $schemaConfig = $this->getSchemaConfig($configName, $arguments['topic']); + $override = array_merge($this->filterValues($options), $this->filterValues($arguments)); $config = array_merge( $topicConfig, $brokerConfig, @@ -89,6 +78,9 @@ public function make(array $options, array $arguments): Consumer $this->validate(array_merge($config, $override)); + $topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'] = $consumerConfig['partition']; + $topicConfig['consumer_group'] = $consumerGroupId; + return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); } @@ -110,31 +102,31 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } - private function getConsumerConfig(array $topicConfig, ?string $consumerGroupId = null): array + private function getConsumerConfig(array $topicConfig, array $arguments, string $consumerGroupId): array { - if ( - !$consumerGroupId && 1 === count( - $topicConfig['consumer']['consumer_groups'] - ) - ) { - $consumerGroupId = current( - array_keys($topicConfig['consumer']['consumer_groups']) - ); + $consumerConfig = $topicConfig['consumer']['consumer_groups'][$consumerGroupId] ?? null; + if (!$consumerConfig) { + throw new ConfigurationException("Consumer group '{$consumerGroupId}' not found"); } - $consumerGroupId = $consumerGroupId ?? 'default'; - $consumerConfig = $topicConfig['consumer']['consumer_groups'][$consumerGroupId] ?? null; $consumerConfig['consumer_group'] = $consumerGroupId; - if (!$consumerConfig) { - throw new ConfigurationException( - "Consumer group '{$consumerGroupId}' not found" - ); + if(isset($arguments['partition'])){ + $consumerConfig['partition'] = $arguments['partition']; } return $consumerConfig; } + private function getConsumerGroup(array $topicConfig, ?string $consumerGroupId): string + { + if (!$consumerGroupId && 1 === count($topicConfig['consumer']['consumer_groups'])) { + $consumerGroupId = current(array_keys($topicConfig['consumer']['consumer_groups'])); + } + + return $consumerGroupId ?? 'default'; + } + private function getMiddlewares(string $configName, array $topicConfig): array { return array_merge( From e97bf42c0026885e034667ca10637ed4d2dc083d Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 21 Mar 2022 15:58:27 -0300 Subject: [PATCH 091/152] chore: replace configManager by configOptions --- src/Connectors/Consumer/Config.php | 17 +- src/Connectors/Consumer/Factory.php | 12 +- src/Connectors/Producer/Connector.php | 6 +- src/Consumers/LowLevel.php | 8 +- src/Middlewares/AvroSchemaDecoder.php | 8 +- src/Middlewares/AvroSchemaMixedEncoder.php | 18 +- src/Producer.php | 23 +- src/Producer/Poll.php | 14 +- .../Factories/ConsumerFactory.php | 14 +- tests/Integration/ConsumerTest.php | 2 +- tests/Integration/ProducerTest.php | 31 ++- tests/Unit/Connectors/Consumer/ConfigTest.php | 8 +- .../Unit/Connectors/Consumer/FactoryTest.php | 75 +++--- .../Connectors/Consumer/HighLevelTest.php | 27 +- .../Unit/Connectors/Consumer/LowLevelTest.php | 28 ++- .../Connectors/Producer/ConnectorTest.php | 47 ++-- tests/Unit/Consumers/LowLevelTest.php | 19 +- .../Middlewares/AvroSchemaDecoderTest.php | 70 ++++++ .../AvroSchemaMixedEncoderTest.php | 97 ++++--- tests/Unit/Producer/PollTest.php | 69 +++-- tests/Unit/ProducerTest.php | 238 +++++------------- .../Factories/ConsumerFactoryTest.php | 1 + 22 files changed, 430 insertions(+), 402 deletions(-) create mode 100644 tests/Unit/Middlewares/AvroSchemaDecoderTest.php diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index aace08a9..4db8f76a 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -3,13 +3,10 @@ namespace Metamorphosis\Connectors\Consumer; use InvalidArgumentException; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\AbstractConfig; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\TopicHandler\ConfigOptions\Consumer; use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; -use Metamorphosis\TopicHandler\Consumer\AbstractHandler; /** * This class is responsible for handling all configuration made on the @@ -43,17 +40,11 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; - /** - * @param string $handlerClass - * @param int|null $times - * @return Consumer - */ public function makeWithConfigOptions(string $handlerClass, ?int $times = null): ?Consumer { - /** @var AbstractHandler */ $handler = app($handlerClass); $configOptions = $handler->getConfigOptions(); - if(is_null($configOptions)){ + if (is_null($configOptions)) { throw new InvalidArgumentException('Handler class cannot be null'); } @@ -77,8 +68,10 @@ public function make(array $options, array $arguments): Consumer ); $this->validate(array_merge($config, $override)); + if (isset($topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'])) { + $topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'] = $consumerConfig['partition']; + } - $topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'] = $consumerConfig['partition']; $topicConfig['consumer_group'] = $consumerGroupId; return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); @@ -111,7 +104,7 @@ private function getConsumerConfig(array $topicConfig, array $arguments, string $consumerConfig['consumer_group'] = $consumerGroupId; - if(isset($arguments['partition'])){ + if (isset($arguments['partition'])) { $consumerConfig['partition'] = $arguments['partition']; } diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index 893e9030..6a7c61aa 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -4,16 +4,20 @@ use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Middlewares\Handler\Dispatcher; -use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; /** * This factory will determine what kind of connector will be used. * Basically, if the user pass --partition and --offset as argument * means that we will use the low level approach. */ + +/** + * TODO Rename this class to ConsumerFactory, it will improve semantics + */ class Factory { - public static function make(ConfigOptions $configOptions): Manager + public static function make(ConsumerConfigOptions $configOptions): Manager { $autoCommit = $configOptions->isAutoCommit(); $commitAsync = $configOptions->isCommitASync(); @@ -26,14 +30,14 @@ public static function make(ConfigOptions $configOptions): Manager return new Manager($consumer, $handler, $dispatcher, $autoCommit, $commitAsync); } - protected static function requiresPartition(ConfigOptions $configOptions): bool + protected static function requiresPartition(ConsumerConfigOptions $configOptions): bool { $partition = $configOptions->getPartition(); return !is_null($partition) && $partition >= 0; } - public static function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface + public static function getConsumer(bool $autoCommit, ConsumerConfigOptions $configOptions): ConsumerInterface { if (self::requiresPartition($configOptions)) { return app(LowLevel::class)->getConsumer($autoCommit, $configOptions); diff --git a/src/Connectors/Producer/Connector.php b/src/Connectors/Producer/Connector.php index 68eafb07..1109a598 100644 --- a/src/Connectors/Producer/Connector.php +++ b/src/Connectors/Producer/Connector.php @@ -3,7 +3,7 @@ namespace Metamorphosis\Connectors\Producer; use Metamorphosis\Authentication\Factory; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Metamorphosis\TopicHandler\Producer\HandleableResponseInterface; use Metamorphosis\TopicHandler\Producer\HandlerInterface; use RdKafka\Conf; @@ -12,7 +12,7 @@ class Connector { - public function getProducerTopic(HandlerInterface $handler, ConfigOptions $configOptions): KafkaProducer + public function getProducerTopic(HandlerInterface $handler, ProducerConfigOptions $producerConfigOptions): KafkaProducer { $conf = resolve(Conf::class); @@ -29,7 +29,7 @@ function ($kafka, Message $message) use ($handler) { ); } - $broker = $configOptions->getBroker(); + $broker = $producerConfigOptions->getBroker(); $conf->set('metadata.broker.list', $broker->getConnections()); Factory::authenticate($conf, $broker->getAuth()); diff --git a/src/Consumers/LowLevel.php b/src/Consumers/LowLevel.php index e4a31ea0..e9fee56c 100644 --- a/src/Consumers/LowLevel.php +++ b/src/Consumers/LowLevel.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Consumers; -use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; use RdKafka\ConsumerTopic; use RdKafka\Message; @@ -14,12 +14,12 @@ class LowLevel implements ConsumerInterface private ?int $timeout; - public function __construct(ConsumerTopic $consumer, ConfigOptions $configOptions) + public function __construct(ConsumerTopic $consumer, ConsumerConfigOptions $consumerConfigOptions) { $this->consumer = $consumer; - $this->partition = $configOptions->getPartition(); - $this->timeout = $configOptions->getTimeout(); + $this->partition = $consumerConfigOptions->getPartition(); + $this->timeout = $consumerConfigOptions->getTimeout(); } public function consume(): ?Message diff --git a/src/Middlewares/AvroSchemaDecoder.php b/src/Middlewares/AvroSchemaDecoder.php index 8d185fdd..1aeffcff 100644 --- a/src/Middlewares/AvroSchemaDecoder.php +++ b/src/Middlewares/AvroSchemaDecoder.php @@ -8,6 +8,7 @@ use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; class AvroSchemaDecoder implements MiddlewareInterface { @@ -18,14 +19,13 @@ class AvroSchemaDecoder implements MiddlewareInterface */ private $avroSchema; - public function __construct(AvroSchema $avroSchema, ClientFactory $factory) + public function __construct(ClientFactory $factory, ConsumerConfigOptions $consumerConfigOptions) { - $this->avroSchema = $avroSchema; - if (!$this->avroSchema->getUrl()) { + if (!$consumerConfigOptions->getAvroSchema()->getUrl()) { throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaDecoder Middleware"); } - $this->decoder = new MessageDecoder($factory->make($avroSchema)); + $this->decoder = new MessageDecoder($factory->make($consumerConfigOptions->getAvroSchema())); } public function process(RecordInterface $record, Closure $next) diff --git a/src/Middlewares/AvroSchemaMixedEncoder.php b/src/Middlewares/AvroSchemaMixedEncoder.php index 11534359..6969a80e 100644 --- a/src/Middlewares/AvroSchemaMixedEncoder.php +++ b/src/Middlewares/AvroSchemaMixedEncoder.php @@ -8,7 +8,7 @@ use Metamorphosis\Avro\Serializer\Encoders\SchemaId; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptionsProducer; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; /** * Fetches a schema for a topic by subject and version (currently only 'latest') @@ -22,25 +22,25 @@ class AvroSchemaMixedEncoder implements MiddlewareInterface private CachedSchemaRegistryClient $schemaRegistry; /** - * @var ConfigOptionsProducer + * @var ProducerConfigOptions */ - private $configOptionsProducer; + private $producerConfigOptions; - public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, ConfigOptionsProducer $configOptionsProducer) + public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, ProducerConfigOptions $producerConfigOptions) { - if (!$configOptionsProducer->getAvroSchema()->getUrl()) { + if (!$producerConfigOptions->getAvroSchema()->getUrl()) { throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaEncoder Middleware"); } -// $schemaRegistry = $factory->make($configOptionsProducer->getAvroSchema()); + $schemaRegistry = $factory->make($producerConfigOptions->getAvroSchema()); $this->schemaIdEncoder = $schemaIdEncoder; - $this->schemaRegistry = $schemaIdEncoder->getRegistry(); - $this->configOptionsProducer = $configOptionsProducer; + $this->schemaRegistry = $schemaRegistry; + $this->producerConfigOptions = $producerConfigOptions; } public function process(RecordInterface $record, Closure $next) { - $topic = $this->configOptionsProducer->getTopicId(); + $topic = $this->producerConfigOptions->getTopicId(); $schema = $this->schemaRegistry->getBySubjectAndVersion("{$topic}-value", 'latest'); $arrayPayload = json_decode($record->getPayload(), true); $encodedPayload = $this->schemaIdEncoder->encode( diff --git a/src/Producer.php b/src/Producer.php index e1187cf0..ee277fdf 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -7,7 +7,7 @@ use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\Middlewares\Handler\Producer as ProducerMiddleware; use Metamorphosis\Producer\Poll; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Metamorphosis\TopicHandler\Producer\HandlerInterface; @@ -32,21 +32,26 @@ public function produce(HandlerInterface $producerHandler): void public function build(HandlerInterface $producerHandler): Dispatcher { - $configOptions = $producerHandler->getConfigOptions(); + $producerConfigOptions = $producerHandler->getConfigOptions(); - $middlewares = $configOptions->getMiddlewares(); - $middlewares[] = $this->getProducerMiddleware($producerHandler, $configOptions); + $middlewares = $producerConfigOptions->getMiddlewares(); + + foreach ($middlewares as &$middleware) { + $middleware = is_string($middleware) ? app($middleware, ['producerConfigOptions' => $producerConfigOptions]) : $middleware; + } + + $middlewares[] = $this->getProducerMiddleware($producerHandler, $producerConfigOptions); return new Dispatcher($middlewares); } - public function getProducerMiddleware(HandlerInterface $producerHandler, ConfigOptions $configOptions): ProducerMiddleware + public function getProducerMiddleware(HandlerInterface $producerHandler, ProducerConfigOptions $producerConfigOptions): ProducerMiddleware { - $producer = $this->connector->getProducerTopic($producerHandler, $configOptions); + $producer = $this->connector->getProducerTopic($producerHandler, $producerConfigOptions); - $topic = $producer->newTopic($configOptions->getTopicId()); - $poll = app(Poll::class, ['producer' => $producer, 'configOptions' => $configOptions]); - $partition = $configOptions->getPartition(); + $topic = $producer->newTopic($producerConfigOptions->getTopicId()); + $poll = app(Poll::class, ['producer' => $producer, 'producerConfigOptions' => $producerConfigOptions]); + $partition = $producerConfigOptions->getPartition(); return app( ProducerMiddleware::class, diff --git a/src/Producer/Poll.php b/src/Producer/Poll.php index 4c3b9c81..c07508ec 100644 --- a/src/Producer/Poll.php +++ b/src/Producer/Poll.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Producer; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use RdKafka\Producer; use RuntimeException; @@ -24,13 +24,13 @@ class Poll private Producer $producer; - public function __construct(Producer $producer, ConfigOptions $configOptions) + public function __construct(Producer $producer, ProducerConfigOptions $producerConfigOptions) { - $this->isAsync = $configOptions->isAsync(); - $this->maxPollRecords = $configOptions->getMaxPollRecords(); - $this->requiredAcknowledgment = $configOptions->isRequiredAcknowledgment(); - $this->maxFlushAttempts = $configOptions->getFlushAttempts(); - $this->timeout = $configOptions->getTimeout(); + $this->isAsync = $producerConfigOptions->isAsync(); + $this->maxPollRecords = $producerConfigOptions->getMaxPollRecords(); + $this->requiredAcknowledgment = $producerConfigOptions->isRequiredAcknowledgment(); + $this->maxFlushAttempts = $producerConfigOptions->getFlushAttempts(); + $this->timeout = $producerConfigOptions->getTimeout(); $this->producer = $producer; } diff --git a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php index 103b882e..7bd58c93 100644 --- a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php +++ b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php @@ -22,20 +22,16 @@ public static function make( private static function getConsumerGroupConfig(array $topicData): array { $topicData['topicId'] = $topicData['topic_id']; + $topicData['consumerGroup'] = $topicData['consumer_group']; - $consumer = current($topicData['consumer']); - $topicData['consumerGroup'] = key($consumer); + $consumerGroup = $topicData['consumerGroup']; + $consumer = current($topicData['consumer'])[$consumerGroup]; - return array_merge( - $topicData, - self::convertConfigAttributes($consumer) - ); + return array_merge_recursive($topicData, self::convertConfigAttributes($consumer)); } - private static function convertConfigAttributes(array $topic): array + private static function convertConfigAttributes(array $consumerConfig): array { - $consumerConfig = current($topic); - if (isset($consumerConfig['auto_commit'])) { $consumerConfig['autoCommit'] = $consumerConfig['auto_commit']; } diff --git a/tests/Integration/ConsumerTest.php b/tests/Integration/ConsumerTest.php index 1d4397e8..7088e46b 100644 --- a/tests/Integration/ConsumerTest.php +++ b/tests/Integration/ConsumerTest.php @@ -51,7 +51,7 @@ public function testItShouldSetup(): void MessageProducerWithConfigOptions::class, [ 'record' => ['id' => 'MESSAGE_ID'], - 'configOptions' => $producerConfigOptions, + 'producer' => $producerConfigOptions, 'key' => 1, ] ); diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 1689fe87..9780d74c 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -27,6 +27,7 @@ public function testShouldRunAProducerAndReceiveMessagesWithAHighLevelConsumer() { // Given That I $this->haveAConsumerHandlerConfigured(); + $this->haveAConsumerPartitionConfigured(); $this->haveSomeRandomMessagesProduced(); // I Expect That @@ -69,6 +70,11 @@ protected function haveAConsumerHandlerConfigured(): void ); } + protected function haveAConsumerPartitionConfigured(): void + { + config(['kafka.topics.default.consumer.consumer_groups.test-consumer-group.partition' => -1]); + } + protected function runTheConsumer(): void { $this->artisan( @@ -133,9 +139,9 @@ protected function runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingT private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $configOptionsProducer = $this->createConfigOptionsProducer('kafka-test'); - //$producer = app(MessageProducer::class, ['record' => $this->highLevelMessage, 'configOptions'=> $configOptionsProducer]); - $producer = new MessageProducer($this->highLevelMessage, $configOptionsProducer, 'recordId123'); + $producerConfigOptions = $this->createProducerConfigOptions('kafka-test'); +// $producer = app(MessageProducer::class, ['record' => $this->highLevelMessage, 'producer'=> $producerConfigOptions]); + $producer = new MessageProducer($this->highLevelMessage, $producerConfigOptions, 'recordId123'); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -143,10 +149,8 @@ private function haveSomeRandomMessagesProduced(): void private function produceRecordMessage(string $record): string { - $configOptionsProducer = $this->createConfigOptionsProducer('low_level'); - $producer = new MessageProducer($record, $configOptionsProducer, 'recordId123'); - //$producer = app(MessageProducer::class, ['record'=>$record, 'configOptions' => $a]); - //$producer->topic = 'low_level'; + $producerConfigOptions = $this->createProducerConfigOptions('low_level'); + $producer = new MessageProducer($record, $producerConfigOptions, 'recordId123'); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -186,20 +190,15 @@ private function haveFourProducedMessages(): void $this->produceRecordMessage($this->secondLowLevelMessage); } - private function createConfigOptionsProducer(string $topicId = 'kafka-test'): ProducerConfigOptions + private function createProducerConfigOptions(string $topicId = 'kafka-test'): ProducerConfigOptions { - $brokerOptions = new Broker('kafka:9092', new None()); + $broker = new Broker('kafka:9092', new None()); return new ProducerConfigOptions( $topicId, - $brokerOptions, + $broker, null, null, - [], - 20000, - false, - true, - 10, - 500 + [] ); } } diff --git a/tests/Unit/Connectors/Consumer/ConfigTest.php b/tests/Unit/Connectors/Consumer/ConfigTest.php index 9e1c26f9..539b6ea0 100644 --- a/tests/Unit/Connectors/Consumer/ConfigTest.php +++ b/tests/Unit/Connectors/Consumer/ConfigTest.php @@ -84,10 +84,10 @@ public function testShouldValidateConsumerConfig(): void ]); // Actions - $configManager = $config->make($options, $arguments); + $configManager = $config->makeWithConfigOptions(ConsumerHandlerDummy::class); // Assertions - $this->assertArraySubset($expected, $configManager->get()); + $this->assertArraySubset($expected, $configManager->toArray()); } public function testShouldNotSetRuntimeConfigWhenOptionsIsInvalid(): void @@ -109,7 +109,7 @@ public function testShouldNotSetRuntimeConfigWhenOptionsIsInvalid(): void $configManager = $config->make($options, $arguments); // Assertions - $this->assertEmpty($configManager->get()); + $this->assertEmpty($configManager->toArray()); } public function testShouldNotSetRuntimeConfigWhenKafkaConfigIsInvalid(): void @@ -132,6 +132,6 @@ public function testShouldNotSetRuntimeConfigWhenKafkaConfigIsInvalid(): void $configManager = $config->make($options, $arguments); // Assertions - $this->assertEmpty($configManager->get()); + $this->assertEmpty($configManager->toArray()); } } diff --git a/tests/Unit/Connectors/Consumer/FactoryTest.php b/tests/Unit/Connectors/Consumer/FactoryTest.php index a90471d6..50a158f2 100644 --- a/tests/Unit/Connectors/Consumer/FactoryTest.php +++ b/tests/Unit/Connectors/Consumer/FactoryTest.php @@ -11,48 +11,6 @@ class FactoryTest extends LaravelTestCase { - public function testItMakesManagerWithLowLevelConsumer(): void - { - // Set - $config = new Config(); - $configManager = $config->make( - ['timeout' => 61], - ['topic' => 'topic_key', 'consumer_group' => 'with-partition'] - ); - $manager = Factory::make($configManager); - - // Assertions - $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); - } - - public function testItMakesManagerWithLowLevelConsumerWhenPartitionIsNotValid(): void - { - // Set - $config = new Config(); - $configManager = $config->make( - ['timeout' => 61], - ['topic' => 'topic_key', 'consumer_group' => 'with-partition', 'partition' => -1] - ); - $manager = Factory::make($configManager); - - // Assertions - $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); - } - - public function testItMakesHighLevelClass(): void - { - // Set - $config = new Config(); - $configManager = $config->make( - ['timeout' => 61], - ['topic' => 'topic_key', 'consumer_group' => 'without-partition'] - ); - $manager = Factory::make($configManager); - - // Assertions - $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); - } - protected function setUp(): void { parent::setUp(); @@ -88,4 +46,37 @@ protected function setUp(): void ], ]); } + + public function testItMakesManagerWithLowLevelConsumer(): void + { + // Set + $config = new Config(); + $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); + } + + public function testItMakesManagerWithLowLevelConsumerWhenPartitionIsNotValid(): void + { + // Set + $config = new Config(); + $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition', 'partition' => -1]); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); + } + + public function testItMakesHighLevelClass(): void + { + // Set + $config = new Config(); + $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'without-partition']); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); + } } diff --git a/tests/Unit/Connectors/Consumer/HighLevelTest.php b/tests/Unit/Connectors/Consumer/HighLevelTest.php index 681c7ce5..97ab8e91 100644 --- a/tests/Unit/Connectors/Consumer/HighLevelTest.php +++ b/tests/Unit/Connectors/Consumer/HighLevelTest.php @@ -3,8 +3,11 @@ namespace Tests\Unit\Connectors\Consumer; use Metamorphosis\Connectors\Consumer\HighLevel; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\HighLevel as HighLevelConsumer; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; use Tests\LaravelTestCase; class HighLevelTest extends LaravelTestCase @@ -12,20 +15,20 @@ class HighLevelTest extends LaravelTestCase public function testItShouldMakeConnectorSetup(): void { // Set - $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'connections' => $connections, - 'consumer_group' => 'some-group', - 'topic_id' => 'some_topic', - 'offset_reset' => 'earliest', - 'timeout' => 1000, - 'max_poll_interval_ms' => 900000, - ]); $connector = new HighLevel(); + $brokerOptions = new Broker('kafka:9092', new None()); + $consumerConfigOptions = new ConsumerConfigOptions( + 'kafka-test', + $brokerOptions, + null, + 1, + 0, + 'some-group', + new AvroSchemaConfigOptions('http://url.teste') + ); // Actions - $result = $connector->getConsumer(false, $configManager); + $result = $connector->getConsumer(false, $consumerConfigOptions); // Assertions $this->assertInstanceOf(HighLevelConsumer::class, $result); diff --git a/tests/Unit/Connectors/Consumer/LowLevelTest.php b/tests/Unit/Connectors/Consumer/LowLevelTest.php index b3a081df..21c12291 100644 --- a/tests/Unit/Connectors/Consumer/LowLevelTest.php +++ b/tests/Unit/Connectors/Consumer/LowLevelTest.php @@ -3,8 +3,11 @@ namespace Tests\Unit\Connectors\Consumer; use Metamorphosis\Connectors\Consumer\LowLevel; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\LowLevel as LowLevelConsumer; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; use Tests\LaravelTestCase; class LowLevelTest extends LaravelTestCase @@ -12,21 +15,20 @@ class LowLevelTest extends LaravelTestCase public function testItShouldMakeConnectorSetup(): void { // Set - $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'connections' => $connections, - 'consumer_group' => 'some-group', - 'topic' => 'some_topic', - 'offset_reset' => 'earliest', - 'max_poll_interval_ms' => 900000, - 'offset' => 0, - 'partition' => 1, - ]); $connector = new LowLevel(); + $brokerOptions = new Broker('kafka:9092', new None()); + $consumerConfigOptions = new ConsumerConfigOptions( + 'kafka-test', + $brokerOptions, + null, + 1, + 0, + 'some-group', + new AvroSchemaConfigOptions('http://url.teste') + ); // Actions - $result = $connector->getConsumer(true, $configManager); + $result = $connector->getConsumer(true, $consumerConfigOptions); // Assertions $this->assertInstanceOf(LowLevelConsumer::class, $result); diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index e0cb6123..f456dfb7 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -3,7 +3,8 @@ namespace Tests\Unit\Connectors\Producer; use Metamorphosis\Connectors\Producer\Connector; -use Metamorphosis\ProducerConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Metamorphosis\TopicHandler\Producer\HandleableResponseInterface; @@ -27,12 +28,12 @@ public function testItShouldMakeSetup(): void KafkaProducer::class, m::mock(KafkaProducer::class) ); - $configManager = m::mock(ProducerConfigManager::class); - $configOptions = m::mock(ProducerConfigOptions::class); + + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); - $handler = new class ('record', $configOptions) extends AbstractProducer implements HandleableResponseInterface { - /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */ + $handler = new class('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface { public function success(Message $message): void { } @@ -49,18 +50,14 @@ public function failed(Message $message): void ->withAnyArgs(); $conf->expects() - ->set('metadata.broker.list', 'kafka:9092'); - - $configManager->expects() - ->get('connections') - ->andReturn('kafka:9092'); + ->set('metadata.broker.list', 0); - $configManager->expects() - ->get('auth.type') - ->andReturn('none'); + $producerConfigOptions->expects() + ->getBroker() + ->andReturn($broker); // Actions - $result = $connector->getProducerTopic($handler, $configManager); + $result = $connector->getProducerTopic($handler, $producerConfigOptions); // Assertions $this->assertInstanceOf(KafkaProducer::class, $result); @@ -77,12 +74,12 @@ public function testItShouldMakeSetupWithoutHandleResponse(): void KafkaProducer::class, m::mock(KafkaProducer::class) ); - $configManager = m::mock(ProducerConfigManager::class); - $configOptions = m::mock(ProducerConfigOptions::class); + + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); - $handler = new class ('record', $configOptions) extends AbstractProducer implements HandlerInterface { - /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */ + $handler = new class('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface { public function success(Message $message): void { } @@ -98,18 +95,14 @@ public function failed(Message $message): void ->never(); $conf->expects() - ->set('metadata.broker.list', 'kafka:9092'); - - $configManager->expects() - ->get('connections') - ->andReturn('kafka:9092'); + ->set('metadata.broker.list', 0); - $configManager->expects() - ->get('auth.type') - ->andReturn('none'); + $producerConfigOptions->expects() + ->getBroker() + ->andReturn($broker); // Actions - $result = $connector->getProducerTopic($handler, $configManager); + $result = $connector->getProducerTopic($handler, $producerConfigOptions); // Assertions $this->assertInstanceOf(KafkaProducer::class, $result); diff --git a/tests/Unit/Consumers/LowLevelTest.php b/tests/Unit/Consumers/LowLevelTest.php index 6b35f993..cc8bb988 100644 --- a/tests/Unit/Consumers/LowLevelTest.php +++ b/tests/Unit/Consumers/LowLevelTest.php @@ -4,6 +4,10 @@ use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\LowLevel; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; use Mockery as m; use RdKafka\ConsumerTopic; use RdKafka\Message; @@ -19,10 +23,23 @@ public function testItShouldConsume(): void $configManager = new ConsumerConfigManager(); $configManager->set(compact('timeout', 'partition')); + $brokerOptions = new Broker('kafka:9092', new None()); + $consumerConfigOptions = new ConsumerConfigOptions( + 'kafka-test', + $brokerOptions, + null, + $partition, + null, + '', + new AvroSchemaConfigOptions('http://url.teste'), + [], + $timeout + ); + $consumerTopic = m::mock(ConsumerTopic::class); $message = new Message(); - $lowLevelConsumer = new LowLevel($consumerTopic, $configManager); + $lowLevelConsumer = new LowLevel($consumerTopic, $consumerConfigOptions); // Expectations $consumerTopic->expects() diff --git a/tests/Unit/Middlewares/AvroSchemaDecoderTest.php b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php new file mode 100644 index 00000000..6c8d31b0 --- /dev/null +++ b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php @@ -0,0 +1,70 @@ +getAvroSchema(); + $avroSchema = new AvroSchema('string'); + $decoder = m::mock(Schema::class); + $clientFactory = m::mock(ClientFactory::class); + $cachedSchemaRegistryClient = m::mock(CachedSchemaRegistryClient::class); + $expected = 'my awesome message'; + + $message = new Message(); + $message->payload = "\x01\x00\x00\x00\fmy-topic-key\x00\x00\x00\x05\$my awesome message"; + $message->err = 0; + + $closure = Closure::fromCallable(function ($producerRecord) { + return $producerRecord; + }); + + $consumerRecord = new ConsumerRecord($message); + + // Expectations + $clientFactory->expects() + ->make($avroSchemaConfigOptions) + ->andReturn($cachedSchemaRegistryClient); + + $cachedSchemaRegistryClient->expects() + ->getBySubjectAndVersion('my-topic-key', 5) + ->andReturn($decoder); + + $decoder->expects() + ->getAvroSchema() + ->andReturn($avroSchema); + + $avroSchemaDecoder = new AvroSchemaDecoder($clientFactory, $consumerConfigOptions); + + $result = $avroSchemaDecoder->process($consumerRecord, $closure); + + $this->assertSame($expected, $result->getPayload()); + } +} diff --git a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php index 9412bd4b..fcff2d6c 100644 --- a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php +++ b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php @@ -1,6 +1,8 @@ getSchemaFixture(); + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + 'kafka-test', + $broker, + null, + new AvroSchemaConfigOptions('subjects/kafka-test-value/versions/latest', []) + ); + $avroSchemaConfigOptions = $producerConfigOptions->getAvroSchema(); - $schemaTest = $this->getSchemaTest(); - $configOptionsProducer = $this->createProducer(); + $clientFactory = m::mock(ClientFactory::class); - $record = new ProducerRecord($schemaTest, 'kafka-test'); $cachedSchemaRegistryClient = m::mock(CachedSchemaRegistryClient::class); + $schemaIdEncoder = m::mock(SchemaId::class, [$cachedSchemaRegistryClient]); + + $schema = new Schema(); + $parsedSchema = $schema->parse($avroSchema, '123', 'kafka-test-value', 'latest'); + $record = $this->getRecord($parsedSchema->getAvroSchema()); + $producerRecord = new ProducerRecord($record, 'kafka-test'); + + $closure = Closure::fromCallable(function ($producerRecord) { + return $producerRecord; + }); + + $payload = json_decode($producerRecord->getPayload(), true); + $encodedMessage = 'binary_message'; - $schema = (new Schema())->parse($schemaTest, '123'); - $clientFactory = new ClientFactory(); - $schemaIdEncoder = m::mock(SchemaId::class, [$cachedSchemaRegistryClient])->makePartial(); + // Expectations + $clientFactory->expects() + ->make($avroSchemaConfigOptions) + ->andReturn($cachedSchemaRegistryClient); - $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder($schemaIdEncoder, $clientFactory, $configOptionsProducer); + $cachedSchemaRegistryClient->expects() + ->getBySubjectAndVersion('kafka-test-value', 'latest') + ->andReturn($schema); - //expect - $cachedSchemaRegistryClient->shouldReceive('getBySubjectAndVersion')->andReturn($schema); - $schemaIdEncoder->shouldReceive('encode')->andReturn('string'); + $schemaIdEncoder->expects() + ->encode($schema, $payload) + ->andReturn($encodedMessage); - //act - $avroSchemaMixedEncoder->process($record, $closure); + // Actions + $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder($schemaIdEncoder, $clientFactory, $producerConfigOptions); + $result = $avroSchemaMixedEncoder->process($producerRecord, $closure); - //assert - $this->assertInstanceOf(AvroSchemaMixedEncoder::class, $avroSchemaMixedEncoder); + // Assertions + $this->assertSame($record, $result->getOriginal()); + $this->assertSame($encodedMessage, $result->getPayload()); } - private function createProducer() + private function getRecord(AvroSchema $avroSchema): string { - $brokerOptions = new Broker('kafka:9092', new None()); - return new ProducerConfigOptions( - 'kafka-test', - $brokerOptions, - null, - new AvroSchema('http://url.teste', []), - [], - 20000, - false, - true, - 10, - 500 - ); + $defaultValues = [ + 'null' => null, + 'boolean' => true, + 'string' => 'abc', + 'int' => 1, + 'long' => 1.0, + 'float' => 1.0, + 'double' => 1.0, + 'array' => [], + ]; + + $result = []; + foreach ($avroSchema->fields() as $field) { + $result[$field->name()] = $defaultValues[$field->type->type]; + } + + return json_encode($result); } - private function getSchemaTest(): string + private function getSchemaFixture(): string { return file_get_contents(__DIR__.'/../fixtures/schemas/sales_price.avsc'); } diff --git a/tests/Unit/Producer/PollTest.php b/tests/Unit/Producer/PollTest.php index 8bda848d..3ef21c96 100644 --- a/tests/Unit/Producer/PollTest.php +++ b/tests/Unit/Producer/PollTest.php @@ -4,6 +4,10 @@ use Metamorphosis\Producer\Poll; use Metamorphosis\ProducerConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Mockery as m; use RdKafka\Producer as KafkaProducer; use RuntimeException; @@ -14,17 +18,21 @@ class PollTest extends LaravelTestCase public function testItShouldHandleMessageWithoutAcknowledgment(): void { // Set - $configManager = new ProducerConfigManager(); - $configManager->set([ - 'topic_id' => 'topic_name', - 'timeout' => 4000, - 'is_async' => true, - 'max_poll_records' => 500, - 'flush_attempts' => 10, - 'required_acknowledgment' => false, - ]); + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + 'topic_name', + $broker, + null, + new AvroSchemaConfigOptions('string', []), + [], + 4000, + true, + false, + 500, + 10 + ); $kafkaProducer = m::mock(KafkaProducer::class); - $poll = new Poll($kafkaProducer, $configManager); + $poll = new Poll($kafkaProducer, $producerConfigOptions); // Expectations $kafkaProducer->expects() @@ -51,8 +59,21 @@ public function testShouldThrowExceptionWhenFlushFailed(): void 'partition' => 0, ]); + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + 'topic_name', + $broker, + null, + new AvroSchemaConfigOptions('string', []), + [], + 100, + false, + true, + 500, + 10 + ); $kafkaProducer = m::mock(KafkaProducer::class); - $poll = new Poll($kafkaProducer, $configManager); + $poll = new Poll($kafkaProducer, $producerConfigOptions); // Expectations $kafkaProducer->expects() @@ -72,18 +93,22 @@ public function testShouldThrowExceptionWhenFlushFailed(): void public function testItShouldHandleResponseEveryTimeWhenAsyncModeIsTrue(): void { // Set - $configManager = new ProducerConfigManager(); - $configManager->set([ - 'topic_id' => 'topic_name', - 'timeout' => 4000, - 'is_async' => false, - 'max_poll_records' => 500, - 'flush_attempts' => 10, - 'required_acknowledgment' => true, - 'partition' => 0, - ]); + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + 'topic_name', + $broker, + null, + new AvroSchemaConfigOptions('string', []), + [], + 4000, + false, + true, + 10, + 500 + ); + $kafkaProducer = m::mock(KafkaProducer::class); - $poll = new Poll($kafkaProducer, $configManager); + $poll = new Poll($kafkaProducer, $producerConfigOptions); // Expectations $kafkaProducer->expects() diff --git a/tests/Unit/ProducerTest.php b/tests/Unit/ProducerTest.php index 31650aef..02808390 100644 --- a/tests/Unit/ProducerTest.php +++ b/tests/Unit/ProducerTest.php @@ -8,8 +8,8 @@ use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\Middlewares\Handler\Producer as ProducerMiddleware; use Metamorphosis\Producer; -use Metamorphosis\ProducerConfigManager; use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; use Metamorphosis\TopicHandler\ConfigOptions\Broker; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Metamorphosis\TopicHandler\Producer\AbstractProducer; @@ -31,38 +31,22 @@ public function testItShouldProduceRecordAsArrayThroughMiddlewareQueue(): void ); $config = m::mock(Config::class); $connector = m::mock(Connector::class); - $configManager = m::mock(ProducerConfigManager::class)->makePartial(); $producer = new Producer($config, $connector); $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class($record, $topic) extends AbstractProducer { + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker + ); + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $config->expects() - ->makeByTopic($topic) - ->andReturn($configManager); - - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topic); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - - $configManager->expects() - ->get('timeout') - ->andReturn(1000); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() @@ -86,36 +70,27 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void ProducerMiddleware::class, m::mock(ProducerMiddleware::class) ); + $config = m::mock(Config::class); $connector = m::mock(Connector::class); - $configManager = m::mock(ProducerConfigManager::class)->makePartial(); + + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker + ); + $producer = new Producer($config, $connector); $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class($record, $topic) extends AbstractProducer { + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $config->expects() - ->makeByTopic($topic) - ->andReturn($configManager); - - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topic); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() @@ -127,10 +102,7 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void ->withAnyArgs(); // Actions - $result = $producer->produce($producerHandler); - - // Assertions - $this->assertNull($result); + $producer->produce($producerHandler); } public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): void @@ -145,52 +117,29 @@ public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): vo $config = m::mock(Config::class); $connector = m::mock(Connector::class); $producer = new Producer($config, $connector); - $configManager = m::mock(ProducerConfigManager::class); + $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class($record, $topic) extends AbstractProducer { + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker, + 0, + new AvroSchemaConfigOptions('string'), + [], + 1000, + false, + true, + 500, + 1 + ); + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topic); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - - $configManager->expects() - ->get('max_poll_records') - ->andReturn(500); - - $configManager->expects() - ->get('required_acknowledgment') - ->andReturn(true); - - $configManager->expects() - ->get('flush_attempts') - ->andReturn(1); - - $configManager->expects() - ->get('timeout') - ->andReturn(1000); - - $configManager->expects() - ->get('is_async') - ->andReturn(false); - - $config->expects() - ->makeByTopic($topic) - ->andReturn($configManager); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() @@ -223,50 +172,27 @@ public function testShouldBuildDispatcher(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $configManager = m::mock(ProducerConfigManager::class); - $producerHandler = new class($record, $topic) extends AbstractProducer { + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker, + null, + new AvroSchemaConfigOptions('string'), + [], + 1000, + true, + true, + 500, + 1 + ); + + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $config->expects() - ->makeByTopic($topic) - ->andReturn($configManager); - - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topic); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - - $configManager->expects() - ->get('max_poll_records') - ->andReturn(500); - - $configManager->expects() - ->get('is_async') - ->andReturn(true); - - $configManager->expects() - ->get('required_acknowledgment') - ->andReturn(true); - - $configManager->expects() - ->get('flush_attempts') - ->andReturn(1); - - $configManager->expects() - ->get('timeout') - ->andReturn(1000); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() @@ -288,7 +214,7 @@ public function testShouldBuildDispatcherWithConfigOptions(): void { // Set $record = json_encode(['message' => 'some message']); - $topicId = 'TOPIC-ID'; + $topic = 'TOPIC-ID'; $config = m::mock(Config::class); $connector = m::mock(Connector::class); @@ -296,56 +222,30 @@ public function testShouldBuildDispatcherWithConfigOptions(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $configManager = m::mock(ProducerConfigManager::class); - $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); - $broker = new Broker($connections, new None()); - $configOptions = new ProducerConfigOptions($topicId, $broker); - $producerHandler = new class($record, $configOptions) extends AbstractProducer { + + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker, + null, + new AvroSchemaConfigOptions('string'), + [], + 1000, + true, + true, + 500, + 1 + ); + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $config->expects() - ->make($configOptions) - ->andReturn($configManager); - - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topicId); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - - $configManager->expects() - ->get('max_poll_records') - ->andReturn(500); - - $configManager->expects() - ->get('is_async') - ->andReturn(true); - - $configManager->expects() - ->get('required_acknowledgment') - ->andReturn(true); - - $configManager->expects() - ->get('flush_attempts') - ->andReturn(1); - - $configManager->expects() - ->get('timeout') - ->andReturn(1000); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() - ->newTopic($topicId) + ->newTopic($topic) ->andReturn($producerTopic); $kafkaProducer->expects() diff --git a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php index cb78b0f7..f5c07f80 100644 --- a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php +++ b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php @@ -33,6 +33,7 @@ public function testShouldMakeConfigOptionWithAvroSchema(): void ]; $topicData = [ 'topic_id' => 'kafka-test', + 'consumer_group' => 'test-consumer-group', 'consumer' => [ 'consumer_groups' => [ 'test-consumer-group' => [ From 505c21c298840fc6b4ee763639f7eb2a844d2f78 Mon Sep 17 00:00:00 2001 From: David Franca Date: Mon, 21 Mar 2022 16:47:14 -0300 Subject: [PATCH 092/152] fix: update consumer middlewares --- src/Connectors/Consumer/Factory.php | 11 ++++++++++- src/Console/ConsumerCommand.php | 5 +++++ src/Facades/Metamorphosis.php | 1 + src/Producer.php | 1 - tests/Integration/ProducerTest.php | 27 ++++++++++++++++++++------- 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index 6a7c61aa..0da13cac 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -3,6 +3,7 @@ namespace Metamorphosis\Connectors\Consumer; use Metamorphosis\Consumers\ConsumerInterface; +use Metamorphosis\Middlewares\Handler\Consumer as ConsumerMiddleware; use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; @@ -23,9 +24,17 @@ public static function make(ConsumerConfigOptions $configOptions): Manager $commitAsync = $configOptions->isCommitASync(); $consumer = self::getConsumer($autoCommit, $configOptions); + $handler = app($configOptions->getHandler()); - $dispatcher = self::getMiddlewareDispatcher($configOptions->getMiddlewares()); + $middlewares = $configOptions->getMiddlewares(); + foreach ($middlewares as &$middleware) { + $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $configOptions]) : $middleware; + } + + $middlewares[] = app(ConsumerMiddleware::class, ['consumerTopicHandler' => $handler]); + + $dispatcher = self::getMiddlewareDispatcher($middlewares); return new Manager($consumer, $handler, $dispatcher, $autoCommit, $commitAsync); } diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index 5c233db4..bd4272e1 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -40,6 +40,11 @@ public function handle(Config $config) { $consumer = $config->make($this->option(), $this->argument()); + $middlewares = $consumer->getMiddlewares(); + foreach ($middlewares as &$middleware) { + $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $consumer]) : $middleware; + } + $this->writeStartingConsumer($consumer); $manager = Factory::make($consumer); diff --git a/src/Facades/Metamorphosis.php b/src/Facades/Metamorphosis.php index 70f903c8..c009a5f5 100644 --- a/src/Facades/Metamorphosis.php +++ b/src/Facades/Metamorphosis.php @@ -3,6 +3,7 @@ namespace Metamorphosis\Facades; use Illuminate\Support\Facades\Facade; +use Metamorphosis\TopicHandler\Producer\HandlerInterface; /** * @method static void produce(HandlerInterface $producerHandler) diff --git a/src/Producer.php b/src/Producer.php index ee277fdf..4fa9ee5b 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -35,7 +35,6 @@ public function build(HandlerInterface $producerHandler): Dispatcher $producerConfigOptions = $producerHandler->getConfigOptions(); $middlewares = $producerConfigOptions->getMiddlewares(); - foreach ($middlewares as &$middleware) { $middleware = is_string($middleware) ? app($middleware, ['producerConfigOptions' => $producerConfigOptions]) : $middleware; } diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 9780d74c..ec43989b 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -77,6 +77,8 @@ protected function haveAConsumerPartitionConfigured(): void protected function runTheConsumer(): void { + config(['kafka.topics.default.topic_id' => $this->topicId]); + $this->artisan( 'kafka:consume', [ @@ -139,9 +141,13 @@ protected function runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingT private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $producerConfigOptions = $this->createProducerConfigOptions('kafka-test'); -// $producer = app(MessageProducer::class, ['record' => $this->highLevelMessage, 'producer'=> $producerConfigOptions]); - $producer = new MessageProducer($this->highLevelMessage, $producerConfigOptions, 'recordId123'); + $this->topicId = 'kafka-test-'.Str::random(5); + $producerConfigOptions = $this->createProducerConfigOptions(); + $producer = app(MessageProducer::class, [ + 'record' => $this->highLevelMessage, + 'producer' => $producerConfigOptions, + 'key' => 'recordId123', + ]); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -150,7 +156,11 @@ private function haveSomeRandomMessagesProduced(): void private function produceRecordMessage(string $record): string { $producerConfigOptions = $this->createProducerConfigOptions('low_level'); - $producer = new MessageProducer($record, $producerConfigOptions, 'recordId123'); + $producer = app(MessageProducer::class, [ + 'record' => $record, + 'producer' => $producerConfigOptions, + 'key' => 'recordId123' + ]); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -190,15 +200,18 @@ private function haveFourProducedMessages(): void $this->produceRecordMessage($this->secondLowLevelMessage); } - private function createProducerConfigOptions(string $topicId = 'kafka-test'): ProducerConfigOptions + private function createProducerConfigOptions(): ProducerConfigOptions { $broker = new Broker('kafka:9092', new None()); return new ProducerConfigOptions( - $topicId, + $this->topicId, $broker, null, null, - [] + [], + 2000, + false, + true ); } } From 36042daec493552b9afddb5eff10b1d89e1c98e0 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:21:58 -0300 Subject: [PATCH 093/152] fix: get and set options from consumer command --- src/Connectors/Consumer/Config.php | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 4db8f76a..1b0b5c6e 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -56,7 +56,7 @@ public function make(array $options, array $arguments): Consumer $configName = $options['config_name'] ?? 'kafka'; $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); $consumerGroupId = $this->getConsumerGroup($topicConfig, $arguments['consumer_group']); - $consumerConfig = $this->getConsumerConfig($topicConfig, $arguments, $consumerGroupId); + $consumerConfig = $this->getConsumerConfig($topicConfig, $options, $consumerGroupId); $brokerConfig = $this->getBrokerConfig($configName, $topicConfig['broker']); $schemaConfig = $this->getSchemaConfig($configName, $arguments['topic']); $override = array_merge($this->filterValues($options), $this->filterValues($arguments)); @@ -68,8 +68,19 @@ public function make(array $options, array $arguments): Consumer ); $this->validate(array_merge($config, $override)); - if (isset($topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'])) { - $topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'] = $consumerConfig['partition']; + + if (isset($topicConfig['consumer']['consumer_groups'][$consumerGroupId])) { + if (isset($options['partition'])) { + $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['partition'] = $options['partition']; + } + + if (isset($options['offset'])) { + $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['offset'] = $options['offset']; + } + + if (isset($options['timeout'])) { + $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['timeout'] = $options['timeout']; + } } $topicConfig['consumer_group'] = $consumerGroupId; @@ -95,7 +106,7 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } - private function getConsumerConfig(array $topicConfig, array $arguments, string $consumerGroupId): array + private function getConsumerConfig(array $topicConfig, array $options, string $consumerGroupId): array { $consumerConfig = $topicConfig['consumer']['consumer_groups'][$consumerGroupId] ?? null; if (!$consumerConfig) { @@ -104,10 +115,6 @@ private function getConsumerConfig(array $topicConfig, array $arguments, string $consumerConfig['consumer_group'] = $consumerGroupId; - if (isset($arguments['partition'])) { - $consumerConfig['partition'] = $arguments['partition']; - } - return $consumerConfig; } From 2a02ecf727264b29504ef96c54fd49cb779de9b5 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:23:16 -0300 Subject: [PATCH 094/152] chore: remove inutilized code --- src/Console/ConsumerCommand.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index bd4272e1..70f151e0 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -39,12 +39,6 @@ class ConsumerCommand extends BaseCommand public function handle(Config $config) { $consumer = $config->make($this->option(), $this->argument()); - - $middlewares = $consumer->getMiddlewares(); - foreach ($middlewares as &$middleware) { - $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $consumer]) : $middleware; - } - $this->writeStartingConsumer($consumer); $manager = Factory::make($consumer); From 75c1852113ebb179f6715d27eab7394d4afb07fd Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:26:34 -0300 Subject: [PATCH 095/152] chore: create public variable for message --- tests/Integration/ProducerTest.php | 34 ++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index ec43989b..5a456a71 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -18,7 +18,22 @@ class ProducerTest extends LaravelTestCase protected string $firstLowLevelMessage; - protected string $secondLowLevelMessage; + /** + * @var string + */ + protected $secondLowLevelMessage; + + /** + * @var string + */ + protected $topicId; + + protected function setUp(): void + { + parent::setUp(); + $this->withoutAuthentication(); + $this->topicId = 'kafka-test-'.Str::random(5); + } /** * @group runProducer @@ -51,13 +66,6 @@ public function testShouldRunAProducerAndReceiveMessagesWithALowLevelConsumer(): $this->runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingToTwoMessagesConsumed(); } - protected function setUp(): void - { - parent::setUp(); - - $this->withoutAuthentication(); - } - protected function withoutAuthentication(): void { config(['kafka.brokers.default.auth' => []]); @@ -141,8 +149,8 @@ protected function runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingT private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $this->topicId = 'kafka-test-'.Str::random(5); - $producerConfigOptions = $this->createProducerConfigOptions(); + + $producerConfigOptions = $this->createProducerConfigOptions($this->topicId); $producer = app(MessageProducer::class, [ 'record' => $this->highLevelMessage, 'producer' => $producerConfigOptions, @@ -159,7 +167,7 @@ private function produceRecordMessage(string $record): string $producer = app(MessageProducer::class, [ 'record' => $record, 'producer' => $producerConfigOptions, - 'key' => 'recordId123' + 'key' => 'recordId123', ]); Metamorphosis::produce($producer); @@ -200,11 +208,11 @@ private function haveFourProducedMessages(): void $this->produceRecordMessage($this->secondLowLevelMessage); } - private function createProducerConfigOptions(): ProducerConfigOptions + private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { $broker = new Broker('kafka:9092', new None()); return new ProducerConfigOptions( - $this->topicId, + $topicId, $broker, null, null, From 589f9a1be77164b215dafe7f20cd937e799a8ca3 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:34:21 -0300 Subject: [PATCH 096/152] chore: create public variable for message --- tests/Integration/ProducerWithAvroTest.php | 65 ++++++++++++++++++---- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index 449370da..014100ef 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -6,15 +6,30 @@ use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\HandlerStack; use GuzzleHttp\Psr7\Response; +use Illuminate\Support\Facades\Log; use Metamorphosis\Facades\Metamorphosis; use Metamorphosis\Middlewares\AvroSchemaDecoder; use Metamorphosis\Middlewares\AvroSchemaMixedEncoder; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Tests\Integration\Dummies\MessageConsumer; use Tests\Integration\Dummies\MessageProducer; use Tests\LaravelTestCase; class ProducerWithAvroTest extends LaravelTestCase { + /** + * @var string[] + */ + protected $records; + + public function setUp(): void + { + parent::setUp(); + $this->records = ['saleOrderId' => 'SALE_ORDER_ID', 'productId' => 'PRODUCT_ID']; + } + public function testShouldRunAProducerMessagesWithAAvroSchema(): void { // Given That I @@ -22,7 +37,9 @@ public function testShouldRunAProducerMessagesWithAAvroSchema(): void // When I $this->haveSomeRandomMessagesProduced(); - $this->expectNotToPerformAssertions(); + + // I expect that + $this->myMessagesHaveBeenLogged(); } protected function haveAHandlerConfigured(): void @@ -93,14 +110,17 @@ protected function haveAHandlerConfigured(): void private function haveSomeRandomMessagesProduced(): void { - $saleOrderProducer = app( - MessageProducer::class, - ['record' => ['saleOrderId' => 'SALE_ORDER_ID'], 'topic' => 'sale_order'] - ); - $productProducer = app( - MessageProducer::class, - ['record' => ['productId' => 'PRODUCT_ID'], 'topic' => 'product'] - ); + $producerConfigOptionsSale = $this->createProducerConfigOptions('sale_order'); + $producerConfigOptionsProduct = $this->createProducerConfigOptions('product'); + + $saleOrderProducer = app(MessageProducer::class, [ + 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], + 'producer' => $producerConfigOptionsSale, + ]); + $productProducer = app(MessageProducer::class, [ + 'record' => ['productId' => 'PRODUCT_ID'], + 'producer' => $producerConfigOptionsProduct, + ]); $saleOrderSchemaResponse = '{ "subject":"sale_order-value", @@ -129,7 +149,32 @@ private function haveSomeRandomMessagesProduced(): void $productDispatcher = Metamorphosis::build($productProducer); $productDispatcher->handle($productProducer->createRecord()); + } - $saleOrderDispatcher->handle($saleOrderProducer->createRecord()); + private function myMessagesHaveBeenLogged(): void + { + $this->setLogExpectationsFor($this->records['saleOrderId']); + $this->setLogExpectationsFor($this->records['productId']); + } + + private function setLogExpectationsFor(string $message): void + { + Log::shouldReceive('info') + ->with($message); + } + + private function createProducerConfigOptions(string $topicId): ProducerConfigOptions + { + $broker = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( + $topicId, + $broker, + null, + null, + [], + 2000, + false, + true + ); } } From 8569e1d5271e92fe4d8a5b9771b1618a61b96c3e Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:37:24 -0300 Subject: [PATCH 097/152] chore: create public variable for message --- tests/Integration/ProducerWithConfigOptionsTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 4e256a42..f4f58b94 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -88,7 +88,7 @@ private function haveSomeRandomMessageProduced(): void MessageProducerWithConfigOptions::class, [ 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], - 'configOptions' => $this->producerConfigOptions, + 'producer' => $this->producerConfigOptions, 'key' => 1, ] ); From e67052ee2877c3d59bd15ef01eb1ac75d15f636e Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:38:44 -0300 Subject: [PATCH 098/152] fix: change parameter order --- tests/Unit/Connectors/Consumer/FactoryTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Unit/Connectors/Consumer/FactoryTest.php b/tests/Unit/Connectors/Consumer/FactoryTest.php index 50a158f2..ae3dedf6 100644 --- a/tests/Unit/Connectors/Consumer/FactoryTest.php +++ b/tests/Unit/Connectors/Consumer/FactoryTest.php @@ -58,11 +58,11 @@ public function testItMakesManagerWithLowLevelConsumer(): void $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); } - public function testItMakesManagerWithLowLevelConsumerWhenPartitionIsNotValid(): void + public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid(): void { // Set $config = new Config(); - $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition', 'partition' => -1]); + $configConsumer = $config->make(['timeout' => 61, 'partition' => -1], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); $manager = Factory::make($configConsumer); // Assertions From 981526b6edf2b21ddc2e8d92ba5fe5c78ff49c07 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 16:44:23 -0300 Subject: [PATCH 099/152] docs: add CHANGELOG file --- CHANGELOG.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..0ecf5eae --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,40 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- + +### Fixed + +- + +### Removed +- Remove deprecated class AbstractHandler + +## [4.1.0] - 2022-03-23 + +### Added +- Added AvroSchemaMixedEncoderTest +- Added AvroSchemaDecoderTest +- Added ProducerWithConfigOptionsTest +- Added ConfigOptionsCommand to run commands with ConfigOptions class +- Added pt_BR contributing section +- Added setup-dev script on composer +- Added grumphp commit validation + +### Fixed +- Fixed parameters and options override on Consumer\Config class +- Update instructions on contribute section +- Update project install section + +### Changed +- Updated class from ConfigManager to ConfigOptions on unit tests +- Updated class from ConfigManager to ConfigOptions where any config request was made +- Consumer and Producer middlewares resolution From 5edaa00b8e524c668629527d3e954e8819a5162c Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 17:12:50 -0300 Subject: [PATCH 100/152] docs: add CHANGELOG file --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ecf5eae..32abee8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - +### Changed + +- + ### Removed - Remove deprecated class AbstractHandler From 506731b2f61d6445c56f0f8df2540a5aa7193d78 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 11:50:50 -0300 Subject: [PATCH 101/152] chore: rename parameter from 'producer' to 'configOptions' This would be a breaking change, so it's better keep as 'configOptions' by now. --- src/TopicHandler/Producer/AbstractProducer.php | 4 ++-- tests/Integration/ConsumerTest.php | 2 +- tests/Integration/ProducerTest.php | 4 ++-- tests/Integration/ProducerWithAvroTest.php | 4 ++-- tests/Integration/ProducerWithConfigOptionsTest.php | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/TopicHandler/Producer/AbstractProducer.php b/src/TopicHandler/Producer/AbstractProducer.php index 946db3ce..694aedd0 100644 --- a/src/TopicHandler/Producer/AbstractProducer.php +++ b/src/TopicHandler/Producer/AbstractProducer.php @@ -20,11 +20,11 @@ class AbstractProducer implements HandlerInterface */ protected $producer; - public function __construct($record, Producer $producer, string $key = null) + public function __construct($record, Producer $configOptions, string $key = null) { $this->record = $record; $this->key = $key; - $this->producer = $producer; + $this->producer = $configOptions; } public function getConfigOptions(): Producer diff --git a/tests/Integration/ConsumerTest.php b/tests/Integration/ConsumerTest.php index 7088e46b..1d4397e8 100644 --- a/tests/Integration/ConsumerTest.php +++ b/tests/Integration/ConsumerTest.php @@ -51,7 +51,7 @@ public function testItShouldSetup(): void MessageProducerWithConfigOptions::class, [ 'record' => ['id' => 'MESSAGE_ID'], - 'producer' => $producerConfigOptions, + 'configOptions' => $producerConfigOptions, 'key' => 1, ] ); diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 5a456a71..73852c4c 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -153,7 +153,7 @@ private function haveSomeRandomMessagesProduced(): void $producerConfigOptions = $this->createProducerConfigOptions($this->topicId); $producer = app(MessageProducer::class, [ 'record' => $this->highLevelMessage, - 'producer' => $producerConfigOptions, + 'configOptions' => $producerConfigOptions, 'key' => 'recordId123', ]); @@ -166,7 +166,7 @@ private function produceRecordMessage(string $record): string $producerConfigOptions = $this->createProducerConfigOptions('low_level'); $producer = app(MessageProducer::class, [ 'record' => $record, - 'producer' => $producerConfigOptions, + 'configOptions' => $producerConfigOptions, 'key' => 'recordId123', ]); diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index 014100ef..b0c801ad 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -115,11 +115,11 @@ private function haveSomeRandomMessagesProduced(): void $saleOrderProducer = app(MessageProducer::class, [ 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], - 'producer' => $producerConfigOptionsSale, + 'configOptions' => $producerConfigOptionsSale, ]); $productProducer = app(MessageProducer::class, [ 'record' => ['productId' => 'PRODUCT_ID'], - 'producer' => $producerConfigOptionsProduct, + 'configOptions' => $producerConfigOptionsProduct, ]); $saleOrderSchemaResponse = '{ diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index f4f58b94..4e256a42 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -88,7 +88,7 @@ private function haveSomeRandomMessageProduced(): void MessageProducerWithConfigOptions::class, [ 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], - 'producer' => $this->producerConfigOptions, + 'configOptions' => $this->producerConfigOptions, 'key' => 1, ] ); From ed2ec847cdf01512cf037529ce8641789c49d8a1 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 14:44:51 -0300 Subject: [PATCH 102/152] chore: remove annotation --- src/Facades/Metamorphosis.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Facades/Metamorphosis.php b/src/Facades/Metamorphosis.php index c009a5f5..b3c934ca 100644 --- a/src/Facades/Metamorphosis.php +++ b/src/Facades/Metamorphosis.php @@ -3,11 +3,7 @@ namespace Metamorphosis\Facades; use Illuminate\Support\Facades\Facade; -use Metamorphosis\TopicHandler\Producer\HandlerInterface; -/** - * @method static void produce(HandlerInterface $producerHandler) - */ class Metamorphosis extends Facade { protected static function getFacadeAccessor() From 52c5903244e84c766e24c7137ed9ac39ae8bd887 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 20:42:10 -0300 Subject: [PATCH 103/152] chore: update middleware resolution on consumerCommand --- src/Console/ConsumerCommand.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index 70f151e0..bd4272e1 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -39,6 +39,12 @@ class ConsumerCommand extends BaseCommand public function handle(Config $config) { $consumer = $config->make($this->option(), $this->argument()); + + $middlewares = $consumer->getMiddlewares(); + foreach ($middlewares as &$middleware) { + $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $consumer]) : $middleware; + } + $this->writeStartingConsumer($consumer); $manager = Factory::make($consumer); From c2b2c085e7fbf249b38ca8010711b06ed4188442 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 21:43:32 -0300 Subject: [PATCH 104/152] chore: remove ManageTest --- tests/Unit/ManagerTest.php | 66 -------------------------------------- 1 file changed, 66 deletions(-) diff --git a/tests/Unit/ManagerTest.php b/tests/Unit/ManagerTest.php index 846db3f5..e69de29b 100644 --- a/tests/Unit/ManagerTest.php +++ b/tests/Unit/ManagerTest.php @@ -1,66 +0,0 @@ -set([ - 'topic' => 'products', - 'middlewares' => [ - Log::class, - app(JsonDecode::class), - ], - 'other' => 'config', - ]); - - // Actions - $other = $manager->get('other'); - $middlewares = $manager->middlewares(); - - // Assertions - $this->assertSame('config', $other); - $this->assertInstanceOf(Log::class, $middlewares[0]); - $this->assertInstanceOf(JsonDecode::class, $middlewares[1]); - } - - public function testShouldRemoveOldMiddlewareBeforeAddOthers(): void - { - // Set - $manager = new ConsumerConfigManager(); - $firstConfig = [ - 'topic' => 'products', - 'middlewares' => [ - Log::class, - app(JsonDecode::class), - ], - 'other' => 'config', - ]; - $secondConfig = [ - 'topic' => 'products', - 'middlewares' => [ - JsonDecode::class, - ], - 'other' => 'config', - ]; - - // Actions - $manager->set($firstConfig); - $manager->set($secondConfig); - $other = $manager->get('other'); - $middlewares = $manager->middlewares(); - - // Assertions - $this->assertSame('config', $other); - $this->assertInstanceOf(JsonDecode::class, $middlewares[0]); - $this->assertCount(1, $middlewares); - } -} From d2b71993eca2ca43c8a77a863fe1d5cd33e045ae Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 21:47:10 -0300 Subject: [PATCH 105/152] chore: remove occurrences of ConsumerConfigManager --- src/Consumer.php | 4 +--- tests/Unit/Connectors/Consumer/ManagerTest.php | 2 -- tests/Unit/Consumers/LowLevelTest.php | 3 --- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Consumer.php b/src/Consumer.php index 1d35a0c1..367556fa 100644 --- a/src/Consumer.php +++ b/src/Consumer.php @@ -15,10 +15,8 @@ class Consumer private Dispatcher $dispatcher; - public function __construct(ConsumerConfigManager $configManager, ConsumerConfigOptions $configOptions) + public function __construct(ConsumerConfigOptions $configOptions) { - $configManager->set($configOptions->toArray()); - $this->consumer = Factory::getConsumer(true, $configOptions); $this->dispatcher = new Dispatcher($configOptions->getMiddlewares()); } diff --git a/tests/Unit/Connectors/Consumer/ManagerTest.php b/tests/Unit/Connectors/Consumer/ManagerTest.php index d69d72eb..230513b2 100644 --- a/tests/Unit/Connectors/Consumer/ManagerTest.php +++ b/tests/Unit/Connectors/Consumer/ManagerTest.php @@ -4,7 +4,6 @@ use Exception; use Metamorphosis\Connectors\Consumer\Manager; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Exceptions\ResponseTimeoutException; use Metamorphosis\Exceptions\ResponseWarningException; @@ -14,7 +13,6 @@ use Mockery as m; use RdKafka\Message as KafkaMessage; use Tests\LaravelTestCase; -use Tests\Unit\Dummies\ConsumerHandlerDummy; class ManagerTest extends LaravelTestCase { diff --git a/tests/Unit/Consumers/LowLevelTest.php b/tests/Unit/Consumers/LowLevelTest.php index cc8bb988..b39ac2aa 100644 --- a/tests/Unit/Consumers/LowLevelTest.php +++ b/tests/Unit/Consumers/LowLevelTest.php @@ -2,7 +2,6 @@ namespace Tests\Unit\Consumers; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\LowLevel; use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; @@ -20,8 +19,6 @@ public function testItShouldConsume(): void // Set $timeout = 2; $partition = 3; - $configManager = new ConsumerConfigManager(); - $configManager->set(compact('timeout', 'partition')); $brokerOptions = new Broker('kafka:9092', new None()); $consumerConfigOptions = new ConsumerConfigOptions( From b5eeb205aecfc7822d5d2509ebfd1e57cd6fc9ca Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 21:47:56 -0300 Subject: [PATCH 106/152] chore: remove TODO comment --- src/Connectors/Consumer/Factory.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index 0da13cac..c82c4ad2 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -13,9 +13,6 @@ * means that we will use the low level approach. */ -/** - * TODO Rename this class to ConsumerFactory, it will improve semantics - */ class Factory { public static function make(ConsumerConfigOptions $configOptions): Manager From 7ad9a61cb05289c2b064ad40b8d752c0e0ee393e Mon Sep 17 00:00:00 2001 From: hcdias Date: Fri, 25 Mar 2022 10:20:06 -0300 Subject: [PATCH 107/152] chore: remove ProducerConfigManager occurences --- src/ConsumerConfigManager.php | 54 ------------------- .../Unit/Middlewares/Handler/ProducerTest.php | 1 - tests/Unit/Producer/PollTest.php | 12 ----- 3 files changed, 67 deletions(-) delete mode 100644 src/ConsumerConfigManager.php diff --git a/src/ConsumerConfigManager.php b/src/ConsumerConfigManager.php deleted file mode 100644 index 88207c97..00000000 --- a/src/ConsumerConfigManager.php +++ /dev/null @@ -1,54 +0,0 @@ -setConfig($config, $consumerHandler); - $this->setCommandConfig($commandConfig); - - $middlewares = $this->get('middlewares', []); - $this->middlewares = []; - $this->remove('middlewares'); - - foreach ($middlewares as $middleware) { - $this->middlewares[] = is_string($middleware) ? app($middleware) : $middleware; - } - - if (!$consumerHandler) { - return; - } - - $this->middlewares[] = new ConsumerMiddleware($consumerHandler); - } - - private function setCommandConfig(?array $commandConfig): void - { - if (!$commandConfig) { - return; - } - - $this->setting = array_merge($this->setting, $commandConfig); - } - - private function setConfig(array $config, ?AbstractHandler $handler): void - { - if (!$handler || !$overrideConfig = $handler->getConfigOptions()) { - $this->setting = $config; - - return; - } - - $this->setting = $overrideConfig->toArray(); - } -} diff --git a/tests/Unit/Middlewares/Handler/ProducerTest.php b/tests/Unit/Middlewares/Handler/ProducerTest.php index 3a7bbc70..df2750e6 100644 --- a/tests/Unit/Middlewares/Handler/ProducerTest.php +++ b/tests/Unit/Middlewares/Handler/ProducerTest.php @@ -5,7 +5,6 @@ use Closure; use Metamorphosis\Middlewares\Handler\Producer; use Metamorphosis\Producer\Poll; -use Metamorphosis\ProducerConfigManager; use Metamorphosis\Record\ProducerRecord; use Mockery as m; use RdKafka\ProducerTopic as KafkaTopicProducer; diff --git a/tests/Unit/Producer/PollTest.php b/tests/Unit/Producer/PollTest.php index 3ef21c96..0576cd6c 100644 --- a/tests/Unit/Producer/PollTest.php +++ b/tests/Unit/Producer/PollTest.php @@ -3,7 +3,6 @@ namespace Tests\Unit\Producer; use Metamorphosis\Producer\Poll; -use Metamorphosis\ProducerConfigManager; use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; use Metamorphosis\TopicHandler\ConfigOptions\Broker; @@ -48,17 +47,6 @@ public function testItShouldHandleMessageWithoutAcknowledgment(): void public function testShouldThrowExceptionWhenFlushFailed(): void { // Set - $configManager = new ProducerConfigManager(); - $configManager->set([ - 'topic_id' => 'topic_name', - 'timeout' => 1000, - 'is_async' => false, - 'max_poll_records' => 500, - 'flush_attempts' => 3, - 'required_acknowledgment' => true, - 'partition' => 0, - ]); - $broker = new Broker('kafka:9092', new None()); $producerConfigOptions = new ProducerConfigOptions( 'topic_name', From 430148df60c16b0a323993ec98018339481f308a Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 19 Apr 2022 16:10:10 -0300 Subject: [PATCH 108/152] chore: remove unecessary interface --- src/Avro/AvroClientInterface.php | 6 ------ src/Avro/CachedSchemaRegistryClientInterface.php | 7 ------- 2 files changed, 13 deletions(-) delete mode 100644 src/Avro/AvroClientInterface.php delete mode 100644 src/Avro/CachedSchemaRegistryClientInterface.php diff --git a/src/Avro/AvroClientInterface.php b/src/Avro/AvroClientInterface.php deleted file mode 100644 index d7617023..00000000 --- a/src/Avro/AvroClientInterface.php +++ /dev/null @@ -1,6 +0,0 @@ - Date: Tue, 19 Apr 2022 16:12:26 -0300 Subject: [PATCH 109/152] chore: remove unecessary code block --- src/Console/ConsumerCommand.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index bd4272e1..5c233db4 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -40,11 +40,6 @@ public function handle(Config $config) { $consumer = $config->make($this->option(), $this->argument()); - $middlewares = $consumer->getMiddlewares(); - foreach ($middlewares as &$middleware) { - $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $consumer]) : $middleware; - } - $this->writeStartingConsumer($consumer); $manager = Factory::make($consumer); From f1598e2d57bfdd111ad8644e122168bf1bec2be3 Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 19 Apr 2022 16:13:45 -0300 Subject: [PATCH 110/152] chore: define constant visibility --- src/Avro/ClientFactory.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Avro/ClientFactory.php b/src/Avro/ClientFactory.php index b30d0c1b..388a8fed 100644 --- a/src/Avro/ClientFactory.php +++ b/src/Avro/ClientFactory.php @@ -7,7 +7,7 @@ class ClientFactory { - const REQUEST_TIMEOUT = 2000; + protected const REQUEST_TIMEOUT = 2000; public function make(AvroSchema $avroSchema): CachedSchemaRegistryClient { From f7bc97024204919d9e64cb6485f44f43c42c01a3 Mon Sep 17 00:00:00 2001 From: hcdias Date: Fri, 1 Apr 2022 15:59:18 -0300 Subject: [PATCH 111/152] chore: change config file format --- config/kafka.php | 172 ++++-------------- config/service.php | 28 +++ src/Connectors/AbstractConfig.php | 24 +-- src/Connectors/Consumer/Config.php | 56 +----- src/Connectors/Producer/Config.php | 12 +- src/MetamorphosisServiceProvider.php | 6 +- .../Factories/ConsumerFactory.php | 6 +- 7 files changed, 86 insertions(+), 218 deletions(-) create mode 100644 config/service.php diff --git a/config/kafka.php b/config/kafka.php index 00912858..9e74693b 100644 --- a/config/kafka.php +++ b/config/kafka.php @@ -1,128 +1,46 @@ [ - 'default' => [ - 'url' => '', - // Disable SSL verification on schema request. - 'ssl_verify' => true, - // This option will be put directly into a Guzzle http request - // Use this to do authorizations or send any headers you want. - // Here is a example of basic authentication on AVRO schema. - 'request_options' => [ - 'headers' => [ - 'Authorization' => [ - 'Basic ' . base64_encode( - env('AVRO_SCHEMA_USERNAME') - . ':' - . env('AVRO_SCHEMA_PASSWORD') - ), - ], - ], - ], - ], - ], - - /* - |-------------------------------------------------------------------------- - | Brokers - |-------------------------------------------------------------------------- - | - | Here you may specify the connections details for each broker configured - | on topic's broker key. - | - */ - - 'brokers' => [ - 'default' => [ - 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), - - // If your broker doest not have authentication, you can - // remove this configuration, or set as empty. - // The Authentication types may be "ssl" or "none" - 'auth' => [ - 'type' => 'ssl', // ssl and none - 'ca' => storage_path('ca.pem'), - 'certificate' => storage_path('kafka.cert'), - 'key' => storage_path('kafka.key'), - ], - ], - ], - 'topics' => [ - // This is your topic "keyword" where you will put all configurations needed - // on this specific topic. 'default' => [ - // The topic id is where you want to send or consume - // your messages from kafka. 'topic_id' => 'kafka-test', - - // Here you may point the key of the broker configured above. - 'broker' => 'default', - - // Configurations specific for consumer 'consumer' => [ - // You may define more than one consumer group per topic. - // If there is just one defined, it will be used by default, - // otherwise, you may pass which consumer group should be used - // when using the consumer command. - 'consumer_groups' => [ - 'test-consumer-group' => [ - - // Action to take when there is no initial - // offset in offset store or the desired offset is out of range. - // This config will be passed to 'auto.offset.reset'. - // The valid options are: smallest, earliest, beginning, largest, latest, end, error. - 'offset_reset' => 'earliest', - - // The offset at which to start consumption. This only applies if partition is set. - // You can use a positive integer or any of the constants: RD_KAFKA_OFFSET_BEGINNING, - // RD_KAFKA_OFFSET_END, RD_KAFKA_OFFSET_STORED. - 'offset' => 0, - - // The partition to consume. It can be null, - // if you don't wish do specify one. - 'partition' => 0, - - // A consumer class that implements ConsumerTopicHandler - 'handler' => '\App\Kafka\Consumers\ConsumerExample', - - // A Timeout to listen to a message. That means: how much - // time we need to wait until receiving a message? - 'timeout' => 20000, - - // A max interval for consumer to make poll calls. That means: how much - // time we need to wait for poll calls until consider the consumer has inactive. - 'max_poll_interval_ms' => 300000, - - // Once you've enabled this, the Kafka consumer will commit the - // offset of the last message received in response to its poll() call - 'auto_commit' => true, - - // If commit_async is false process block until offsets are committed or the commit fails. - // Only works when auto_commit is false - 'commit_async' => false, - - // An array of middlewares applied only for this consumer_group - 'middlewares' => [], - ], - ], + 'consumer_group' => 'test-consumer-group', + // Action to take when there is no initial + // offset in offset store or the desired offset is out of range. + // This config will be passed to 'auto.offset.reset'. + // The valid options are: smallest, earliest, beginning, largest, latest, end, error. + 'offset_reset' => 'earliest', + + // The offset at which to start consumption. This only applies if partition is set. + // You can use a positive integer or any of the constants: RD_KAFKA_OFFSET_BEGINNING, + // RD_KAFKA_OFFSET_END, RD_KAFKA_OFFSET_STORED. + 'offset' => 0, + + // The partition to consume. It can be null, + // if you don't wish do specify one. + 'partition' => 0, + + // A consumer class that implements ConsumerTopicHandler + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + + // A Timeout to listen to a message. That means: how much + // time we need to wait until receiving a message? + 'timeout' => 20000, + + // Once you've enabled this, the Kafka consumer will commit the + // offset of the last message received in response to its poll() call + 'auto_commit' => true, + + // If commit_async is false process block until offsets are committed or the commit fails. + // Only works when auto_commit is false + 'commit_async' => false, + + // An array of middlewares applied only for this consumer_group + 'middlewares' => [], ], - // Configurations specific for producer 'producer' => [ - // Sets to true if you want to know if a message was successfully posted. 'required_acknowledgment' => true, @@ -153,28 +71,4 @@ ], ], ], - - /* - |-------------------------------------------------------------------------- - | Global Middlewares - |-------------------------------------------------------------------------- - | - | Here you may specify the global middlewares that will be applied for every - | consumed topic. Middlewares work between the received data from broker and - | before being passed into consumers. - | Available middlewares: log, avro-decode - | - */ - - 'middlewares' => [ - 'consumer' => [ - \Metamorphosis\Middlewares\Log::class, - ], - 'producer' => [ - \Metamorphosis\Middlewares\Log::class, - ], - 'global' => [ - \Metamorphosis\Middlewares\Log::class, - ], - ], ]; diff --git a/config/service.php b/config/service.php new file mode 100644 index 00000000..82cb1299 --- /dev/null +++ b/config/service.php @@ -0,0 +1,28 @@ + [ + 'url' => '', + 'request_options' => [ + 'headers' => [ + 'Authorization' => [ + 'Basic '.base64_encode( + env('AVRO_SCHEMA_USERNAME').':'.env('AVRO_SCHEMA_PASSWORD') + ), + ], + ], + ], + 'ssl_verify' => true, + 'username' => 'USERNAME', + 'password' => 'PASSWORD', + ], + 'broker' => [ + 'connections' => 'kafka:9092', + 'auth' => [ + 'type' => 'ssl', // ssl and none + 'ca' => storage_path('ca.pem'), + 'certificate' => storage_path('kafka.cert'), + 'key' => storage_path('kafka.key'), + ], + ], +]; diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index 9942dc2e..fa8b8ac1 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -7,20 +7,17 @@ abstract class AbstractConfig { - /** - * @var mixed[] - */ - protected array $rules = []; - - protected function getBrokerConfig(string $configName, string $brokerId): array + protected function getBrokerConfig(string $servicesFile): array { - if (!$brokerConfig = config("{$configName}.brokers.{$brokerId}")) { - throw new ConfigurationException( - "Broker '{$brokerId}' configuration not found" - ); + if (!$brokerConfig = config($servicesFile.'.broker')) { + throw new ConfigurationException("Broker configuration not found on '{$servicesFile}'"); } + return $brokerConfig; + } - return (array) $brokerConfig; + protected function getSchemaConfig(string $servicesFile): array + { + return config($servicesFile.'.avro_schema', []); } protected function validate(array $config): void @@ -31,9 +28,4 @@ protected function validate(array $config): void throw new ConfigurationException($validator->errors()->toJson()); } } - - protected function getSchemaConfig(string $configName, string $topicId): array - { - return config($configName . '.avro_schemas.' . $topicId, []); - } } diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 1b0b5c6e..aabcfd8b 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -54,37 +54,26 @@ public function makeWithConfigOptions(string $handlerClass, ?int $times = null): public function make(array $options, array $arguments): Consumer { $configName = $options['config_name'] ?? 'kafka'; - $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); - $consumerGroupId = $this->getConsumerGroup($topicConfig, $arguments['consumer_group']); - $consumerConfig = $this->getConsumerConfig($topicConfig, $options, $consumerGroupId); - $brokerConfig = $this->getBrokerConfig($configName, $topicConfig['broker']); - $schemaConfig = $this->getSchemaConfig($configName, $arguments['topic']); - $override = array_merge($this->filterValues($options), $this->filterValues($arguments)); - $config = array_merge( - $topicConfig, - $brokerConfig, - $consumerConfig, - $schemaConfig - ); + $service = $options['service'] ?? 'service'; - $this->validate(array_merge($config, $override)); + $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); + $brokerConfig = $this->getBrokerConfig($service); + $schemaConfig = $this->getSchemaConfig($service); - if (isset($topicConfig['consumer']['consumer_groups'][$consumerGroupId])) { + if (isset($topicConfig['consumer'])) { if (isset($options['partition'])) { - $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['partition'] = $options['partition']; + $topicConfig['consumer']['partition'] = $options['partition']; } if (isset($options['offset'])) { - $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['offset'] = $options['offset']; + $topicConfig['consumer']['offset'] = $options['offset']; } if (isset($options['timeout'])) { - $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['timeout'] = $options['timeout']; + $topicConfig['consumer']['timeout'] = $options['timeout']; } } - $topicConfig['consumer_group'] = $consumerGroupId; - return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); } @@ -106,35 +95,6 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } - private function getConsumerConfig(array $topicConfig, array $options, string $consumerGroupId): array - { - $consumerConfig = $topicConfig['consumer']['consumer_groups'][$consumerGroupId] ?? null; - if (!$consumerConfig) { - throw new ConfigurationException("Consumer group '{$consumerGroupId}' not found"); - } - - $consumerConfig['consumer_group'] = $consumerGroupId; - - return $consumerConfig; - } - - private function getConsumerGroup(array $topicConfig, ?string $consumerGroupId): string - { - if (!$consumerGroupId && 1 === count($topicConfig['consumer']['consumer_groups'])) { - $consumerGroupId = current(array_keys($topicConfig['consumer']['consumer_groups'])); - } - - return $consumerGroupId ?? 'default'; - } - - private function getMiddlewares(string $configName, array $topicConfig): array - { - return array_merge( - config($configName . '.middlewares.consumer', []), - $topicConfig['consumer']['middlewares'] ?? [] - ); - } - /** * Sometimes that user may pass `--partition=0` as argument. * So if we just use array_filter here, this option will diff --git a/src/Connectors/Producer/Config.php b/src/Connectors/Producer/Config.php index 90a9cd1d..e3cf8e80 100644 --- a/src/Connectors/Producer/Config.php +++ b/src/Connectors/Producer/Config.php @@ -50,15 +50,9 @@ public function make(ProducerConfigOptions $configOptions): AbstractConfigManage public function makeByTopic(string $topicId): AbstractConfigManager { $topicConfig = $this->getTopicConfig($topicId); - $topicConfig['middlewares'] = array_merge( - config('kafka.middlewares.producer', []), - $topicConfig['producer']['middlewares'] ?? [] - ); - $brokerConfig = $this->getBrokerConfig( - 'kafka', - $topicConfig['broker'] - ); - $schemaConfig = $this->getSchemaConfig('kafka', $topicId); + $topicConfig['middlewares'] = $topicConfig['producer']['middlewares'] ?? []; + $brokerConfig = $this->getBrokerConfig('service'); + $schemaConfig = $this->getSchemaConfig('service'); $config = array_merge($topicConfig, $brokerConfig, $schemaConfig); $this->validate($config); diff --git a/src/MetamorphosisServiceProvider.php b/src/MetamorphosisServiceProvider.php index 4938fbac..36dffdf5 100644 --- a/src/MetamorphosisServiceProvider.php +++ b/src/MetamorphosisServiceProvider.php @@ -14,10 +14,12 @@ class MetamorphosisServiceProvider extends ServiceProvider public function boot() { $this->publishes([ - __DIR__ . '/../config/kafka.php' => config_path('kafka.php'), + __DIR__.'/../config/kafka.php' => config_path('kafka.php'), + __DIR__.'/../config/service.php' => config_path('service.php'), ], 'config'); - $this->mergeConfigFrom(__DIR__ . '/../config/kafka.php', 'kafka'); + $this->mergeConfigFrom(__DIR__.'/../config/kafka.php', 'kafka'); + $this->mergeConfigFrom(__DIR__.'/../config/service.php', 'service'); } public function register() diff --git a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php index 7bd58c93..4dd62736 100644 --- a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php +++ b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php @@ -22,10 +22,8 @@ public static function make( private static function getConsumerGroupConfig(array $topicData): array { $topicData['topicId'] = $topicData['topic_id']; - $topicData['consumerGroup'] = $topicData['consumer_group']; - - $consumerGroup = $topicData['consumerGroup']; - $consumer = current($topicData['consumer'])[$consumerGroup]; + $consumer = $topicData['consumer']; + $topicData['consumerGroup'] = $consumer['consumer_group']; return array_merge_recursive($topicData, self::convertConfigAttributes($consumer)); } From cc0d3ec4a6e64c5b2eb7810ccbc9ffa1536a4691 Mon Sep 17 00:00:00 2001 From: hcdias Date: Fri, 1 Apr 2022 16:09:05 -0300 Subject: [PATCH 112/152] chore: adjust tests to match new config file format --- tests/Integration/ProducerTest.php | 23 +++---- tests/Unit/Connectors/Consumer/ConfigTest.php | 7 ++- .../Unit/Connectors/Consumer/FactoryTest.php | 52 ++++++++++++++++ tests/Unit/Connectors/Producer/ConfigTest.php | 2 - tests/Unit/Console/ConsumerCommandTest.php | 61 ++++++++----------- .../Factories/ConsumerFactoryTest.php | 22 +++---- 6 files changed, 102 insertions(+), 65 deletions(-) diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 73852c4c..94ab5626 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -68,19 +68,17 @@ public function testShouldRunAProducerAndReceiveMessagesWithALowLevelConsumer(): protected function withoutAuthentication(): void { - config(['kafka.brokers.default.auth' => []]); + config(['service.broker.auth' => []]); } protected function haveAConsumerHandlerConfigured(): void { - config( - ['kafka.topics.default.consumer.consumer_groups.test-consumer-group.handler' => MessageConsumer::class] - ); + config(['kafka.topics.default.consumer.handler' => MessageConsumer::class]); } protected function haveAConsumerPartitionConfigured(): void { - config(['kafka.topics.default.consumer.consumer_groups.test-consumer-group.partition' => -1]); + config(['kafka.topics.default.consumer.partition' => -1]); } protected function runTheConsumer(): void @@ -107,15 +105,12 @@ protected function haveALowLevelConsumerConfigured(): void 'topic_id' => 'low_level', 'broker' => 'default', 'consumer' => [ - 'consumer_groups' => [ - 'test-consumer-group' => [ - 'offset_reset' => 'earliest', - 'offset' => 0, - 'handler' => MessageConsumer::class, - 'timeout' => 20000, - 'middlewares' => [], - ], - ], + 'consumer_group' => 'test-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'handler' => MessageConsumer::class, + 'timeout' => 20000, + 'middlewares' => [], ], 'producer' => [ 'required_acknowledgment' => true, diff --git a/tests/Unit/Connectors/Consumer/ConfigTest.php b/tests/Unit/Connectors/Consumer/ConfigTest.php index 539b6ea0..ffeaf5cd 100644 --- a/tests/Unit/Connectors/Consumer/ConfigTest.php +++ b/tests/Unit/Connectors/Consumer/ConfigTest.php @@ -8,6 +8,7 @@ use Mockery as m; use Tests\LaravelTestCase; use Tests\Unit\Dummies\ConsumerHandlerDummy; +use TypeError; class ConfigTest extends LaravelTestCase { @@ -104,8 +105,10 @@ public function testShouldNotSetRuntimeConfigWhenOptionsIsInvalid(): void 'consumer_group' => 'default', ]; + // Expectations + $this->expectException(TypeError::class); + // Actions - $this->expectException(ConfigurationException::class); $configManager = $config->make($options, $arguments); // Assertions @@ -115,7 +118,7 @@ public function testShouldNotSetRuntimeConfigWhenOptionsIsInvalid(): void public function testShouldNotSetRuntimeConfigWhenKafkaConfigIsInvalid(): void { // Set - config(['kafka.brokers.default.connections' => null]); + config(['service.broker' => null]); $config = new Config(); $options = [ 'partition' => 0, diff --git a/tests/Unit/Connectors/Consumer/FactoryTest.php b/tests/Unit/Connectors/Consumer/FactoryTest.php index ae3dedf6..2ccb9cd9 100644 --- a/tests/Unit/Connectors/Consumer/FactoryTest.php +++ b/tests/Unit/Connectors/Consumer/FactoryTest.php @@ -50,6 +50,8 @@ protected function setUp(): void public function testItMakesManagerWithLowLevelConsumer(): void { // Set + $this->haveAConsumerWithPartitionConfigured(); + $config = new Config(); $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); $manager = Factory::make($configConsumer); @@ -61,6 +63,7 @@ public function testItMakesManagerWithLowLevelConsumer(): void public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid(): void { // Set + $this->haveAConsumerWithoutPartitionConfigured(); $config = new Config(); $configConsumer = $config->make(['timeout' => 61, 'partition' => -1], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); $manager = Factory::make($configConsumer); @@ -72,6 +75,7 @@ public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid() public function testItMakesHighLevelClass(): void { // Set + $this->haveAConsumerWithoutPartitionConfigured(); $config = new Config(); $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'without-partition']); $manager = Factory::make($configConsumer); @@ -79,4 +83,52 @@ public function testItMakesHighLevelClass(): void // Assertions $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); } + + private function haveAConsumerWithPartitionConfigured() + { + config([ + 'kafka' => [ + 'topics' => [ + 'topic_key' => [ + 'topic_id' => 'topic_name', + 'consumer' => [ + 'consumer_group' => 'with-partition', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => ConsumerHandlerDummy::class, + ], + ], + ], + ], + 'service' => [ + 'broker' => [ + 'connections' => 'kafka:123', + ], + ], + ]); + } + + private function haveAConsumerWithoutPartitionConfigured() + { + config([ + 'kafka' => [ + 'topics' => [ + 'topic_key' => [ + 'topic_id' => 'topic_name', + 'consumer' => [ + 'consumer_group' => 'without-partition', + 'offset_reset' => 'earliest', + 'handler' => ConsumerHandlerDummy::class, + ], + ], + ], + ], + 'service' => [ + 'broker' => [ + 'connections' => 'kafka:123', + ], + ], + ]); + } } diff --git a/tests/Unit/Connectors/Producer/ConfigTest.php b/tests/Unit/Connectors/Producer/ConfigTest.php index 82adae08..d60567e1 100644 --- a/tests/Unit/Connectors/Producer/ConfigTest.php +++ b/tests/Unit/Connectors/Producer/ConfigTest.php @@ -24,7 +24,6 @@ public function testShouldValidateProducerConfig(): void 'max_poll_records' => 500, 'flush_attempts' => 10, 'partition' => -1, - 'broker' => 'default', 'topic' => 'default', 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), 'auth' => [ @@ -70,7 +69,6 @@ public function testShouldNotOverrideDefaultParametersWhenConfigIsSet(): void 'required_acknowledgment' => true, 'max_poll_records' => 3000, 'flush_attempts' => 10, - 'broker' => 'default', 'topic' => 'default', 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), 'auth' => [ diff --git a/tests/Unit/Console/ConsumerCommandTest.php b/tests/Unit/Console/ConsumerCommandTest.php index 26132745..5a5a8fcf 100644 --- a/tests/Unit/Console/ConsumerCommandTest.php +++ b/tests/Unit/Console/ConsumerCommandTest.php @@ -10,6 +10,33 @@ class ConsumerCommandTest extends LaravelTestCase { + protected function setUp(): void + { + parent::setUp(); + + config([ + 'kafka' => [ + 'topics' => [ + 'topic_key' => [ + 'topic_id' => 'topic_name', + 'consumer' => [ + 'consumer_group' => 'default', + 'offset_reset' => 'earliest', + 'handler' => ConsumerHandlerDummy::class, + 'timeout' => 123, + ], + ], + ], + ], + 'service' => [ + 'broker' => [ + 'connections' => 'test_kafka:6680', + 'auth' => [], + ], + ], + ]); + } + public function testItCallsCommandWithInvalidTopic(): void { // Set @@ -131,38 +158,4 @@ public function testItOverridesBrokerConnectionWhenCallingCommand(): void $this->artisan($command, $parameters); } - - protected function setUp(): void - { - parent::setUp(); - - config([ - 'kafka' => [ - 'brokers' => [ - 'default' => [ - 'connections' => env( - 'KAFKA_BROKER_CONNECTIONS', - 'kafka:9092' - ), - 'auth' => [], - ], - ], - 'topics' => [ - 'topic_key' => [ - 'topic_id' => 'topic_name', - 'broker' => 'default', - 'consumer' => [ - 'consumer_groups' => [ - 'default' => [ - 'offset_reset' => 'earliest', - 'handler' => ConsumerHandlerDummy::class, - 'timeout' => 123, - ], - ], - ], - ], - ], - ], - ]); - } } diff --git a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php index f5c07f80..02aa3368 100644 --- a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php +++ b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php @@ -33,20 +33,16 @@ public function testShouldMakeConfigOptionWithAvroSchema(): void ]; $topicData = [ 'topic_id' => 'kafka-test', - 'consumer_group' => 'test-consumer-group', 'consumer' => [ - 'consumer_groups' => [ - 'test-consumer-group' => [ - 'middlewares' => [], - 'auto_commit' => true, - 'commit_async' => true, - 'offset_reset' => 'earliest', - 'handler' => '\App\Kafka\Consumers\ConsumerExample', - 'partition' => 0, - 'offset' => 0, - 'timeout' => 20000, - ], - ], + 'consumer_group' => 'test-consumer-group', + 'middlewares' => [], + 'auto_commit' => true, + 'commit_async' => true, + 'offset_reset' => 'earliest', + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'partition' => 0, + 'offset' => 0, + 'timeout' => 20000, ], ]; $expected = [ From 4c1e0a432f6c04aa42e6e563303353c5ee6ac454 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 15:51:49 -0300 Subject: [PATCH 113/152] chore: update docs --- CHANGELOG.md | 25 ++----- docs/advanced.md | 35 ++++++--- docs/quick-usage.md | 179 +++++++++++++++++++++++--------------------- 3 files changed, 124 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32abee8f..009e3b39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,36 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- - -### Fixed - -- - -### Changed - -- - -### Removed -- Remove deprecated class AbstractHandler - -## [4.1.0] - 2022-03-23 - -### Added +- Added file `config/service.php` to configure broker and authentication - Added AvroSchemaMixedEncoderTest - Added AvroSchemaDecoderTest - Added ProducerWithConfigOptionsTest - Added ConfigOptionsCommand to run commands with ConfigOptions class - Added pt_BR contributing section - Added setup-dev script on composer -- Added grumphp commit validation +- Added grumphp commit validation + +### Fixed -### Fixed - Fixed parameters and options override on Consumer\Config class - Update instructions on contribute section - Update project install section ### Changed + - Updated class from ConfigManager to ConfigOptions on unit tests - Updated class from ConfigManager to ConfigOptions where any config request was made - Consumer and Producer middlewares resolution + +### Removed diff --git a/docs/advanced.md b/docs/advanced.md index f290b06c..acca8234 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -275,29 +275,42 @@ stdout_logfile=/var/log/default/kafka-consumer-price-update.log Although you can run this simple command, it provides some options you can pass to make it more flexible to your needs. -- `--broker=` - - Sometimes, you may want to change which broker the consumer should connect to (maybe for testing/debug purposes). - For that, you just nedd to call the `--broker` option with another broker connection key already set in the `config/kafka.php` file. - - `$ php artisan kafka:consume price-update --broker='some-other-broker'` - `--offset=` And if you need to start the consumption of a topic in a specific offset (it can be useful for debug purposes) you can pass the `--offset=` option, but for this, it will be required to specify the partition too. - `$ php artisan kafka:consume price-update --partition=2 --offset=34` + + $ php artisan kafka:consume price-update --partition=2 --offset=34 + - `--partition=` - If you wish do specify in which partition the consumer must be attached, you can set the option `--partition=`. + Set in which partition the consumer must be attached. + + + $ php artisan kafka:consume price-update --partition=2 --offset=34 - `$ php artisan kafka:consume price-update --partition=2 --offset=34` - `--timeout=` - You can specify what would be the timeout for the consumer, by using the `--timeout=` option, the time is in milliseconds. + Set the timeout for the consumer in milliseconds. + + + $ php artisan kafka:consume price-update --timeout=23000 + + +- `--config_name=` + + Specify from what file topics configuration should be read. + + + $ php artisan kafka:consume topic-name --config_name=config.file + + +- `--service_name=` - `$ php artisan kafka:consume price-update --timeout=23000` + Specify from what file services configurations should be read. + `$ php artisan kafka:consume price-update --service_name=config.file` diff --git a/docs/quick-usage.md b/docs/quick-usage.md index 3d003da4..7fd95905 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -8,76 +8,104 @@ - [Produce Message](#produce-message) -### Config file: `config/kafka.php` - -The config file holds all information about brokers, topics, consumer groups and middlewares. - -To quickly start using, we can focus in two sections: -- Brokers - - An array of brokers, with connection and authentication configurations: - - - `connections`: *required*. can be a `string` with multiple connections separated by comma or an `array` of connections (as `string`) - - - `auth`: *optional*. out of the box, the package can connect with SSL Authentication only or without any authentication - - ```php - 'brokers' => [ - 'price_brokers' => [ - 'connections' => 'localhost:8091,localhost:8092', - 'auth' => [ - 'type' => 'ssl', - 'ca' => storage_path('ca.pem'), - 'certificate' => storage_path('kafka.cert'), - 'key' => storage_path('kafka.key'), - ], - ], - 'stock_brokers' => [ - 'connections' => ['localhost:8091', 'localhost:8092'], - 'auth' => [], // can be an empty array or even don't have this key in the broker config - ], - ], - ``` - -- Topics - - An array of topics configuration, such as the topic name, which broker connection should use, consumer groups and middlewares. - - Here we can specify the group consumers, each topic can have multiple groups, - and each group holds the configuration for which consumer, offset_reset (for setting initial offset) and middleware it must use. - - ```php - 'topics' => [ - 'price_update' => [ - 'topic' => 'products.price.update', - 'broker' => 'price_brokers', - 'consumer_groups' => [ - 'default' => [ - 'offset_reset' => 'smallest', - 'handler' => '\App\Kafka\Consumers\PriceUpdateConsumer', - ], - ], - ], - ], - ``` +### Configure using files + +To get started using configuration files, at least two files are needed. A file to keep the topics +configuration and a file to keep the broker and schema configuration. In this example, we will use the files `config/kafka.php` and `config/service.php`. + +### File `config/kafka.php`: + +This file keeps configurations about topics, consumers and producers. +It should return an array of topics containing the topic name, topic_id, consumer, producer and the settings for each one of them: + + +```php + [ + 'this_is_your_topic_name' => [ + 'topic_id' => "this_is_your_topic_id", + 'consumer' => [ + 'consumer_group' => 'your-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'timeout' => 20000, + 'auto_commit' => true, + 'commit_async' => false, + 'middlewares' => [], + ], + + 'producer' => [ + 'required_acknowledgment' => true, + 'is_async' => true, + 'max_poll_records' => 500, + 'flush_attempts' => 10, + 'middlewares' => [], + 'timeout' => 10000, + 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1, + ], + ] + ], +]; +``` + +### File `config/service.php` + +This file keeps configurations about **broker** and **schema** utilized. + + +```php + [ + 'url' => '', + 'request_options' => [ + 'headers' => [ + 'Authorization' => [ + 'Basic ' . base64_encode( + env('AVRO_SCHEMA_USERNAME').':'.env('AVRO_SCHEMA_PASSWORD') + ), + ], + ], + ], + + 'ssl_verify' => true, + 'username' => 'USERNAME', + 'password' => 'PASSWORD', + ], + + 'broker' => [ + 'connections' => 'kafka:9092', + 'auth' => [ + 'type' => 'ssl', + 'ca' => storage_path('ca.pem'), + 'certificate' => storage_path('kafka.cert'), + 'key' => storage_path('kafka.key'), + ], + ], +]; +``` + ### Consumer -After setting up the required configs, you need to create the consumer, which will handle all records received -from the topic specified in the config. +After setting up the required configuration, you must create a consumer to handle records received +from the specified topic in your configuration. -#### Creating Consumer +#### Creating a Consumer -Creating the consumer is easy as running the following command: +To create a consumer run the following command: ```bash $ php artisan make:kafka-consumer PriceUpdateConsumer ``` -This will create a KafkaConsumer class inside the application, on the app/Kafka/Consumers/ directory - -There, you'll have a handler method, which will send all records from the topic to the Consumer, -also, methods will be available for handling exceptions +This will create a KafkaConsumer class on the app/Kafka/Consumers/ directory with the following +content: ```php use App\Kafka\Consumers\PriceUpdateConsumer; @@ -109,20 +137,19 @@ class PriceUpdateConsumer extends AbstractHandler ``` -#### Running consumer +#### Running the consumer -Now you just need to start consuming the topic. +To start consuming the topic, the simplest way to see it working is by running the kafka:consume command along with the topic name, topic configuration file and service configuration file: -The simplest way to see it working is by running the kafka:consume command along with the topic name -declared in the topics config key: ```bash -$ php artisan kafka:consume price-update +$ php artisan kafka:consume this_is_your_topic_name --config_name=config.file --service_name=service.file ``` This command will run in a `while true`, that means, it will never stop running. But, errors can happen, so we strongly advice you to run this command along with [supervisor](http://supervisord.org/running.html), like this example below: + ```bash [program:kafka-consumer-price-update] process_name=%(program_name)s_%(process_num)02d @@ -137,30 +164,10 @@ stdout_logfile=/var/log/default/kafka-consumer-price-update.log That's it. For more information about usage, middlewares, broker authentication, consumer groups and other advanced topics, please have a look at our [Advanced Usage Guide](advanced.md). - -### Producer - -Producer also required configs, which will produce all records using parameters specified in the config. - -```php - 'brokers' => [ - 'local-dev' => [ - 'connections' => 'kafka:9092', - ], - ], - 'topics' => [ - 'product-updated' => [ - 'topic_id' => 'product_updated', - 'broker' => 'local-dev', - ], - ], -``` ### Produce Message -Creating Producer handler. - -The Producer must extends AbstractHandler class and can be empty. +To create a producer handler, create a class that extends `Metamorphosis\TopicHandler\Producer\AbstractHandler` class: ```php Date: Mon, 4 Apr 2022 15:53:44 -0300 Subject: [PATCH 114/152] chore: update console parameters Added a new option to specify the service file --- src/Connectors/Consumer/Config.php | 2 +- src/Console/ConsumerCommand.php | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index aabcfd8b..81ba46a6 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -54,7 +54,7 @@ public function makeWithConfigOptions(string $handlerClass, ?int $times = null): public function make(array $options, array $arguments): Consumer { $configName = $options['config_name'] ?? 'kafka'; - $service = $options['service'] ?? 'service'; + $service = $options['service_name'] ?? 'service'; $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); $brokerConfig = $this->getBrokerConfig($service); diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index 5c233db4..93420418 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -34,7 +34,8 @@ class ConsumerCommand extends BaseCommand {--broker= : Override broker connection from config.} {--timeout= : Sets timeout for consumer.} {--times= : Amount of messages to be consumed.} - {--config_name= : Change default name for laravel config file.}'; + {--config_name= : Change default name for laravel config file.} + {--service_name= : Change default name for services config file.}'; public function handle(Config $config) { From c00a633fdabacca084a28ca98ee3cf9cafe2d65c Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 16:10:57 -0300 Subject: [PATCH 115/152] chore: remove unused method --- src/Connectors/Consumer/Config.php | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 81ba46a6..33673cae 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -94,18 +94,4 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } - - /** - * Sometimes that user may pass `--partition=0` as argument. - * So if we just use array_filter here, this option will - * be removed. - * - * This code makes sure that only null values will be removed. - */ - private function filterValues(array $options = []): array - { - return array_filter($options, function ($value) { - return !is_null($value); - }); - } } From da8228e85aa66afa5ea309f6af7a1551564271de Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 16:46:50 -0300 Subject: [PATCH 116/152] chore: add how to use data objects --- docs/quick-usage.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/quick-usage.md b/docs/quick-usage.md index 7fd95905..f306cba2 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -1,6 +1,7 @@ ## Quick Usage Guide -- [Config file](#config) +- [Configure with files](#config) +- [Configure using data objects](#config-dto) - [Consumer](#consumer) - [Creating Consumer](#creating-consumer) - [Running](#running-consumer) @@ -90,7 +91,6 @@ return [ ]; ``` - ### Consumer @@ -144,7 +144,7 @@ To start consuming the topic, the simplest way to see it working is by running t ```bash $ php artisan kafka:consume this_is_your_topic_name --config_name=config.file --service_name=service.file -``` +``` This command will run in a `while true`, that means, it will never stop running. But, errors can happen, so we strongly advice you to run this command along with [supervisor](http://supervisord.org/running.html), @@ -162,6 +162,27 @@ redirect_stderr=true stdout_logfile=/var/log/default/kafka-consumer-price-update.log ``` +### Using data objects + +To configure and consume using classes: + +```php + use Metamorphosis\Consumer; + use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; + + $topic = config('yourConfig.topics.topic-id'); + $broker = config('yourService.broker'); + $avro = config('yourService.avro_schema'); + + $consumerConfiguration = ConsumerFactory::make($broker, $topic, $avro); + $consumer = app(Consumer::class, ['configOptions' => $consumerConfiguration]); + + $consumer->consume(); +``` + + + + That's it. For more information about usage, middlewares, broker authentication, consumer groups and other advanced topics, please have a look at our [Advanced Usage Guide](advanced.md). From fbb9168d057ecf804a7a264bbb52003e53258a43 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 16:48:54 -0300 Subject: [PATCH 117/152] chore: remove blank spaces --- docs/quick-usage.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/quick-usage.md b/docs/quick-usage.md index f306cba2..5178a312 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -180,9 +180,6 @@ To configure and consume using classes: $consumer->consume(); ``` - - - That's it. For more information about usage, middlewares, broker authentication, consumer groups and other advanced topics, please have a look at our [Advanced Usage Guide](advanced.md). From dba78f53d8eb961821fd7681f9dd04b5086af0d9 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 16:56:07 -0300 Subject: [PATCH 118/152] chore: fix codacy warnings --- CHANGELOG.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 009e3b39..b282caa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added file `config/service.php` to configure broker and authentication -- Added AvroSchemaMixedEncoderTest -- Added AvroSchemaDecoderTest -- Added ProducerWithConfigOptionsTest -- Added ConfigOptionsCommand to run commands with ConfigOptions class -- Added pt_BR contributing section -- Added setup-dev script on composer -- Added grumphp commit validation +- Added file `config/service.php` to configure broker and authentication +- Added AvroSchemaMixedEncoderTest +- Added AvroSchemaDecoderTest +- Added ProducerWithConfigOptionsTest +- Added ConfigOptionsCommand to run commands with ConfigOptions class +- Added pt_BR contributing section +- Added setup-dev script on composer +- Added grumphp commit validation ### Fixed From e21ebbd1ab1a59787df49aaaccd77973942a6758 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 17:00:33 -0300 Subject: [PATCH 119/152] chore: fix codacy warnings --- CHANGELOG.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b282caa2..b237f25e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added file `config/service.php` to configure broker and authentication -- Added AvroSchemaMixedEncoderTest -- Added AvroSchemaDecoderTest -- Added ProducerWithConfigOptionsTest -- Added ConfigOptionsCommand to run commands with ConfigOptions class -- Added pt_BR contributing section -- Added setup-dev script on composer -- Added grumphp commit validation +- Added AvroSchemaMixedEncoderTest +- Added AvroSchemaDecoderTest +- Added ProducerWithConfigOptionsTest +- Added ConfigOptionsCommand to run commands with ConfigOptions class +- Added pt_BR contributing section +- Added setup-dev script on composer +- Added grumphp commit validation +- Added file `config/service.php` to configure broker and authentication ### Fixed From 3634104c99358d9151d22c775e0e55763e4ed75f Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 17:39:32 -0300 Subject: [PATCH 120/152] docs: add commens explaining parameters --- config/kafka.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/kafka.php b/config/kafka.php index 9e74693b..3e7f0764 100644 --- a/config/kafka.php +++ b/config/kafka.php @@ -2,8 +2,14 @@ return [ 'topics' => [ + // This is your topic "keyword" where you will put all configurations needed + // on this specific topic. 'default' => [ + // The topic id is where you want to send or consume + // your messages from kafka. 'topic_id' => 'kafka-test', + + //your consumer configurations 'consumer' => [ 'consumer_group' => 'test-consumer-group', // Action to take when there is no initial From 472b0c92e78e025aec1a702542dcfb6db9f2144e Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 17:41:06 -0300 Subject: [PATCH 121/152] docs: document finished method --- docs/advanced.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index acca8234..14e40d1e 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -181,10 +181,17 @@ $ php artisan make:kafka-consumer PriceUpdateHandler This will create a KafkaConsumer class inside the application, on the `app/Kafka/Consumers/` directory. There, you'll have a `handler` method, which will send all records from the topic to the Consumer. -Methods will be available for handling exceptions: + +Available methods: + - `warning` method will be call whenever something not critical is received from the topic. Like a message informing that there's no more records to consume. - - `failure` method will be call whenever something critical happens, like an error to decode the record. + + + - `failure` will be call whenever something critical happens, like an error to decode the record. + + + - `finished` will be call when queue finishes ```php use App\Repository; @@ -230,6 +237,11 @@ class PriceUpdateHandler extends AbstractHandler { // handle failure exception } + + public function finished(): void + { + //handle queue end + } } ``` From cf3c17a21f66fad7529b3ee5c5f96a74d47404c7 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 17:41:52 -0300 Subject: [PATCH 122/152] docs: add anchor to section --- docs/quick-usage.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/quick-usage.md b/docs/quick-usage.md index 5178a312..b2cf9a92 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -162,7 +162,8 @@ redirect_stderr=true stdout_logfile=/var/log/default/kafka-consumer-price-update.log ``` -### Using data objects + +#### Using data objects To configure and consume using classes: From 984d80ba600d30314f5465e4d0c9c86a615fac3f Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 5 Apr 2022 14:34:32 -0300 Subject: [PATCH 123/152] chore: remove blank spaces on docs --- CHANGELOG.md | 2 +- docs/advanced.md | 10 ++++------ docs/quick-usage.md | 2 -- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b237f25e..47140732 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added ConfigOptionsCommand to run commands with ConfigOptions class - Added pt_BR contributing section - Added setup-dev script on composer -- Added grumphp commit validation +- Added grumphp commit validation - Added file `config/service.php` to configure broker and authentication ### Fixed diff --git a/docs/advanced.md b/docs/advanced.md index 14e40d1e..686d80ef 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -136,8 +136,7 @@ If you wish, you may set a middleware to run of a topic level or a consumer grou ], ``` -The order matters here, they'll be execute as queue, from the most global scope to the most specific (global scope > topic scope > group_consumers scope). - +The order matters here, they'll execute as queue, from the most global scope to the most specific (global scope > topic scope > group_consumers scope). ### Schemas @@ -245,7 +244,6 @@ class PriceUpdateHandler extends AbstractHandler } ``` - #### Creating Middleware You can create a middleware class, that works between the received data from broker and before being passed into consumers, using the follow command: @@ -287,7 +285,6 @@ stdout_logfile=/var/log/default/kafka-consumer-price-update.log Although you can run this simple command, it provides some options you can pass to make it more flexible to your needs. - - `--offset=` And if you need to start the consumption of a topic in a specific offset (it can be useful for debug purposes) @@ -323,6 +320,7 @@ Although you can run this simple command, it provides some options you can pass - `--service_name=` - Specify from what file services configurations should be read. + Specify from what file services configurations should be read. + - `$ php artisan kafka:consume price-update --service_name=config.file` + $ php artisan kafka:consume price-update --service_name=config.file diff --git a/docs/quick-usage.md b/docs/quick-usage.md index b2cf9a92..88488205 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -19,7 +19,6 @@ configuration and a file to keep the broker and schema configuration. In this ex This file keeps configurations about topics, consumers and producers. It should return an array of topics containing the topic name, topic_id, consumer, producer and the settings for each one of them: - ```php Date: Tue, 5 Apr 2022 14:50:42 -0300 Subject: [PATCH 124/152] chore: fix codacy --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47140732..101df70f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added pt_BR contributing section - Added setup-dev script on composer - Added grumphp commit validation -- Added file `config/service.php` to configure broker and authentication ### Fixed From f4d12b9d1bd95317f4a790f6341f3e43a3905288 Mon Sep 17 00:00:00 2001 From: Hugo Carvalho Date: Thu, 7 Apr 2022 11:03:20 -0300 Subject: [PATCH 125/152] Update docs/quick-usage.md Apply suggested changes Co-authored-by: Diego Felix --- docs/quick-usage.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/quick-usage.md b/docs/quick-usage.md index 88488205..d6a143c8 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -12,7 +12,8 @@ ### Configure using files To get started using configuration files, at least two files are needed. A file to keep the topics -configuration and a file to keep the broker and schema configuration. In this example, we will use the files `config/kafka.php` and `config/service.php`. +configuration and a file to keep the broker and schema configuration. In this example, we will use the files +`config/kafka.php` and `config/service.php`. ### File `config/kafka.php`: From dd50c5f6437976c4aa67856970cd07c676f9c049 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 25 Apr 2022 11:10:53 -0300 Subject: [PATCH 126/152] fix(codacy): fix codacy warnings --- CHANGELOG.md | 30 +++-- docs/quick-usage.pt.md | 154 ++++++++++++++++---------- readme.md | 1 + src/Connectors/Consumer/Config.php | 2 +- src/Console/ConfigOptionsCommand.php | 3 +- src/Middlewares/AvroSchemaDecoder.php | 6 - 6 files changed, 109 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 101df70f..342b1ef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,28 +5,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] - ### Added -- Added AvroSchemaMixedEncoderTest -- Added AvroSchemaDecoderTest -- Added ProducerWithConfigOptionsTest -- Added ConfigOptionsCommand to run commands with ConfigOptions class -- Added pt_BR contributing section -- Added setup-dev script on composer -- Added grumphp commit validation +- Added AvroSchemaMixedEncoderTest +- Added AvroSchemaDecoderTest +- Added ProducerWithConfigOptionsTest +- Added ConfigOptionsCommand to run commands with ConfigOptions class +- Added pt_BR contributing section +- Added setup-dev script on composer +- Added grumphp commit validation ### Fixed -- Fixed parameters and options override on Consumer\Config class -- Update instructions on contribute section -- Update project install section +- Fixed parameters and options override on Consumer\Config class +- Update instructions on contribute section +- Update project install section ### Changed -- Updated class from ConfigManager to ConfigOptions on unit tests -- Updated class from ConfigManager to ConfigOptions where any config request was made -- Consumer and Producer middlewares resolution - -### Removed +- Updated class from ConfigManager to ConfigOptions on unit tests +- Updated class from ConfigManager to ConfigOptions where any config request was made +- Consumer and Producer middlewares resolution diff --git a/docs/quick-usage.pt.md b/docs/quick-usage.pt.md index daf4dbfd..c3e14753 100644 --- a/docs/quick-usage.pt.md +++ b/docs/quick-usage.pt.md @@ -1,64 +1,96 @@ -## Guia rápido +## Quick Usage Guide -- [Arquivo de configuração](#config) -- [Consumer](#consumer) - - [Criando um Consumer](#creating-consumer) - - [Rodando o Consumer](#running-consumer) +- [Configurar usando arquivos](#config) +- [Configurar usando objetos](#config-dto) +- [Consumidor](#consumer) + - [Criando um consumidor](#creating-consumer) + - [Executando um consumidor](#running-consumer) +- [Produtor](#producer) + - [Produzindo mensagens](#produce-message) -### Arquivo de configuração: `config/kafka.php` - -Esse arquivo contém todas as informações sobre *brokers*, tópicos, *consumer groups* e *middlewares*. - -Para começar a usar, podemos focar em duas seções: - -- Brokers - - Uma lista de *brokers*, com configurações de conexão e autenticação. - - - `connections`: *obrigatório*. Pode ser uma `string` com múltiplas conexões separadas por vírgula ou uma `array` de conexões. - - - `auth`: *opcional*. É possivel se conectar sem autenticação ou usando autenticação SSL. - - ```php - 'brokers' => [ - 'price_brokers' => [ - 'connections' => 'localhost:8091,localhost:8092', - 'auth' => [ - 'type' => 'ssl', - 'ca' => storage_path('ca.pem'), - 'certificate' => storage_path('kafka.cert'), - 'key' => storage_path('kafka.key'), - ], - ], - 'stock_brokers' => [ - 'connections' => ['localhost:8091', 'localhost:8092'], - 'auth' => [], // pode ser uma array vazia ou até mesmo não ter essa chave aqui. - ], - ], - ``` - -- Tópicos - - Uma lista de configuração de tópicos, como nome, qual *broker* usar, *consumer group* e *middlewares*. - - Aqui você pode especificar os *consumer groups*. Cada tópico pode ter vários grupos, - e cada grupo tem a sua configuração para cada *consumer*, *offset_reset* (para definir um *offset* inicial) e *middlewares* que devem ser usados. - - ```php - 'topics' => [ - 'price_update' => [ - 'topic' => 'products.price.update', - 'broker' => 'price_brokers', - 'consumer_groups' => [ - 'default' => [ - 'offset_reset' => 'smallest', - 'handler' => '\App\Kafka\Consumers\PriceUpdateConsumer', - ], - ], - ], - ], - ``` +### Configurar usando arquivos + +Para começar a usar arquivos de configuração, são necessários pelo menos dois arquivos. Um arquivo para manter os tópicos +configuração e um arquivo para manter a configuração do broker e do esquema. Neste exemplo, usaremos os arquivos +`config/kafka.php` e `config/service.php`. + + +### Arquivo `config/kafka.php`: + +Este arquivo mantém configurações sobre tópicos, consumidores e produtores. +Deve retornar um array de tópicos contendo o nome do tópico, topic_id, consumidor, produtor e as configurações de cada um deles: + + +```php + [ + 'this_is_your_topic_name' => [ + 'topic_id' => "this_is_your_topic_id", + 'consumer' => [ + 'consumer_group' => 'your-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'timeout' => 20000, + 'auto_commit' => true, + 'commit_async' => false, + 'middlewares' => [], + ], + + 'producer' => [ + 'required_acknowledgment' => true, + 'is_async' => true, + 'max_poll_records' => 500, + 'flush_attempts' => 10, + 'middlewares' => [], + 'timeout' => 10000, + 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1, + ], + ] + ], +]; +``` + +### File `config/service.php` + +Esse arquivo possui as configurações de **broker** e **schema** utilizados. + +```php + [ + 'url' => '', + 'request_options' => [ + 'headers' => [ + 'Authorization' => [ + 'Basic ' . base64_encode( + env('AVRO_SCHEMA_USERNAME').':'.env('AVRO_SCHEMA_PASSWORD') + ), + ], + ], + ], + + 'ssl_verify' => true, + 'username' => 'USERNAME', + 'password' => 'PASSWORD', + ], + + 'broker' => [ + 'connections' => 'kafka:9092', + 'auth' => [ + 'type' => 'ssl', + 'ca' => storage_path('ca.pem'), + 'certificate' => storage_path('kafka.cert'), + 'key' => storage_path('kafka.key'), + ], + ], +]; +``` ### Consumer @@ -113,11 +145,11 @@ class PriceUpdateConsumer extends AbstractHandler Agora é só consumir o tópico. -A forma mais simples de ver tudo isso funcionando é rodando o comando `kafka:consume` com o nome do tópico que foi configurado: +Para começar a consumir o tópico, a maneira mais simples de vê-lo funcionando é executando o comando kafka:consume junto com o nome do tópico, arquivo de configuração do tópico e arquivo de configuração do serviço: ```bash -$ php artisan kafka:consume price-update -``` +$ php artisan kafka:consume this_is_your_topic_name --config_name=config.file --service_name=service.file +``` Esse comando rodará em um laço infinito (while true), isso significa que ele nunca irá parar de rodar por conta própria. Mas erros podem acontecer, então, recomendamos fortemente que você execute este comando com o auxílio de um [supervisor](http://supervisord.org/running.html), como no exemplo abaixo: diff --git a/readme.md b/readme.md index 76b81013..9206ab91 100644 --- a/readme.md +++ b/readme.md @@ -15,6 +15,7 @@ - [Installation](#installation) - [Quick Usage Guide](docs/quick-usage.md) - [Advanced Usage Guide](docs/advanced.md) +- [Upgrade Guide](docs/upgrade.md) - [Contributing](docs/CONTRIBUTING.md) - [License](#license) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 33673cae..fd74f86c 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -40,7 +40,7 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; - public function makeWithConfigOptions(string $handlerClass, ?int $times = null): ?Consumer + public function makeWithConfigOptions(string $handlerClass): ?Consumer { $handler = app($handlerClass); $configOptions = $handler->getConfigOptions(); diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index eb4c1071..c65bf454 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Console; use Illuminate\Console\Command as BaseCommand; -use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; @@ -26,7 +25,7 @@ class ConfigOptionsCommand extends BaseCommand {handler : handler.} {--times= : Amount of messages to be consumed.}'; - public function handle(Config $config) + public function handle() { $consumerHandler = app($this->argument('handler')); diff --git a/src/Middlewares/AvroSchemaDecoder.php b/src/Middlewares/AvroSchemaDecoder.php index 1aeffcff..d7a39f90 100644 --- a/src/Middlewares/AvroSchemaDecoder.php +++ b/src/Middlewares/AvroSchemaDecoder.php @@ -7,18 +7,12 @@ use Metamorphosis\Avro\Serializer\MessageDecoder; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; -use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema; use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; class AvroSchemaDecoder implements MiddlewareInterface { private MessageDecoder $decoder; - /** - * @var AvroSchema - */ - private $avroSchema; - public function __construct(ClientFactory $factory, ConsumerConfigOptions $consumerConfigOptions) { if (!$consumerConfigOptions->getAvroSchema()->getUrl()) { From 4b6b0bab0ff0f01e338de82e1c1f964ebac242ed Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 3 May 2022 16:12:16 -0300 Subject: [PATCH 127/152] docs: add upgrade guide --- docs/upgrade.md | 76 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 docs/upgrade.md diff --git a/docs/upgrade.md b/docs/upgrade.md new file mode 100644 index 00000000..5c4ef6fe --- /dev/null +++ b/docs/upgrade.md @@ -0,0 +1,76 @@ + +## Upgrade guide + +To upgrade from version X.x to version X.y: + +Move your `avroschema` and `broker` section from old `config/kafka.php` file into a new file: + + +```php + [ + 'this_is_your_topic_name' => [ + 'topic_id' => "this_is_your_topic_id", + 'consumer' => [ + 'consumer_group' => 'your-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'timeout' => 20000, + 'auto_commit' => true, + 'commit_async' => false, + 'middlewares' => [], + ], + + 'producer' => [ + 'required_acknowledgment' => true, + 'is_async' => true, + 'max_poll_records' => 500, + 'flush_attempts' => 10, + 'middlewares' => [], + 'timeout' => 10000, + 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1, + ], + ] + ], +]; +``` + +Upgrade your topic configuration files: + +```php + [ + 'this_is_your_topic_name' => [ + 'topic_id' => "this_is_your_topic_id", + 'consumer' => [ + 'consumer_group' => 'your-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'timeout' => 20000, + 'auto_commit' => true, + 'commit_async' => false, + 'middlewares' => [], + ], + + 'producer' => [ + 'required_acknowledgment' => true, + 'is_async' => true, + 'max_poll_records' => 500, + 'flush_attempts' => 10, + 'middlewares' => [], + 'timeout' => 10000, + 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1, + ], + ] + ], +]; +``` + From c1ca85a9054e04abd0c6ec2cebb22c6097ae9b8e Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 5 Apr 2023 18:48:45 -0300 Subject: [PATCH 128/152] fix: add handleMiddlewares method --- src/Connectors/Consumer/Config.php | 8 ++++++++ tests/Unit/AbstractConfigManagerTest.php | 0 2 files changed, 8 insertions(+) delete mode 100644 tests/Unit/AbstractConfigManagerTest.php diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index fd74f86c..f868a2cb 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -94,4 +94,12 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } + + private function getMiddlewares(string $configName, array $topicConfig): array + { + return array_merge( + config($configName . '.middlewares.consumer', []), + $topicConfig['consumer']['middlewares'] ?? [] + ); + } } diff --git a/tests/Unit/AbstractConfigManagerTest.php b/tests/Unit/AbstractConfigManagerTest.php deleted file mode 100644 index e69de29b..00000000 From ba3a99c3f5acea1fcff65548c31592df9adbfc4e Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 14:01:28 -0300 Subject: [PATCH 129/152] style: fix coding standard violations Signed-off-by: hcdias --- config/service.php | 6 ++- src/Authentication/Factory.php | 10 ++++- src/Authentication/SASLAuthentication.php | 10 ++--- src/Authentication/SSLAuthentication.php | 10 ++--- src/Connectors/AbstractConfig.php | 9 +++-- src/Connectors/Consumer/Config.php | 6 ++- src/Connectors/Consumer/Factory.php | 39 +++++++++++++------ src/Connectors/Consumer/LowLevel.php | 15 +++++-- src/Connectors/Producer/Connector.php | 6 ++- src/Console/ConfigOptionsCommand.php | 7 ++-- src/Console/ConsumerCommand.php | 6 +-- src/MetamorphosisServiceProvider.php | 8 ++-- src/Middlewares/AvroSchemaDecoder.php | 8 +++- src/Middlewares/AvroSchemaMixedEncoder.php | 25 +++++++----- src/Producer.php | 28 ++++++++++--- .../Factories/ConsumerFactory.php | 5 ++- .../Producer/AbstractProducer.php | 2 +- 17 files changed, 137 insertions(+), 63 deletions(-) diff --git a/config/service.php b/config/service.php index 82cb1299..64f81077 100644 --- a/config/service.php +++ b/config/service.php @@ -6,8 +6,10 @@ 'request_options' => [ 'headers' => [ 'Authorization' => [ - 'Basic '.base64_encode( - env('AVRO_SCHEMA_USERNAME').':'.env('AVRO_SCHEMA_PASSWORD') + 'Basic' . base64_encode( + env('AVRO_SCHEMA_USERNAME') . ':' . env( + 'AVRO_SCHEMA_PASSWORD' + ) ), ], ], diff --git a/src/Authentication/Factory.php b/src/Authentication/Factory.php index c8a8f38a..55fb7053 100644 --- a/src/Authentication/Factory.php +++ b/src/Authentication/Factory.php @@ -24,11 +24,17 @@ public static function authenticate(Conf $conf, AuthInterface $configOptions): v break; case self::TYPE_SSL: - app(SSLAuthentication::class, compact('conf', 'configOptions')); + app( + SSLAuthentication::class, + compact('conf', 'configOptions') + ); break; case self::TYPE_SASL_SSL: - app(SASLAuthentication::class, compact('conf', 'configOptions')); + app( + SASLAuthentication::class, + compact('conf', 'configOptions') + ); break; default: diff --git a/src/Authentication/SASLAuthentication.php b/src/Authentication/SASLAuthentication.php index 3918558d..f4a0e3b0 100644 --- a/src/Authentication/SASLAuthentication.php +++ b/src/Authentication/SASLAuthentication.php @@ -9,10 +9,7 @@ class SASLAuthentication implements AuthenticationInterface { private Conf $conf; - /** - * @var SaslSsl - */ - private $configOptions; + private SaslSsl $configOptions; public function __construct(Conf $conf, SaslSsl $configOptions) { @@ -29,7 +26,10 @@ private function authenticate(): void // The mechanisms key is optional when configuring this kind of authentication // If the user does not specify the mechanism, the default will be 'PLAIN'. // But, to make config more clear, we are asking the user every time. - $this->conf->set('sasl.mechanisms', $this->configOptions->getMechanisms()); + $this->conf->set( + 'sasl.mechanisms', + $this->configOptions->getMechanisms() + ); $this->conf->set('sasl.username', $this->configOptions->getUsername()); $this->conf->set('sasl.password', $this->configOptions->getPassword()); } diff --git a/src/Authentication/SSLAuthentication.php b/src/Authentication/SSLAuthentication.php index 5dfc06e4..e698dfe5 100644 --- a/src/Authentication/SSLAuthentication.php +++ b/src/Authentication/SSLAuthentication.php @@ -9,10 +9,7 @@ class SSLAuthentication implements AuthenticationInterface { private Conf $conf; - /** - * @var Ssl - */ - private $configOptions; + private Ssl $configOptions; public function __construct(Conf $conf, Ssl $configOptions) { @@ -26,7 +23,10 @@ private function authenticate(): void { $this->conf->set('security.protocol', $this->configOptions->getType()); $this->conf->set('ssl.ca.location', $this->configOptions->getCa()); - $this->conf->set('ssl.certificate.location', $this->configOptions->getCertificate()); + $this->conf->set( + 'ssl.certificate.location', + $this->configOptions->getCertificate() + ); $this->conf->set('ssl.key.location', $this->configOptions->getKey()); } } diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index fa8b8ac1..b71c683c 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -9,15 +9,18 @@ abstract class AbstractConfig { protected function getBrokerConfig(string $servicesFile): array { - if (!$brokerConfig = config($servicesFile.'.broker')) { - throw new ConfigurationException("Broker configuration not found on '{$servicesFile}'"); + if (!$brokerConfig = config($servicesFile . '.broker')) { + throw new ConfigurationException( + "Broker configuration not found on '{$servicesFile}'" + ); } + return $brokerConfig; } protected function getSchemaConfig(string $servicesFile): array { - return config($servicesFile.'.avro_schema', []); + return config($servicesFile . '.avro_schema', []); } protected function validate(array $config): void diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index f868a2cb..d13976be 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -74,7 +74,11 @@ public function make(array $options, array $arguments): Consumer } } - return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); + return ConsumerFactory::make( + $brokerConfig, + $topicConfig, + $schemaConfig + ); } /** diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index c82c4ad2..923fb817 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -26,32 +26,49 @@ public static function make(ConsumerConfigOptions $configOptions): Manager $middlewares = $configOptions->getMiddlewares(); foreach ($middlewares as &$middleware) { - $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $configOptions]) : $middleware; + $middleware = is_string($middleware) + ? app( + $middleware, + ['consumerConfigOptions' => $configOptions] + ) + : $middleware; } - $middlewares[] = app(ConsumerMiddleware::class, ['consumerTopicHandler' => $handler]); + $middlewares[] = app( + ConsumerMiddleware::class, + ['consumerTopicHandler' => $handler] + ); $dispatcher = self::getMiddlewareDispatcher($middlewares); - return new Manager($consumer, $handler, $dispatcher, $autoCommit, $commitAsync); - } - - protected static function requiresPartition(ConsumerConfigOptions $configOptions): bool - { - $partition = $configOptions->getPartition(); - - return !is_null($partition) && $partition >= 0; + return new Manager( + $consumer, + $handler, + $dispatcher, + $autoCommit, + $commitAsync + ); } public static function getConsumer(bool $autoCommit, ConsumerConfigOptions $configOptions): ConsumerInterface { if (self::requiresPartition($configOptions)) { - return app(LowLevel::class)->getConsumer($autoCommit, $configOptions); + return app(LowLevel::class)->getConsumer( + $autoCommit, + $configOptions + ); } return app(HighLevel::class)->getConsumer($autoCommit, $configOptions); } + protected static function requiresPartition(ConsumerConfigOptions $configOptions): bool + { + $partition = $configOptions->getPartition(); + + return !is_null($partition) && $partition >= 0; + } + private static function getMiddlewareDispatcher(array $middlewares): Dispatcher { return new Dispatcher($middlewares); diff --git a/src/Connectors/Consumer/LowLevel.php b/src/Connectors/Consumer/LowLevel.php index 7c7395ac..30f4c277 100644 --- a/src/Connectors/Consumer/LowLevel.php +++ b/src/Connectors/Consumer/LowLevel.php @@ -32,9 +32,15 @@ public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): Con $consumer->addBrokers($broker->getConnections()); $topicConf = $this->getTopicConfigs($configOptions); - $topicConsumer = $consumer->newTopic($configOptions->getTopicId(), $topicConf); + $topicConsumer = $consumer->newTopic( + $configOptions->getTopicId(), + $topicConf + ); - $topicConsumer->consumeStart($configOptions->getPartition(), $configOptions->getOffset()); + $topicConsumer->consumeStart( + $configOptions->getPartition(), + $configOptions->getOffset() + ); return new LowLevelConsumer($topicConsumer, $configOptions); } @@ -46,7 +52,10 @@ protected function getTopicConfigs(ConfigOptions $configOptions) // Set where to start consuming messages when there is no initial offset in // offset store or the desired offset is out of range. // 'smallest': start from the beginning - $topicConfig->set('auto.offset.reset', $configOptions->getOffsetReset()); + $topicConfig->set( + 'auto.offset.reset', + $configOptions->getOffsetReset() + ); return $topicConfig; } diff --git a/src/Connectors/Producer/Connector.php b/src/Connectors/Producer/Connector.php index 1109a598..3d427afc 100644 --- a/src/Connectors/Producer/Connector.php +++ b/src/Connectors/Producer/Connector.php @@ -12,8 +12,10 @@ class Connector { - public function getProducerTopic(HandlerInterface $handler, ProducerConfigOptions $producerConfigOptions): KafkaProducer - { + public function getProducerTopic( + HandlerInterface $handler, + ProducerConfigOptions $producerConfigOptions + ): KafkaProducer { $conf = resolve(Conf::class); if ($this->canHandleResponse($handler)) { diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index c65bf454..d592a8b1 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -1,4 +1,5 @@ getTopicId().PHP_EOL; - $text .= ' on consumer group: '.$configOptions->getConsumerGroup().PHP_EOL; - $text .= 'Connecting in '.$configOptions->getBroker()->getConnections().PHP_EOL; + $text = 'Starting consumer for topic: ' . $configOptions->getTopicId() . PHP_EOL; + $text .= ' on consumer group: ' . $configOptions->getConsumerGroup() . PHP_EOL; + $text .= 'Connecting in ' . $configOptions->getBroker()->getConnections() . PHP_EOL; $text .= 'Running consumer..'; $this->output->writeln($text); diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index 93420418..ded31d7f 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -51,9 +51,9 @@ public function handle(Config $config) private function writeStartingConsumer(Consumer $consumer) { - $text = 'Starting consumer for topic: '.$consumer->getTopicId().PHP_EOL; - $text .= ' on consumer group: '.$consumer->getConsumerGroup().PHP_EOL; - $text .= 'Connecting in '.$consumer->getBroker()->getConnections().PHP_EOL; + $text = 'Starting consumer for topic: ' . $consumer->getTopicId() . PHP_EOL; + $text .= ' on consumer group: ' . $consumer->getConsumerGroup() . PHP_EOL; + $text .= 'Connecting in ' . $consumer->getBroker()->getConnections() . PHP_EOL; $text .= 'Running consumer..'; $this->output->writeln($text); diff --git a/src/MetamorphosisServiceProvider.php b/src/MetamorphosisServiceProvider.php index 36dffdf5..6a64d8c4 100644 --- a/src/MetamorphosisServiceProvider.php +++ b/src/MetamorphosisServiceProvider.php @@ -14,12 +14,12 @@ class MetamorphosisServiceProvider extends ServiceProvider public function boot() { $this->publishes([ - __DIR__.'/../config/kafka.php' => config_path('kafka.php'), - __DIR__.'/../config/service.php' => config_path('service.php'), + __DIR__ . '/../config/kafka.php' => config_path('kafka.php'), + __DIR__ . '/../config/service.php' => config_path('service.php'), ], 'config'); - $this->mergeConfigFrom(__DIR__.'/../config/kafka.php', 'kafka'); - $this->mergeConfigFrom(__DIR__.'/../config/service.php', 'service'); + $this->mergeConfigFrom(__DIR__ . '/../config/kafka.php', 'kafka'); + $this->mergeConfigFrom(__DIR__ . '/../config/service.php', 'service'); } public function register() diff --git a/src/Middlewares/AvroSchemaDecoder.php b/src/Middlewares/AvroSchemaDecoder.php index d7a39f90..07f2193d 100644 --- a/src/Middlewares/AvroSchemaDecoder.php +++ b/src/Middlewares/AvroSchemaDecoder.php @@ -16,10 +16,14 @@ class AvroSchemaDecoder implements MiddlewareInterface public function __construct(ClientFactory $factory, ConsumerConfigOptions $consumerConfigOptions) { if (!$consumerConfigOptions->getAvroSchema()->getUrl()) { - throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaDecoder Middleware"); + throw new ConfigurationException( + "Avro schema url not found, it's required to use AvroSchemaDecoder Middleware" + ); } - $this->decoder = new MessageDecoder($factory->make($consumerConfigOptions->getAvroSchema())); + $this->decoder = new MessageDecoder( + $factory->make($consumerConfigOptions->getAvroSchema()) + ); } public function process(RecordInterface $record, Closure $next) diff --git a/src/Middlewares/AvroSchemaMixedEncoder.php b/src/Middlewares/AvroSchemaMixedEncoder.php index 6969a80e..67c09076 100644 --- a/src/Middlewares/AvroSchemaMixedEncoder.php +++ b/src/Middlewares/AvroSchemaMixedEncoder.php @@ -21,18 +21,22 @@ class AvroSchemaMixedEncoder implements MiddlewareInterface private CachedSchemaRegistryClient $schemaRegistry; - /** - * @var ProducerConfigOptions - */ - private $producerConfigOptions; + private ProducerConfigOptions $producerConfigOptions; - public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, ProducerConfigOptions $producerConfigOptions) - { + public function __construct( + SchemaId $schemaIdEncoder, + ClientFactory $factory, + ProducerConfigOptions $producerConfigOptions + ) { if (!$producerConfigOptions->getAvroSchema()->getUrl()) { - throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaEncoder Middleware"); + throw new ConfigurationException( + "Avro schema url not found, it's required to use AvroSchemaEncoder Middleware" + ); } - $schemaRegistry = $factory->make($producerConfigOptions->getAvroSchema()); + $schemaRegistry = $factory->make( + $producerConfigOptions->getAvroSchema() + ); $this->schemaIdEncoder = $schemaIdEncoder; $this->schemaRegistry = $schemaRegistry; $this->producerConfigOptions = $producerConfigOptions; @@ -41,7 +45,10 @@ public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, P public function process(RecordInterface $record, Closure $next) { $topic = $this->producerConfigOptions->getTopicId(); - $schema = $this->schemaRegistry->getBySubjectAndVersion("{$topic}-value", 'latest'); + $schema = $this->schemaRegistry->getBySubjectAndVersion( + "{$topic}-value", + 'latest' + ); $arrayPayload = json_decode($record->getPayload(), true); $encodedPayload = $this->schemaIdEncoder->encode( $schema, diff --git a/src/Producer.php b/src/Producer.php index 4fa9ee5b..b3211f7c 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -36,20 +36,36 @@ public function build(HandlerInterface $producerHandler): Dispatcher $middlewares = $producerConfigOptions->getMiddlewares(); foreach ($middlewares as &$middleware) { - $middleware = is_string($middleware) ? app($middleware, ['producerConfigOptions' => $producerConfigOptions]) : $middleware; + $middleware = is_string($middleware) + ? app( + $middleware, + ['producerConfigOptions' => $producerConfigOptions] + ) + : $middleware; } - $middlewares[] = $this->getProducerMiddleware($producerHandler, $producerConfigOptions); + $middlewares[] = $this->getProducerMiddleware( + $producerHandler, + $producerConfigOptions + ); return new Dispatcher($middlewares); } - public function getProducerMiddleware(HandlerInterface $producerHandler, ProducerConfigOptions $producerConfigOptions): ProducerMiddleware - { - $producer = $this->connector->getProducerTopic($producerHandler, $producerConfigOptions); + public function getProducerMiddleware( + HandlerInterface $producerHandler, + ProducerConfigOptions $producerConfigOptions + ): ProducerMiddleware { + $producer = $this->connector->getProducerTopic( + $producerHandler, + $producerConfigOptions + ); $topic = $producer->newTopic($producerConfigOptions->getTopicId()); - $poll = app(Poll::class, ['producer' => $producer, 'producerConfigOptions' => $producerConfigOptions]); + $poll = app( + Poll::class, + ['producer' => $producer, 'producerConfigOptions' => $producerConfigOptions] + ); $partition = $producerConfigOptions->getPartition(); return app( diff --git a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php index 4dd62736..2d28a28c 100644 --- a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php +++ b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php @@ -25,7 +25,10 @@ private static function getConsumerGroupConfig(array $topicData): array $consumer = $topicData['consumer']; $topicData['consumerGroup'] = $consumer['consumer_group']; - return array_merge_recursive($topicData, self::convertConfigAttributes($consumer)); + return array_merge_recursive( + $topicData, + self::convertConfigAttributes($consumer) + ); } private static function convertConfigAttributes(array $consumerConfig): array diff --git a/src/TopicHandler/Producer/AbstractProducer.php b/src/TopicHandler/Producer/AbstractProducer.php index 694aedd0..ecab7c93 100644 --- a/src/TopicHandler/Producer/AbstractProducer.php +++ b/src/TopicHandler/Producer/AbstractProducer.php @@ -20,7 +20,7 @@ class AbstractProducer implements HandlerInterface */ protected $producer; - public function __construct($record, Producer $configOptions, string $key = null) + public function __construct($record, Producer $configOptions, ?string $key = null) { $this->record = $record; $this->key = $key; From cfdd4a7af7fbe0b586979422acd63cbdc67db44f Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 14:15:25 -0300 Subject: [PATCH 130/152] style: fix coding standard violations Signed-off-by: hcdias --- tests/Integration/ConsumerTest.php | 5 +- tests/Integration/ProducerTest.php | 38 +++++---- tests/Integration/ProducerWithAvroTest.php | 12 ++- tests/Unit/Authentication/FactoryTest.php | 6 +- .../Authentication/SASLAuthenticationTest.php | 6 +- .../Authentication/SSLAuthenticationTest.php | 6 +- tests/Unit/Connectors/Consumer/ConfigTest.php | 4 +- .../Unit/Connectors/Consumer/FactoryTest.php | 83 ++++++++++--------- .../Connectors/Producer/ConnectorTest.php | 14 +++- tests/Unit/Console/ConsumerCommandTest.php | 54 ++++++------ tests/Unit/Consumers/LowLevelTest.php | 5 +- .../Middlewares/AvroSchemaDecoderTest.php | 10 ++- .../AvroSchemaMixedEncoderTest.php | 32 +++++-- tests/Unit/ProducerTest.php | 10 +-- 14 files changed, 177 insertions(+), 108 deletions(-) diff --git a/tests/Integration/ConsumerTest.php b/tests/Integration/ConsumerTest.php index 1d4397e8..2fb7b1bf 100644 --- a/tests/Integration/ConsumerTest.php +++ b/tests/Integration/ConsumerTest.php @@ -59,7 +59,10 @@ public function testItShouldSetup(): void $saleOrderDispatcher = Metamorphosis::build($messageProducer); $saleOrderDispatcher->handle($messageProducer->createRecord()); - $consumer = $this->app->make(Consumer::class, ['configOptions' => $consumerConfigOptions]); + $consumer = $this->app->make( + Consumer::class, + ['configOptions' => $consumerConfigOptions] + ); $expected = '{"id":"MESSAGE_ID"}'; // Actions diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 94ab5626..8fbe5305 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -18,22 +18,9 @@ class ProducerTest extends LaravelTestCase protected string $firstLowLevelMessage; - /** - * @var string - */ - protected $secondLowLevelMessage; + protected string $secondLowLevelMessage; - /** - * @var string - */ - protected $topicId; - - protected function setUp(): void - { - parent::setUp(); - $this->withoutAuthentication(); - $this->topicId = 'kafka-test-'.Str::random(5); - } + protected string $topicId; /** * @group runProducer @@ -66,6 +53,14 @@ public function testShouldRunAProducerAndReceiveMessagesWithALowLevelConsumer(): $this->runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingToTwoMessagesConsumed(); } + protected function setUp(): void + { + parent::setUp(); + + $this->withoutAuthentication(); + $this->topicId = 'kafka-test-' . Str::random(5); + } + protected function withoutAuthentication(): void { config(['service.broker.auth' => []]); @@ -73,7 +68,9 @@ protected function withoutAuthentication(): void protected function haveAConsumerHandlerConfigured(): void { - config(['kafka.topics.default.consumer.handler' => MessageConsumer::class]); + config( + ['kafka.topics.default.consumer.handler' => MessageConsumer::class] + ); } protected function haveAConsumerPartitionConfigured(): void @@ -145,7 +142,9 @@ private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $producerConfigOptions = $this->createProducerConfigOptions($this->topicId); + $producerConfigOptions = $this->createProducerConfigOptions( + $this->topicId + ); $producer = app(MessageProducer::class, [ 'record' => $this->highLevelMessage, 'configOptions' => $producerConfigOptions, @@ -158,7 +157,9 @@ private function haveSomeRandomMessagesProduced(): void private function produceRecordMessage(string $record): string { - $producerConfigOptions = $this->createProducerConfigOptions('low_level'); + $producerConfigOptions = $this->createProducerConfigOptions( + 'low_level' + ); $producer = app(MessageProducer::class, [ 'record' => $record, 'configOptions' => $producerConfigOptions, @@ -206,6 +207,7 @@ private function haveFourProducedMessages(): void private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { $broker = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( $topicId, $broker, diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index b0c801ad..54793d98 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -22,11 +22,12 @@ class ProducerWithAvroTest extends LaravelTestCase /** * @var string[] */ - protected $records; + protected array $records; public function setUp(): void { parent::setUp(); + $this->records = ['saleOrderId' => 'SALE_ORDER_ID', 'productId' => 'PRODUCT_ID']; } @@ -110,8 +111,12 @@ protected function haveAHandlerConfigured(): void private function haveSomeRandomMessagesProduced(): void { - $producerConfigOptionsSale = $this->createProducerConfigOptions('sale_order'); - $producerConfigOptionsProduct = $this->createProducerConfigOptions('product'); + $producerConfigOptionsSale = $this->createProducerConfigOptions( + 'sale_order' + ); + $producerConfigOptionsProduct = $this->createProducerConfigOptions( + 'product' + ); $saleOrderProducer = app(MessageProducer::class, [ 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], @@ -166,6 +171,7 @@ private function setLogExpectationsFor(string $message): void private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { $broker = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( $topicId, $broker, diff --git a/tests/Unit/Authentication/FactoryTest.php b/tests/Unit/Authentication/FactoryTest.php index 717abe16..f13b2e46 100644 --- a/tests/Unit/Authentication/FactoryTest.php +++ b/tests/Unit/Authentication/FactoryTest.php @@ -16,7 +16,11 @@ class FactoryTest extends LaravelTestCase public function testItMakesSslAuthenticationClass(): void { // Set - $configOptionsSsl = new Ssl('path/to/ca', 'path/to/certificate', 'path/to/key'); + $configOptionsSsl = new Ssl( + 'path/to/ca', + 'path/to/certificate', + 'path/to/key' + ); $conf = new Conf(); $expected = [ 'security.protocol' => 'ssl', diff --git a/tests/Unit/Authentication/SASLAuthenticationTest.php b/tests/Unit/Authentication/SASLAuthenticationTest.php index f9e9a474..45a4dc3a 100644 --- a/tests/Unit/Authentication/SASLAuthenticationTest.php +++ b/tests/Unit/Authentication/SASLAuthenticationTest.php @@ -12,7 +12,11 @@ class SASLAuthenticationTest extends LaravelTestCase public function testItShouldValidateAuthenticationConfigurations(): void { // Set - $configSaslSsl = new SaslSsl('PLAIN', 'some-username', 'some-password'); + $configSaslSsl = new SaslSsl( + 'PLAIN', + 'some-username', + 'some-password' + ); $conf = new Conf(); $expected = [ diff --git a/tests/Unit/Authentication/SSLAuthenticationTest.php b/tests/Unit/Authentication/SSLAuthenticationTest.php index 35a9168e..9ea6c6b0 100644 --- a/tests/Unit/Authentication/SSLAuthenticationTest.php +++ b/tests/Unit/Authentication/SSLAuthenticationTest.php @@ -13,7 +13,11 @@ public function testItShouldValidateAuthenticationConfigurations(): void { // Set $conf = new Conf(); - $configSsl = new Ssl('path/to/ca', 'path/to/certificate', 'path/to/key'); + $configSsl = new Ssl( + 'path/to/ca', + 'path/to/certificate', + 'path/to/key' + ); $expected = [ 'security.protocol' => 'ssl', 'ssl.ca.location' => 'path/to/ca', diff --git a/tests/Unit/Connectors/Consumer/ConfigTest.php b/tests/Unit/Connectors/Consumer/ConfigTest.php index ffeaf5cd..224f5666 100644 --- a/tests/Unit/Connectors/Consumer/ConfigTest.php +++ b/tests/Unit/Connectors/Consumer/ConfigTest.php @@ -85,7 +85,9 @@ public function testShouldValidateConsumerConfig(): void ]); // Actions - $configManager = $config->makeWithConfigOptions(ConsumerHandlerDummy::class); + $configManager = $config->makeWithConfigOptions( + ConsumerHandlerDummy::class + ); // Assertions $this->assertArraySubset($expected, $configManager->toArray()); diff --git a/tests/Unit/Connectors/Consumer/FactoryTest.php b/tests/Unit/Connectors/Consumer/FactoryTest.php index 2ccb9cd9..fcd1a6d3 100644 --- a/tests/Unit/Connectors/Consumer/FactoryTest.php +++ b/tests/Unit/Connectors/Consumer/FactoryTest.php @@ -11,6 +11,52 @@ class FactoryTest extends LaravelTestCase { + public function testItMakesManagerWithLowLevelConsumer(): void + { + // Set + $this->haveAConsumerWithPartitionConfigured(); + + $config = new Config(); + $configConsumer = $config->make( + ['timeout' => 61], + ['topic' => 'topic_key', 'consumer_group' => 'with-partition'] + ); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); + } + + public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid(): void + { + // Set + $this->haveAConsumerWithoutPartitionConfigured(); + $config = new Config(); + $configConsumer = $config->make( + ['timeout' => 61, 'partition' => -1], + ['topic' => 'topic_key', 'consumer_group' => 'with-partition'] + ); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); + } + + public function testItMakesHighLevelClass(): void + { + // Set + $this->haveAConsumerWithoutPartitionConfigured(); + $config = new Config(); + $configConsumer = $config->make( + ['timeout' => 61], + ['topic' => 'topic_key', 'consumer_group' => 'without-partition'] + ); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); + } + protected function setUp(): void { parent::setUp(); @@ -47,43 +93,6 @@ protected function setUp(): void ]); } - public function testItMakesManagerWithLowLevelConsumer(): void - { - // Set - $this->haveAConsumerWithPartitionConfigured(); - - $config = new Config(); - $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); - $manager = Factory::make($configConsumer); - - // Assertions - $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); - } - - public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid(): void - { - // Set - $this->haveAConsumerWithoutPartitionConfigured(); - $config = new Config(); - $configConsumer = $config->make(['timeout' => 61, 'partition' => -1], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); - $manager = Factory::make($configConsumer); - - // Assertions - $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); - } - - public function testItMakesHighLevelClass(): void - { - // Set - $this->haveAConsumerWithoutPartitionConfigured(); - $config = new Config(); - $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'without-partition']); - $manager = Factory::make($configConsumer); - - // Assertions - $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); - } - private function haveAConsumerWithPartitionConfigured() { config([ diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index f456dfb7..2869318b 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -33,7 +33,7 @@ public function testItShouldMakeSetup(): void $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); - $handler = new class('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface { + $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface { public function success(Message $message): void { } @@ -57,7 +57,10 @@ public function failed(Message $message): void ->andReturn($broker); // Actions - $result = $connector->getProducerTopic($handler, $producerConfigOptions); + $result = $connector->getProducerTopic( + $handler, + $producerConfigOptions + ); // Assertions $this->assertInstanceOf(KafkaProducer::class, $result); @@ -79,7 +82,7 @@ public function testItShouldMakeSetupWithoutHandleResponse(): void $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); - $handler = new class('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface { + $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface { public function success(Message $message): void { } @@ -102,7 +105,10 @@ public function failed(Message $message): void ->andReturn($broker); // Actions - $result = $connector->getProducerTopic($handler, $producerConfigOptions); + $result = $connector->getProducerTopic( + $handler, + $producerConfigOptions + ); // Assertions $this->assertInstanceOf(KafkaProducer::class, $result); diff --git a/tests/Unit/Console/ConsumerCommandTest.php b/tests/Unit/Console/ConsumerCommandTest.php index 5a5a8fcf..c4737533 100644 --- a/tests/Unit/Console/ConsumerCommandTest.php +++ b/tests/Unit/Console/ConsumerCommandTest.php @@ -10,33 +10,6 @@ class ConsumerCommandTest extends LaravelTestCase { - protected function setUp(): void - { - parent::setUp(); - - config([ - 'kafka' => [ - 'topics' => [ - 'topic_key' => [ - 'topic_id' => 'topic_name', - 'consumer' => [ - 'consumer_group' => 'default', - 'offset_reset' => 'earliest', - 'handler' => ConsumerHandlerDummy::class, - 'timeout' => 123, - ], - ], - ], - ], - 'service' => [ - 'broker' => [ - 'connections' => 'test_kafka:6680', - 'auth' => [], - ], - ], - ]); - } - public function testItCallsCommandWithInvalidTopic(): void { // Set @@ -158,4 +131,31 @@ public function testItOverridesBrokerConnectionWhenCallingCommand(): void $this->artisan($command, $parameters); } + + protected function setUp(): void + { + parent::setUp(); + + config([ + 'kafka' => [ + 'topics' => [ + 'topic_key' => [ + 'topic_id' => 'topic_name', + 'consumer' => [ + 'consumer_group' => 'default', + 'offset_reset' => 'earliest', + 'handler' => ConsumerHandlerDummy::class, + 'timeout' => 123, + ], + ], + ], + ], + 'service' => [ + 'broker' => [ + 'connections' => 'test_kafka:6680', + 'auth' => [], + ], + ], + ]); + } } diff --git a/tests/Unit/Consumers/LowLevelTest.php b/tests/Unit/Consumers/LowLevelTest.php index b39ac2aa..e2347b60 100644 --- a/tests/Unit/Consumers/LowLevelTest.php +++ b/tests/Unit/Consumers/LowLevelTest.php @@ -36,7 +36,10 @@ public function testItShouldConsume(): void $consumerTopic = m::mock(ConsumerTopic::class); $message = new Message(); - $lowLevelConsumer = new LowLevel($consumerTopic, $consumerConfigOptions); + $lowLevelConsumer = new LowLevel( + $consumerTopic, + $consumerConfigOptions + ); // Expectations $consumerTopic->expects() diff --git a/tests/Unit/Middlewares/AvroSchemaDecoderTest.php b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php index 6c8d31b0..ba9edd29 100644 --- a/tests/Unit/Middlewares/AvroSchemaDecoderTest.php +++ b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php @@ -1,4 +1,5 @@ getAvroSchema() ->andReturn($avroSchema); - $avroSchemaDecoder = new AvroSchemaDecoder($clientFactory, $consumerConfigOptions); + $avroSchemaDecoder = new AvroSchemaDecoder( + $clientFactory, + $consumerConfigOptions + ); $result = $avroSchemaDecoder->process($consumerRecord, $closure); diff --git a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php index fcff2d6c..7fc33c0f 100644 --- a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php +++ b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php @@ -1,4 +1,5 @@ getAvroSchema(); $clientFactory = m::mock(ClientFactory::class); - $cachedSchemaRegistryClient = m::mock(CachedSchemaRegistryClient::class); - $schemaIdEncoder = m::mock(SchemaId::class, [$cachedSchemaRegistryClient]); + $cachedSchemaRegistryClient = m::mock( + CachedSchemaRegistryClient::class + ); + $schemaIdEncoder = m::mock( + SchemaId::class, + [$cachedSchemaRegistryClient] + ); $schema = new Schema(); - $parsedSchema = $schema->parse($avroSchema, '123', 'kafka-test-value', 'latest'); + $parsedSchema = $schema->parse( + $avroSchema, + '123', + 'kafka-test-value', + 'latest' + ); $record = $this->getRecord($parsedSchema->getAvroSchema()); $producerRecord = new ProducerRecord($record, 'kafka-test'); @@ -62,7 +76,11 @@ public function testShouldEncodeRecord() ->andReturn($encodedMessage); // Actions - $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder($schemaIdEncoder, $clientFactory, $producerConfigOptions); + $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder( + $schemaIdEncoder, + $clientFactory, + $producerConfigOptions + ); $result = $avroSchemaMixedEncoder->process($producerRecord, $closure); // Assertions @@ -93,6 +111,8 @@ private function getRecord(AvroSchema $avroSchema): string private function getSchemaFixture(): string { - return file_get_contents(__DIR__.'/../fixtures/schemas/sales_price.avsc'); + return file_get_contents( + __DIR__ . '/../fixtures/schemas/sales_price.avsc' + ); } } diff --git a/tests/Unit/ProducerTest.php b/tests/Unit/ProducerTest.php index 02808390..61f43155 100644 --- a/tests/Unit/ProducerTest.php +++ b/tests/Unit/ProducerTest.php @@ -41,7 +41,7 @@ public function testItShouldProduceRecordAsArrayThroughMiddlewareQueue(): void $topic, $broker ); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations @@ -85,7 +85,7 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations @@ -134,7 +134,7 @@ public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): vo 500, 1 ); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations @@ -187,7 +187,7 @@ public function testShouldBuildDispatcher(): void 1 ); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations @@ -236,7 +236,7 @@ public function testShouldBuildDispatcherWithConfigOptions(): void 500, 1 ); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations From 727d31f2e9724e9ea37bbd5114f464a6e55cb63a Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 14:21:32 -0300 Subject: [PATCH 131/152] style: supress phpcs typehing warning Signed-off-by: hcdias --- src/Console/ConfigOptionsCommand.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index d592a8b1..52d62dde 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -11,16 +11,19 @@ class ConfigOptionsCommand extends BaseCommand { /** * @var {inheritdoc} + * @phpcsSuppress SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingAnyTypeHint */ protected $name = 'kafka:consume-config-class'; /** * @var {inheritdoc} + * @phpcsSuppress SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingAnyTypeHint */ protected $description = 'Consumes something with a based class config'; /** * @var {inheritdoc} + * @phpcsSuppress SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingAnyTypeHint */ protected $signature = 'kafka:consume-config-class {handler : handler.} From e1d7e89bb5a9083c313f540d4cd4394159445402 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 14:39:10 -0300 Subject: [PATCH 132/152] style: supress phpcs typehing warning Signed-off-by: hcdias --- src/Connectors/AbstractConfig.php | 7 ++++++- src/Connectors/Consumer/Config.php | 2 +- tests/Unit/Connectors/Producer/ConnectorTest.php | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index b71c683c..5b084abc 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -7,6 +7,11 @@ abstract class AbstractConfig { + /** + * @var string[] + */ + protected array $rules; + protected function getBrokerConfig(string $servicesFile): array { if (!$brokerConfig = config($servicesFile . '.broker')) { @@ -15,7 +20,7 @@ protected function getBrokerConfig(string $servicesFile): array ); } - return $brokerConfig; + return $brokerConfig->all(); } protected function getSchemaConfig(string $servicesFile): array diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index d13976be..1e870981 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -19,7 +19,7 @@ class Config extends AbstractConfig { /** - * @var array + * @var string[] */ protected array $rules = [ 'topic' => 'required', diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index 2869318b..650dbd02 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -34,6 +34,7 @@ public function testItShouldMakeSetup(): void $connector = new Connector(); $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface { + /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */ public function success(Message $message): void { } @@ -83,6 +84,7 @@ public function testItShouldMakeSetupWithoutHandleResponse(): void $connector = new Connector(); $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface { + /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */ public function success(Message $message): void { } From 5954a6ad82647a5f3687d527b73e8acaab1cbe63 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 15:23:31 -0300 Subject: [PATCH 133/152] chore: remove call to inexistent method --- src/Connectors/AbstractConfig.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index 5b084abc..3ed44502 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -20,7 +20,7 @@ protected function getBrokerConfig(string $servicesFile): array ); } - return $brokerConfig->all(); + return $brokerConfig; } protected function getSchemaConfig(string $servicesFile): array From edd632195a0b0e91946bae0f161db15fc16c66e1 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 15:30:05 -0300 Subject: [PATCH 134/152] chore: supress psalm alert --- src/Connectors/AbstractConfig.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index 3ed44502..ef26cbf5 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -12,6 +12,9 @@ abstract class AbstractConfig */ protected array $rules; + /** + * @psalm-suppress InvalidReturnStatement + */ protected function getBrokerConfig(string $servicesFile): array { if (!$brokerConfig = config($servicesFile . '.broker')) { From 0339538b4215615d85839670c90b5feec5fdb207 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 15:55:36 -0300 Subject: [PATCH 135/152] tests: remove configManager related tests --- tests/Unit/Connectors/Consumer/ManagerTest.php | 18 ------------------ .../Unit/Middlewares/Handler/ProducerTest.php | 16 ---------------- tests/Unit/Producer/PollTest.php | 4 ++-- 3 files changed, 2 insertions(+), 36 deletions(-) diff --git a/tests/Unit/Connectors/Consumer/ManagerTest.php b/tests/Unit/Connectors/Consumer/ManagerTest.php index 230513b2..52e3bcc1 100644 --- a/tests/Unit/Connectors/Consumer/ManagerTest.php +++ b/tests/Unit/Connectors/Consumer/ManagerTest.php @@ -184,22 +184,4 @@ function () use ($messages, &$count, $exception) { $runner->handleMessage(); $runner->handleMessage(); } - - protected function setUp(): void - { - parent::setUp(); - - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'connections' => 'kafka:2019', - 'topic' => 'topic_key', - 'broker' => 'default', - 'offset_reset' => 'earliest', - 'offset' => 0, - 'timeout' => 30, - 'handler' => ConsumerHandlerDummy::class, - 'middlewares' => [], - 'consumer_group' => 'consumer-id', - ]); - } } diff --git a/tests/Unit/Middlewares/Handler/ProducerTest.php b/tests/Unit/Middlewares/Handler/ProducerTest.php index df2750e6..561a7c78 100644 --- a/tests/Unit/Middlewares/Handler/ProducerTest.php +++ b/tests/Unit/Middlewares/Handler/ProducerTest.php @@ -37,20 +37,4 @@ public function testItShouldSendMessageToKafkaBroker(): void $producerHandler = new Producer($producerTopic, $poll, 1); $producerHandler->process($record, $closure); } - - protected function setUp(): void - { - parent::setUp(); - - $configManager = new ProducerConfigManager(); - $configManager->set([ - 'topic_id' => 'topic_name', - 'timeout' => 4000, - 'is_async' => true, - 'max_poll_records' => 500, - 'flush_attempts' => 10, - 'required_acknowledgment' => true, - 'partition' => 0, - ]); - } } diff --git a/tests/Unit/Producer/PollTest.php b/tests/Unit/Producer/PollTest.php index 0576cd6c..ba216928 100644 --- a/tests/Unit/Producer/PollTest.php +++ b/tests/Unit/Producer/PollTest.php @@ -68,8 +68,8 @@ public function testShouldThrowExceptionWhenFlushFailed(): void ->poll(0); $kafkaProducer->expects() - ->flush(1000) - ->times(3) + ->flush(100) + ->times(10) ->andReturn(1); $this->expectException(RuntimeException::class); From d026b4a61e7b1b08f42886104073800979ced8c7 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 16:14:30 -0300 Subject: [PATCH 136/152] gha: update kafka broker connection --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index e57a08ef..66b85e9d 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -18,7 +18,7 @@ jobs: - '8.1' - '8.2' env: - KAFKA_BROKER_CONNECTIONS: 'localhost:9092' + KAFKA_BROKER_CONNECTIONS: 'kafka:9092' services: zookeeper: image: bitnami/zookeeper From c0560340692e97aa0f5316ada6f7893e5cdcf0b1 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 18:06:43 -0300 Subject: [PATCH 137/152] gha: undo update kafka broker connection --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 66b85e9d..e57a08ef 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -18,7 +18,7 @@ jobs: - '8.1' - '8.2' env: - KAFKA_BROKER_CONNECTIONS: 'kafka:9092' + KAFKA_BROKER_CONNECTIONS: 'localhost:9092' services: zookeeper: image: bitnami/zookeeper From 9d5392edba8e935f7161dcffb71fe34a822a79fb Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 11:49:20 -0300 Subject: [PATCH 138/152] fix(tests): get connection by env --- tests/Integration/ProducerTest.php | 3 ++- tests/Integration/ProducerWithAvroTest.php | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 8fbe5305..39cedbfd 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -206,7 +206,8 @@ private function haveFourProducedMessages(): void private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { - $broker = new Broker('kafka:9092', new None()); + $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); + $broker = new Broker($connections, new None()); return new ProducerConfigOptions( $topicId, diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index 54793d98..2a83cca3 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -170,7 +170,8 @@ private function setLogExpectationsFor(string $message): void private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { - $broker = new Broker('kafka:9092', new None()); + $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); + $broker = new Broker($connections, new None()); return new ProducerConfigOptions( $topicId, From a10f0d60f580dac61d415251966f11ff057284ee Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 11:56:13 -0300 Subject: [PATCH 139/152] test(producer): add expect not to perform assertion --- tests/Integration/ProducerWithAvroTest.php | 1 + tests/Integration/ProducerWithConfigOptionsTest.php | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index 2a83cca3..09237bfc 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -41,6 +41,7 @@ public function testShouldRunAProducerMessagesWithAAvroSchema(): void // I expect that $this->myMessagesHaveBeenLogged(); + $this->expectNotToPerformAssertions(); } protected function haveAHandlerConfigured(): void diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 4e256a42..ec37a620 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -31,6 +31,7 @@ public function testShouldRunAProducerMessagesWithConfigOptions(): void // I Expect That $this->myMessagesHaveBeenLogged(); + $this->expectNotToPerformAssertions(); // When I $this->runTheConsumer(); From 377b2e44d48ea3d00d005539810d70c87ebf7a27 Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 12:02:25 -0300 Subject: [PATCH 140/152] fix(tests): enable to get broker connection from env --- config/service.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/service.php b/config/service.php index 64f81077..7eacd41f 100644 --- a/config/service.php +++ b/config/service.php @@ -19,7 +19,7 @@ 'password' => 'PASSWORD', ], 'broker' => [ - 'connections' => 'kafka:9092', + 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), 'auth' => [ 'type' => 'ssl', // ssl and none 'ca' => storage_path('ca.pem'), From b792fe0eadc1e455b503bbd11e306be7bdd4ec3c Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 12:11:43 -0300 Subject: [PATCH 141/152] fix(tests): enable to get broker connection from env --- tests/Unit/Connectors/Producer/ConnectorTest.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index 650dbd02..dc2dfe96 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -29,7 +29,8 @@ public function testItShouldMakeSetup(): void m::mock(KafkaProducer::class) ); - $broker = new Broker('kafka:9092', new None()); + $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); + $broker = new Broker($connections, new None()); $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); @@ -79,7 +80,8 @@ public function testItShouldMakeSetupWithoutHandleResponse(): void m::mock(KafkaProducer::class) ); - $broker = new Broker('kafka:9092', new None()); + $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); + $broker = new Broker($connections, new None()); $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); From c0c368e4827f34a5b960b0631cefdb5057815212 Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 12:17:05 -0300 Subject: [PATCH 142/152] fix(tests): adjust mock expectations --- tests/Unit/Connectors/Producer/ConnectorTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index dc2dfe96..6652a0e7 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -52,7 +52,7 @@ public function failed(Message $message): void ->withAnyArgs(); $conf->expects() - ->set('metadata.broker.list', 0); + ->set('metadata.broker.list', $connections); $producerConfigOptions->expects() ->getBroker() @@ -102,7 +102,7 @@ public function failed(Message $message): void ->never(); $conf->expects() - ->set('metadata.broker.list', 0); + ->set('metadata.broker.list', $connections); $producerConfigOptions->expects() ->getBroker() From 0428873db29437bc08f29f88c12428d3be871526 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 16 Oct 2023 15:25:55 -0300 Subject: [PATCH 143/152] chore: add max poll interval on configOptions --- config/kafka.php | 4 ++++ src/Connectors/Consumer/HighLevel.php | 4 ++-- src/Connectors/Consumer/LowLevel.php | 4 ++-- src/TopicHandler/ConfigOptions/Consumer.php | 15 ++++++++++++++- .../ConfigOptions/Factories/ConsumerFactory.php | 5 +++++ tests/Unit/Console/ConsumerCommandTest.php | 2 +- 6 files changed, 28 insertions(+), 6 deletions(-) diff --git a/config/kafka.php b/config/kafka.php index 3e7f0764..4ffe7459 100644 --- a/config/kafka.php +++ b/config/kafka.php @@ -44,6 +44,10 @@ // An array of middlewares applied only for this consumer_group 'middlewares' => [], + + // A max interval for consumer to make poll calls. That means: how much + // time we need to wait for poll calls until consider the consumer has inactive. + 'max_poll_interval_ms' => 300000, ], 'producer' => [ diff --git a/src/Connectors/Consumer/HighLevel.php b/src/Connectors/Consumer/HighLevel.php index 9c832ae0..ebdd09de 100644 --- a/src/Connectors/Consumer/HighLevel.php +++ b/src/Connectors/Consumer/HighLevel.php @@ -14,7 +14,7 @@ class HighLevel implements ConnectorInterface public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface { $conf = $this->getConf($configOptions); - $maxPollIntervalMs = (int) $configOptions->getTimeout(); + $maxPollIntervalMs = $configOptions->getMaxPollInterval(); $conf->set('group.id', $configOptions->getConsumerGroup()); $conf->set('auto.offset.reset', $configOptions->getOffsetReset()); if (!$autoCommit) { @@ -22,7 +22,7 @@ public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): Con } $conf->set( 'max.poll.interval.ms', - $maxPollIntervalMs ?: 300000 + $maxPollIntervalMs ); $consumer = app(KafkaConsumer::class, ['conf' => $conf]); diff --git a/src/Connectors/Consumer/LowLevel.php b/src/Connectors/Consumer/LowLevel.php index 30f4c277..da3c1360 100644 --- a/src/Connectors/Consumer/LowLevel.php +++ b/src/Connectors/Consumer/LowLevel.php @@ -15,10 +15,10 @@ class LowLevel implements ConnectorInterface public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface { $conf = $this->getConf(); - $maxPollIntervalMs = (int) $configOptions->getTimeout(); + $maxPollIntervalMs = $configOptions->getMaxPollInterval(); $conf->set( 'max.poll.interval.ms', - $maxPollIntervalMs ?: 300000 + $maxPollIntervalMs ); $conf->set('group.id', $configOptions->getConsumerGroup()); if (!$autoCommit) { diff --git a/src/TopicHandler/ConfigOptions/Consumer.php b/src/TopicHandler/ConfigOptions/Consumer.php index e2caa016..2019ed68 100644 --- a/src/TopicHandler/ConfigOptions/Consumer.php +++ b/src/TopicHandler/ConfigOptions/Consumer.php @@ -38,6 +38,8 @@ class Consumer private ?int $offset = null; + private int $maxPollInterval = 300000; + public function __construct( string $topicId, Broker $broker, @@ -50,7 +52,8 @@ public function __construct( int $timeout = 1000, bool $autoCommit = true, bool $commitASync = true, - string $offsetReset = 'smallest' + string $offsetReset = 'smallest', + int $maxPollInterval = 300000 ) { $this->broker = $broker; $this->middlewares = $middlewares; @@ -64,6 +67,7 @@ public function __construct( $this->autoCommit = $autoCommit; $this->commitASync = $commitASync; $this->offsetReset = $offsetReset; + $this->maxPollInterval = $maxPollInterval; } public function getTimeout(): int @@ -147,4 +151,13 @@ public function getOffset(): ?int { return $this->offset; } + public function getMaxPollInterval(): int + { + return $this->maxPollInterval; + } + + public function setMaxPollInterval(int $maxPollInterval): void + { + $this->maxPollInterval = $maxPollInterval; + } } diff --git a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php index 2d28a28c..2f0896fb 100644 --- a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php +++ b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php @@ -53,6 +53,11 @@ private static function convertConfigAttributes(array $consumerConfig): array $consumerConfig['offsetReset'] = $consumerConfig['offset_reset']; } + + if (isset($consumerConfig['max_poll_interval_ms'])) { + $consumerConfig['maxPollInterval'] = $consumerConfig['max_poll_interval_ms']; + } + return $consumerConfig; } } diff --git a/tests/Unit/Console/ConsumerCommandTest.php b/tests/Unit/Console/ConsumerCommandTest.php index c4737533..fd26cb27 100644 --- a/tests/Unit/Console/ConsumerCommandTest.php +++ b/tests/Unit/Console/ConsumerCommandTest.php @@ -131,7 +131,6 @@ public function testItOverridesBrokerConnectionWhenCallingCommand(): void $this->artisan($command, $parameters); } - protected function setUp(): void { parent::setUp(); @@ -146,6 +145,7 @@ protected function setUp(): void 'offset_reset' => 'earliest', 'handler' => ConsumerHandlerDummy::class, 'timeout' => 123, + 'max_poll_interval_ms' => 300000, ], ], ], From c1a0490364a50654b7f5350569e34503864a0531 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 16 Oct 2023 16:56:50 -0300 Subject: [PATCH 144/152] chore: add max poll interval on toArray from consumer --- src/TopicHandler/ConfigOptions/Consumer.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/TopicHandler/ConfigOptions/Consumer.php b/src/TopicHandler/ConfigOptions/Consumer.php index 2019ed68..b89dca15 100644 --- a/src/TopicHandler/ConfigOptions/Consumer.php +++ b/src/TopicHandler/ConfigOptions/Consumer.php @@ -138,6 +138,7 @@ public function toArray(): array 'auto_commit' => $this->isAutoCommit(), 'commit_async' => $this->isCommitASync(), 'offset_reset' => $this->getOffsetReset(), + 'max_poll_interval' => $this->getMaxPollInterval(), ]; if ($avroSchema = $this->getAvroSchema()) { From 10721cdd86e38dfc0bd55270e2b39f61a51dc2c7 Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 17 Oct 2023 16:41:09 -0300 Subject: [PATCH 145/152] chore: refact middlewares build from consumer --- src/Connectors/Consumer/Config.php | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 1e870981..673e36dd 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -84,26 +84,23 @@ public function make(array $options, array $arguments): Consumer /** * @psalm-suppress InvalidReturnStatement */ - private function getTopicConfig(string $configName, string $topicId): array + protected function getTopicConfig(string $configName, string $topicId): array { $topicConfig = config($configName . '.topics.' . $topicId); if (!$topicConfig) { throw new ConfigurationException("Topic '{$topicId}' not found"); } - $topicConfig['middlewares'] = $this->getMiddlewares( - $configName, - $topicConfig + $globalMiddlewares = config( + 'kafka.middlewares.consumer', + [] ); - return $topicConfig; - } - - private function getMiddlewares(string $configName, array $topicConfig): array - { - return array_merge( - config($configName . '.middlewares.consumer', []), + $topicConfig['middlewares'] = array_merge( + $globalMiddlewares, $topicConfig['consumer']['middlewares'] ?? [] ); + + return $topicConfig; } } From 97f2e19e5f5ed514f4542e2c0f41d10f634bf1e1 Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 17 Oct 2023 16:42:11 -0300 Subject: [PATCH 146/152] chore: remove unused methods --- src/Connectors/Producer/Config.php | 43 ------------------------------ src/Producer.php | 10 ------- 2 files changed, 53 deletions(-) diff --git a/src/Connectors/Producer/Config.php b/src/Connectors/Producer/Config.php index e3cf8e80..5c4edec5 100644 --- a/src/Connectors/Producer/Config.php +++ b/src/Connectors/Producer/Config.php @@ -2,11 +2,7 @@ namespace Metamorphosis\Connectors\Producer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\AbstractConfig; -use Metamorphosis\Exceptions\ConfigurationException; -use Metamorphosis\ProducerConfigManager; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; class Config extends AbstractConfig { @@ -38,43 +34,4 @@ class Config extends AbstractConfig 'partition' => RD_KAFKA_PARTITION_UA, 'ssl_verify' => false, ]; - - public function make(ProducerConfigOptions $configOptions): AbstractConfigManager - { - $configManager = app(ProducerConfigManager::class); - $configManager->set($configOptions->toArray()); - - return $configManager; - } - - public function makeByTopic(string $topicId): AbstractConfigManager - { - $topicConfig = $this->getTopicConfig($topicId); - $topicConfig['middlewares'] = $topicConfig['producer']['middlewares'] ?? []; - $brokerConfig = $this->getBrokerConfig('service'); - $schemaConfig = $this->getSchemaConfig('service'); - $config = array_merge($topicConfig, $brokerConfig, $schemaConfig); - - $this->validate($config); - $config = array_merge($this->default, $config); - - $configManager = app(ProducerConfigManager::class); - $configManager->set($config); - - return $configManager; - } - - private function getTopicConfig(string $topicId): array - { - $topicConfig = array_merge( - config('kafka.topics.' . $topicId, []), - config('kafka.topics.' . $topicId . '.producer', []) - ); - if (!$topicConfig) { - throw new ConfigurationException("Topic '{$topicId}' not found"); - } - $topicConfig['topic'] = $topicId; - - return $topicConfig; - } } diff --git a/src/Producer.php b/src/Producer.php index b3211f7c..26363ffb 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -8,7 +8,6 @@ use Metamorphosis\Middlewares\Handler\Producer as ProducerMiddleware; use Metamorphosis\Producer\Poll; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; -use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Metamorphosis\TopicHandler\Producer\HandlerInterface; class Producer @@ -73,13 +72,4 @@ public function getProducerMiddleware( compact('topic', 'poll', 'partition') ); } - - private function getConfigManager(HandlerInterface $producerHandler): AbstractConfigManager - { - if ($producerHandler instanceof AbstractProducer) { - return $this->config->make($producerHandler->getConfigOptions()); - } - - return $this->config->makeByTopic($producerHandler->getTopic()); - } } From 549e050d96fa5299fe286ae0b52c173ec61a3988 Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 17 Oct 2023 16:43:57 -0300 Subject: [PATCH 147/152] chore: fix kafka parameter --- src/TopicHandler/ConfigOptions/Consumer.php | 2 +- tests/Unit/TopicHandler/ConfigOptions/ConsumerTest.php | 3 ++- .../ConfigOptions/Factories/ConsumerFactoryTest.php | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/TopicHandler/ConfigOptions/Consumer.php b/src/TopicHandler/ConfigOptions/Consumer.php index b89dca15..2d1e173a 100644 --- a/src/TopicHandler/ConfigOptions/Consumer.php +++ b/src/TopicHandler/ConfigOptions/Consumer.php @@ -138,7 +138,7 @@ public function toArray(): array 'auto_commit' => $this->isAutoCommit(), 'commit_async' => $this->isCommitASync(), 'offset_reset' => $this->getOffsetReset(), - 'max_poll_interval' => $this->getMaxPollInterval(), + 'max_poll_interval_ms' => $this->getMaxPollInterval(), ]; if ($avroSchema = $this->getAvroSchema()) { diff --git a/tests/Unit/TopicHandler/ConfigOptions/ConsumerTest.php b/tests/Unit/TopicHandler/ConfigOptions/ConsumerTest.php index 481127d1..1aa4cc32 100644 --- a/tests/Unit/TopicHandler/ConfigOptions/ConsumerTest.php +++ b/tests/Unit/TopicHandler/ConfigOptions/ConsumerTest.php @@ -25,7 +25,7 @@ public function testShouldConvertConfigOptionsToArray(): void [], 200, false, - true + true, ); $expected = [ @@ -41,6 +41,7 @@ public function testShouldConvertConfigOptionsToArray(): void 'auto_commit' => false, 'commit_async' => true, 'offset_reset' => 'smallest', + 'max_poll_interval_ms' => 300000, ]; // Actions diff --git a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php index 02aa3368..8a84e7dd 100644 --- a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php +++ b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php @@ -72,6 +72,7 @@ public function testShouldMakeConfigOptionWithAvroSchema(): void 'auto_commit' => true, 'commit_async' => true, 'offset_reset' => 'earliest', + 'max_poll_interval_ms' => 300000, ]; // Actions $result = ConsumerFactory::make( From 468b5c808535f3b091a35c180a81a1fd5fecce68 Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 17 Oct 2023 18:10:21 -0300 Subject: [PATCH 148/152] tests: remove config test --- tests/Unit/Connectors/Producer/ConfigTest.php | 122 ------------------ 1 file changed, 122 deletions(-) delete mode 100644 tests/Unit/Connectors/Producer/ConfigTest.php diff --git a/tests/Unit/Connectors/Producer/ConfigTest.php b/tests/Unit/Connectors/Producer/ConfigTest.php deleted file mode 100644 index d60567e1..00000000 --- a/tests/Unit/Connectors/Producer/ConfigTest.php +++ /dev/null @@ -1,122 +0,0 @@ - 10000, - 'is_async' => true, - 'required_acknowledgment' => true, - 'max_poll_records' => 500, - 'flush_attempts' => 10, - 'partition' => -1, - 'topic' => 'default', - 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), - 'auth' => [ - 'type' => 'ssl', - 'ca' => base_path('storage/ca.pem'), - 'certificate' => base_path('storage/kafka.cert'), - 'key' => base_path('storage/kafka.key'), - ], - ]; - - // Actions - $result = $config->makeByTopic($topicId); - - // Assertions - $this->assertArraySubset($expected, $result->get()); - } - - public function testShouldNotSetRuntimeConfigWhenKafkaConfigIsInvalid(): void - { - // Set - config(['kafka.topics.default.producer.required_acknowledgment' => 3]); - $config = new Config(); - $topicId = 'default'; - - // Actions - $this->expectException(ConfigurationException::class); - $result = $config->makeByTopic($topicId); - - // Assertions - $this->assertEmpty($result->get()); - } - - public function testShouldNotOverrideDefaultParametersWhenConfigIsSet(): void - { - // Set - config(['kafka.topics.default.producer.max_poll_records' => 3000]); - $config = new Config(); - $topicId = 'default'; - - $expected = [ - 'timeout' => 10000, - 'is_async' => true, - 'required_acknowledgment' => true, - 'max_poll_records' => 3000, - 'flush_attempts' => 10, - 'topic' => 'default', - 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), - 'auth' => [ - 'type' => 'ssl', - 'ca' => base_path('storage/ca.pem'), - 'certificate' => base_path('storage/kafka.cert'), - 'key' => base_path('storage/kafka.key'), - ], - ]; - - // Actions - $result = $config->makeByTopic($topicId); - - // Assertions - $this->assertArraySubset($expected, $result->get()); - } - - public function testShouldOverrideDefaultParametersWhenConfigOptionsExists(): void - { - // Set - config(['kafka.topics.default.producer.max_poll_records' => 3000]); - $config = new Config(); - $broker = new Broker( - 'kafka:9092', - new SaslSsl('PLAIN', 'USERNAME', 'PASSWORD') - ); - $configOptions = new ProducerConfigOptions('TOPIC-ID', $broker); - - $expected = [ - 'topic_id' => 'TOPIC-ID', - 'connections' => 'kafka:9092', - 'auth' => [ - 'type' => 'sasl_ssl', - 'mechanisms' => 'PLAIN', - 'username' => 'USERNAME', - 'password' => 'PASSWORD', - ], - 'timeout' => 1000, - 'is_async' => true, - 'required_acknowledgment' => false, - 'max_poll_records' => 500, - 'flush_attempts' => 10, - ]; - - // Actions - $result = $config->make($configOptions); - - // Assertions - $this->assertArraySubset($expected, $result->get()); - } -} From 2f781c06e517fc968560fcf5daa77c0b4c593387 Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 17 Oct 2023 18:32:16 -0300 Subject: [PATCH 149/152] chore: remove merge from consumer config middlewares --- src/Connectors/Consumer/Config.php | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 673e36dd..0be88a80 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -91,16 +91,11 @@ protected function getTopicConfig(string $configName, string $topicId): array throw new ConfigurationException("Topic '{$topicId}' not found"); } - $globalMiddlewares = config( + $topicConfig['middlewares'] = config( 'kafka.middlewares.consumer', [] ); - $topicConfig['middlewares'] = array_merge( - $globalMiddlewares, - $topicConfig['consumer']['middlewares'] ?? [] - ); - return $topicConfig; } } From dc753793e1ecc2d4d5d11c41ca4cae99732f6e9b Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 17 Oct 2023 19:04:47 -0300 Subject: [PATCH 150/152] chore: change method visibility --- src/Connectors/Consumer/Config.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 0be88a80..f02c2c43 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -84,7 +84,7 @@ public function make(array $options, array $arguments): Consumer /** * @psalm-suppress InvalidReturnStatement */ - protected function getTopicConfig(string $configName, string $topicId): array + private function getTopicConfig(string $configName, string $topicId): array { $topicConfig = config($configName . '.topics.' . $topicId); if (!$topicConfig) { From 5b7bb455dbe028df70330bedafd100f3f4bc27a6 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 18 Oct 2023 11:57:08 -0300 Subject: [PATCH 151/152] chore: remove remaining ConfigManager classes --- src/AbstractConfigManager.php | 49 ------------------------ src/ProducerConfigManager.php | 27 ------------- tests/Unit/ConsumerConfigManagerTest.php | 0 tests/Unit/ManagerTest.php | 0 tests/Unit/ProducerConfigManagerTest.php | 47 ----------------------- 5 files changed, 123 deletions(-) delete mode 100644 src/AbstractConfigManager.php delete mode 100644 src/ProducerConfigManager.php delete mode 100644 tests/Unit/ConsumerConfigManagerTest.php delete mode 100644 tests/Unit/ManagerTest.php delete mode 100644 tests/Unit/ProducerConfigManagerTest.php diff --git a/src/AbstractConfigManager.php b/src/AbstractConfigManager.php deleted file mode 100644 index 8bce1114..00000000 --- a/src/AbstractConfigManager.php +++ /dev/null @@ -1,49 +0,0 @@ -setting; - } - - return Arr::get($this->setting, $key, $default); - } - - public function has(string $key): bool - { - return !is_null($this->get($key)); - } - - public function middlewares(): array - { - return $this->middlewares; - } - - protected function remove(string $key): void - { - unset($this->setting[$key]); - } -} diff --git a/src/ProducerConfigManager.php b/src/ProducerConfigManager.php deleted file mode 100644 index 59f3b35a..00000000 --- a/src/ProducerConfigManager.php +++ /dev/null @@ -1,27 +0,0 @@ -setting = $config; - - $middlewares = $this->get('middlewares', []); - $this->middlewares = []; - $this->remove('middlewares'); - - foreach ($middlewares as $middleware) { - $this->middlewares[] = is_string($middleware) - ? app( - $middleware, - ['configManager' => $this] - ) - : $middleware; - } - } -} diff --git a/tests/Unit/ConsumerConfigManagerTest.php b/tests/Unit/ConsumerConfigManagerTest.php deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/Unit/ManagerTest.php b/tests/Unit/ManagerTest.php deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/Unit/ProducerConfigManagerTest.php b/tests/Unit/ProducerConfigManagerTest.php deleted file mode 100644 index 26500f5d..00000000 --- a/tests/Unit/ProducerConfigManagerTest.php +++ /dev/null @@ -1,47 +0,0 @@ - [MiddlewareDummy::class], - 'handler' => AbstractProducer::class, - 'broker' => [ - 'default' => [ - 'connections' => 'kafka:9092', - ], - ], - 'topic_id' => 'kafka-test', - ]; - - $configManager = new ProducerConfigManager(); - - $expected = [ - 'handler' => AbstractProducer::class, - 'broker' => [ - 'default' => [ - 'connections' => 'kafka:9092', - ], - ], - 'topic_id' => 'kafka-test', - ]; - - // Actions - $configManager->set($config); - - // Expectations - $this->assertSame($expected, $configManager->get()); - $this->assertInstanceOf( - MiddlewareDummy::class, - current($configManager->middlewares()) - ); - } -} From 93a024e4a5449469d718723fe45bce841ca4ac80 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Thu, 18 Jul 2024 19:39:44 -0300 Subject: [PATCH 152/152] fix: adjust docs and code style Signed-off-by: Hugo Ferreira --- CHANGELOG.md | 5 ++--- docs/quick-usage.pt.md | 4 ++-- src/Connectors/AbstractConfig.php | 1 + src/Console/ConfigOptionsCommand.php | 4 ++-- src/Console/ConsumerCommand.php | 4 ++-- src/TopicHandler/Producer/AbstractProducer.php | 5 +---- tests/Unit/Console/ConsumerCommandTest.php | 1 + tests/Unit/Middlewares/AvroSchemaDecoderTest.php | 2 +- tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php | 2 +- 9 files changed, 13 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 342b1ef1..d3494cc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,11 +18,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed parameters and options override on Consumer\Config class -- Update instructions on contribute section +- Update instructions in the contributions section - Update project install section ### Changed -- Updated class from ConfigManager to ConfigOptions on unit tests -- Updated class from ConfigManager to ConfigOptions where any config request was made +- Updated ConfigManager class for ConfigOptions in unit tests and wherever any configuration requests are made - Consumer and Producer middlewares resolution diff --git a/docs/quick-usage.pt.md b/docs/quick-usage.pt.md index c3e14753..706847ce 100644 --- a/docs/quick-usage.pt.md +++ b/docs/quick-usage.pt.md @@ -1,4 +1,4 @@ -## Quick Usage Guide +## Guia Rápido - [Configurar usando arquivos](#config) - [Configurar usando objetos](#config-dto) @@ -55,7 +55,7 @@ return [ ]; ``` -### File `config/service.php` +### Arquivo `config/service.php` Esse arquivo possui as configurações de **broker** e **schema** utilizados. diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index ef26cbf5..0117b259 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -14,6 +14,7 @@ abstract class AbstractConfig /** * @psalm-suppress InvalidReturnStatement + * @throws ConfigurationException */ protected function getBrokerConfig(string $servicesFile): array { diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index 52d62dde..e9219f19 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -29,7 +29,7 @@ class ConfigOptionsCommand extends BaseCommand {handler : handler.} {--times= : Amount of messages to be consumed.}'; - public function handle() + public function handle(): void { $consumerHandler = app($this->argument('handler')); @@ -43,7 +43,7 @@ public function handle() $runner->run($this->option('times')); } - private function writeStartingConsumer(ConfigOptions $configOptions) + private function writeStartingConsumer(ConfigOptions $configOptions): void { $text = 'Starting consumer for topic: ' . $configOptions->getTopicId() . PHP_EOL; $text .= ' on consumer group: ' . $configOptions->getConsumerGroup() . PHP_EOL; diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index ded31d7f..aee7bc89 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -37,7 +37,7 @@ class ConsumerCommand extends BaseCommand {--config_name= : Change default name for laravel config file.} {--service_name= : Change default name for services config file.}'; - public function handle(Config $config) + public function handle(Config $config): void { $consumer = $config->make($this->option(), $this->argument()); @@ -49,7 +49,7 @@ public function handle(Config $config) $runner->run($this->option('times')); } - private function writeStartingConsumer(Consumer $consumer) + private function writeStartingConsumer(Consumer $consumer): void { $text = 'Starting consumer for topic: ' . $consumer->getTopicId() . PHP_EOL; $text .= ' on consumer group: ' . $consumer->getConsumerGroup() . PHP_EOL; diff --git a/src/TopicHandler/Producer/AbstractProducer.php b/src/TopicHandler/Producer/AbstractProducer.php index ecab7c93..304fc2dd 100644 --- a/src/TopicHandler/Producer/AbstractProducer.php +++ b/src/TopicHandler/Producer/AbstractProducer.php @@ -15,10 +15,7 @@ class AbstractProducer implements HandlerInterface protected ?string $key; - /** - * @var Producer - */ - protected $producer; + protected Producer $producer; public function __construct($record, Producer $configOptions, ?string $key = null) { diff --git a/tests/Unit/Console/ConsumerCommandTest.php b/tests/Unit/Console/ConsumerCommandTest.php index fd26cb27..9c6c9b24 100644 --- a/tests/Unit/Console/ConsumerCommandTest.php +++ b/tests/Unit/Console/ConsumerCommandTest.php @@ -131,6 +131,7 @@ public function testItOverridesBrokerConnectionWhenCallingCommand(): void $this->artisan($command, $parameters); } + protected function setUp(): void { parent::setUp(); diff --git a/tests/Unit/Middlewares/AvroSchemaDecoderTest.php b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php index ba9edd29..6af67efa 100644 --- a/tests/Unit/Middlewares/AvroSchemaDecoderTest.php +++ b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php @@ -19,7 +19,7 @@ class AvroSchemaDecoderTest extends LaravelTestCase { - public function testShouldDecodeRecord() + public function testShouldDecodeRecord(): void { $brokerOptions = new Broker('kafka:9092', new None()); $consumerConfigOptions = new ConsumerConfigOptions( diff --git a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php index 7fc33c0f..ab37c2da 100644 --- a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php +++ b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php @@ -19,7 +19,7 @@ class AvroSchemaMixedEncoderTest extends LaravelTestCase { - public function testShouldEncodeRecord() + public function testShouldEncodeRecord(): void { // Set $avroSchema = $this->getSchemaFixture();