From a83f2cc0167023f7fdb77b59d3a0f945c25b9984 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Mon, 31 Jan 2022 17:13:51 -0300 Subject: [PATCH 01/72] chore(configOptions): add ConfigOptions command to ServiceProvider --- src/MetamorphosisServiceProvider.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/MetamorphosisServiceProvider.php b/src/MetamorphosisServiceProvider.php index 687c739a..4938fbac 100644 --- a/src/MetamorphosisServiceProvider.php +++ b/src/MetamorphosisServiceProvider.php @@ -3,6 +3,7 @@ namespace Metamorphosis; use Illuminate\Support\ServiceProvider; +use Metamorphosis\Console\ConfigOptionsCommand; use Metamorphosis\Console\ConsumerCommand; use Metamorphosis\Console\ConsumerMakeCommand; use Metamorphosis\Console\MiddlewareMakeCommand; @@ -26,6 +27,7 @@ public function register() ConsumerMakeCommand::class, MiddlewareMakeCommand::class, ProducerMakeCommand::class, + ConfigOptionsCommand::class, ]); $this->app->bind('metamorphosis', function ($app) { From 7b0061f4e517838bb059c4d3baaa2c6d5e8c1033 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Mon, 31 Jan 2022 17:16:15 -0300 Subject: [PATCH 02/72] chore(configOptions): call new configOptions artisan command --- .../ProducerWithConfigOptionsTest.php | 41 +++---------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 3382b590..5f5017a0 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -24,14 +24,13 @@ public function testShouldRunAProducerMessagesWithConfigOptions(): void $this->haveAHandlerConfigured(); // I Expect That - $this->myMessagesHaveBeenProduced(); - $this->expectNotToPerformAssertions(); + //$this->myMessagesHaveBeenProduced(); // When I - $this->haveSomeRandomMessageProduced(); + //$this->haveSomeRandomMessageProduced(); // I Expect That - $this->myMessagesHaveBeenLogged(); + //$this->myMessagesHaveBeenLogged(); // When I $this->runTheConsumer(); @@ -41,39 +40,11 @@ protected function runTheConsumer(): void { $dummy = new MessageConsumer($this->consumerConfigOptions); $this->instance('\App\Kafka\Consumers\ConsumerOverride', $dummy); - config([ - 'kafka_new_config' => [ - 'brokers' => [ - 'override' => [ - 'connections' => env( - 'KAFKA_BROKER_CONNECTIONS', - 'kafka:9092' - ), - ], - ], - 'topics' => [ - 'default' => [ - 'broker' => 'override', - 'consumer' => [ - 'consumer_groups' => [ - 'test-consumer-group' => [ - 'handler' => '\App\Kafka\Consumers\ConsumerOverride', - 'offset_reset' => 'earliest', - ], - ], - ], - ], - ], - ], - ]); + $this->artisan( - 'kafka:consume', + 'kafka:consume-config-class', [ - 'topic' => 'default', - 'consumer_group' => 'test-consumer-group', - '--timeout' => 20000, - '--times' => 2, - '--config_name' => 'kafka_new_config', + 'handler' => '\\App\\Kafka\\Consumers\\ConsumerOverride', ] ); } From 574f219b81c38f9530c1ab5794fd606a680603ba Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Mon, 31 Jan 2022 17:19:05 -0300 Subject: [PATCH 03/72] chore(configOptions): add ConfigOptionsCommand --- src/Console/ConfigOptionsCommand.php | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 src/Console/ConfigOptionsCommand.php diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php new file mode 100644 index 00000000..ccd8bb37 --- /dev/null +++ b/src/Console/ConfigOptionsCommand.php @@ -0,0 +1,49 @@ +make($this->option(), $this->argument()); + +// $this->writeStartingConsumer($configManager); + + $config = $this->argument()['handler']; + + $manager = Factory::make($this->argument()['handler']); + + $runner = app(Runner::class, compact('manager')); + //$runner->run($manager->get('times')); + } + + private function writeStartingConsumer(AbstractConfigManager $configManager) + { + $text = 'Starting consumer for topic: '.$configManager->get('topic').PHP_EOL; + $text .= ' on consumer group: '.$configManager->get('consumer_group').PHP_EOL; + $text .= 'Connecting in '.$configManager->get('connections').PHP_EOL; + $text .= 'Running consumer..'; + + $this->output->writeln($text); + } +} From a0e26ea2553919cbef4b7fe9994271615d680d21 Mon Sep 17 00:00:00 2001 From: David Franca Date: Mon, 31 Jan 2022 17:55:13 -0300 Subject: [PATCH 04/72] chore: make config manager with config options --- src/Connectors/Consumer/Config.php | 8 ++++++++ src/Console/ConfigOptionsCommand.php | 13 ++++++------- tests/Integration/ProducerWithConfigOptionsTest.php | 6 +++--- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 306600aa..57b8b92d 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -39,6 +39,14 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; + public function makeWithConfigOptions(string $handlerClass): AbstractConfigManager + { + $configManager = app(ConsumerConfigManager::class); + $configManager->set(['handler' => $handlerClass]); + + return $configManager; + } + public function make(array $options, array $arguments): AbstractConfigManager { $configName = $options['config_name'] ?? 'kafka'; diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index ccd8bb37..37dab14c 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -3,6 +3,7 @@ use Illuminate\Console\Command as BaseCommand; use Metamorphosis\AbstractConfigManager; +use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; @@ -23,18 +24,16 @@ class ConfigOptionsCommand extends BaseCommand */ protected $signature = 'kafka:consume-config-class {handler : handler.}'; - public function handle() + public function handle(Config $config) { -// $configManager = $config->make($this->option(), $this->argument()); + $configManager = $config->makeWithConfigOptions($this->argument()['handler']); -// $this->writeStartingConsumer($configManager); + $this->writeStartingConsumer($configManager); - $config = $this->argument()['handler']; - - $manager = Factory::make($this->argument()['handler']); + $manager = Factory::make($configManager); $runner = app(Runner::class, compact('manager')); - //$runner->run($manager->get('times')); + $runner->run(2); } private function writeStartingConsumer(AbstractConfigManager $configManager) diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 5f5017a0..4a45d8e0 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -24,13 +24,13 @@ public function testShouldRunAProducerMessagesWithConfigOptions(): void $this->haveAHandlerConfigured(); // I Expect That - //$this->myMessagesHaveBeenProduced(); + $this->myMessagesHaveBeenProduced(); // When I - //$this->haveSomeRandomMessageProduced(); + $this->haveSomeRandomMessageProduced(); // I Expect That - //$this->myMessagesHaveBeenLogged(); + $this->myMessagesHaveBeenLogged(); // When I $this->runTheConsumer(); From 1407d9f008c3fd39c5fe634e54a94819819994b4 Mon Sep 17 00:00:00 2001 From: David Franca Date: Mon, 31 Jan 2022 18:44:46 -0300 Subject: [PATCH 05/72] chore: replace config manager to config options --- src/Authentication/Factory.php | 16 +++---- src/Authentication/SASLAuthentication.php | 26 +++------- src/Authentication/SSLAuthentication.php | 26 +++------- src/Connectors/Consumer/Config.php | 4 +- .../Consumer/ConnectorInterface.php | 4 +- src/Connectors/Consumer/Factory.php | 48 ++++++++----------- src/Connectors/Consumer/HighLevel.php | 21 ++++---- src/Connectors/Consumer/LowLevel.php | 31 +++++------- src/Connectors/Producer/Connector.php | 9 ++-- src/Console/ConfigOptionsCommand.php | 23 +++++---- src/Consumers/LowLevel.php | 8 ++-- src/Producer.php | 30 ++++-------- src/Producer/Poll.php | 15 +++--- .../ProducerWithConfigOptionsTest.php | 1 + 14 files changed, 107 insertions(+), 155 deletions(-) diff --git a/src/Authentication/Factory.php b/src/Authentication/Factory.php index 67e3e1fa..c679209a 100644 --- a/src/Authentication/Factory.php +++ b/src/Authentication/Factory.php @@ -4,6 +4,8 @@ use Metamorphosis\AbstractConfigManager; use Metamorphosis\Exceptions\AuthenticationException; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\AuthInterface; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\Conf; class Factory @@ -14,9 +16,9 @@ class Factory private const TYPE_NONE = 'none'; - public static function authenticate(Conf $conf, AbstractConfigManager $configManager): void + public static function authenticate(Conf $conf, AuthInterface $configOptions): void { - $type = $configManager->get('auth.type'); + $type = $configOptions->getType(); switch ($type) { case null: case self::TYPE_NONE: @@ -24,17 +26,11 @@ public static function authenticate(Conf $conf, AbstractConfigManager $configMan break; case self::TYPE_SSL: - app( - SSLAuthentication::class, - compact('conf', 'configManager') - ); + app(SSLAuthentication::class, compact('conf', 'configOptions')); break; case self::TYPE_SASL_SSL: - app( - SASLAuthentication::class, - compact('conf', 'configManager') - ); + app(SASLAuthentication::class, compact('conf', 'configOptions')); break; default: diff --git a/src/Authentication/SASLAuthentication.php b/src/Authentication/SASLAuthentication.php index c0ebb42e..9b2fb0d0 100644 --- a/src/Authentication/SASLAuthentication.php +++ b/src/Authentication/SASLAuthentication.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Authentication; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\SaslSsl; use RdKafka\Conf; class SASLAuthentication implements AuthenticationInterface @@ -11,35 +11,23 @@ class SASLAuthentication implements AuthenticationInterface private AbstractConfigManager $configManager; - public function __construct(Conf $conf, AbstractConfigManager $configManager) + public function __construct(Conf $conf, SaslSsl $config) { $this->conf = $conf; - $this->configManager = $configManager; + $this->config = $config; $this->authenticate(); } private function authenticate(): void { - $this->conf->set( - 'security.protocol', - $this->configManager->get('auth.type') - ); + $this->conf->set('security.protocol', $this->config->getType()); // The mechanisms key is optional when configuring this kind of authentication // If the user does not specify the mechanism, the default will be 'PLAIN'. // But, to make config more clear, we are asking the user every time. - $this->conf->set( - 'sasl.mechanisms', - $this->configManager->get('auth.mechanisms') - ); - $this->conf->set( - 'sasl.username', - $this->configManager->get('auth.username') - ); - $this->conf->set( - 'sasl.password', - $this->configManager->get('auth.password') - ); + $this->conf->set('sasl.mechanisms', $this->config->getMechanisms()); + $this->conf->set('sasl.username', $this->config->getUsername()); + $this->conf->set('sasl.password', $this->config->getPassword()); } } diff --git a/src/Authentication/SSLAuthentication.php b/src/Authentication/SSLAuthentication.php index 246ab5e7..750552e8 100644 --- a/src/Authentication/SSLAuthentication.php +++ b/src/Authentication/SSLAuthentication.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Authentication; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\Ssl; use RdKafka\Conf; class SSLAuthentication implements AuthenticationInterface @@ -11,31 +11,19 @@ class SSLAuthentication implements AuthenticationInterface private AbstractConfigManager $configManager; - public function __construct(Conf $conf, AbstractConfigManager $configManager) + public function __construct(Conf $conf, Ssl $configSsl) { $this->conf = $conf; - $this->configManager = $configManager; + $this->configSsl = $configSsl; $this->authenticate(); } private function authenticate(): void { - $this->conf->set( - 'security.protocol', - $this->configManager->get('auth.type') - ); - $this->conf->set( - 'ssl.ca.location', - $this->configManager->get('auth.ca') - ); - $this->conf->set( - 'ssl.certificate.location', - $this->configManager->get('auth.certificate') - ); - $this->conf->set( - 'ssl.key.location', - $this->configManager->get('auth.key') - ); + $this->conf->set('security.protocol', $this->configSsl->getType()); + $this->conf->set('ssl.ca.location', $this->configSsl->getCa()); + $this->conf->set('ssl.certificate.location', $this->configSsl->getCertificate()); + $this->conf->set('ssl.key.location', $this->configSsl->getKey()); } } diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 57b8b92d..9216cf18 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -39,10 +39,10 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; - public function makeWithConfigOptions(string $handlerClass): AbstractConfigManager + public function makeWithConfigOptions(string $handlerClass, ?int $times = null): AbstractConfigManager { $configManager = app(ConsumerConfigManager::class); - $configManager->set(['handler' => $handlerClass]); + $configManager->set(['handler' => $handlerClass], ['times' => $times]); return $configManager; } diff --git a/src/Connectors/Consumer/ConnectorInterface.php b/src/Connectors/Consumer/ConnectorInterface.php index 3163fca6..738a6d0e 100644 --- a/src/Connectors/Consumer/ConnectorInterface.php +++ b/src/Connectors/Consumer/ConnectorInterface.php @@ -2,10 +2,10 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use Metamorphosis\Consumers\ConsumerInterface; interface ConnectorInterface { - public function getConsumer(bool $autoCommit, AbstractConfigManager $configManager): ConsumerInterface; + public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface; } diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index bc7258e9..f48fe56d 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -5,6 +5,7 @@ use Metamorphosis\AbstractConfigManager; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Middlewares\Handler\Dispatcher; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; /** * This factory will determine what kind of connector will be used. @@ -13,44 +14,33 @@ */ class Factory { - public static function make(AbstractConfigManager $configManager): Manager + public static function make(ConfigOptions $configOptions): Manager { - $autoCommit = $configManager->get('auto_commit', true); - $commitAsync = $configManager->get('commit_async', true); - - $consumer = self::getConsumer($autoCommit, $configManager); - $handler = app($configManager->get('handler')); - - $dispatcher = self::getMiddlewareDispatcher( - $configManager->middlewares() - ); - - return new Manager( - $consumer, - $handler, - $dispatcher, - $autoCommit, - $commitAsync - ); + $autoCommit = $configOptions->isAutoCommit(); + $commitAsync = $configOptions->isCommitASync(); + + $consumer = self::getConsumer($autoCommit, $configOptions); + $handler = app($configOptions->getHandler()); + + $dispatcher = self::getMiddlewareDispatcher($configOptions->getMiddlewares()); + + return new Manager($consumer, $handler, $dispatcher, $autoCommit, $commitAsync); } - public static function getConsumer(bool $autoCommit, AbstractConfigManager $configManager): ConsumerInterface + protected static function requiresPartition(ConfigOptions $configOptions): bool { - if (self::requiresPartition($configManager)) { - return app(LowLevel::class)->getConsumer( - $autoCommit, - $configManager - ); - } + $partition = $configOptions->getPartition(); - return app(HighLevel::class)->getConsumer($autoCommit, $configManager); + return !is_null($partition) && $partition >= 0; } - protected static function requiresPartition(AbstractConfigManager $configManager): bool + public static function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface { - $partition = $configManager->get('partition'); + if (self::requiresPartition($configOptions)) { + return app(LowLevel::class)->getConsumer($autoCommit, $configOptions); + } - return !is_null($partition) && $partition >= 0; + return app(HighLevel::class)->getConsumer($autoCommit, $configOptions); } private static function getMiddlewareDispatcher(array $middlewares): Dispatcher diff --git a/src/Connectors/Consumer/HighLevel.php b/src/Connectors/Consumer/HighLevel.php index 7a61a171..ede233a4 100644 --- a/src/Connectors/Consumer/HighLevel.php +++ b/src/Connectors/Consumer/HighLevel.php @@ -2,38 +2,39 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Authentication\Factory; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Consumers\HighLevel as HighLevelConsumer; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\Conf; use RdKafka\KafkaConsumer; class HighLevel implements ConnectorInterface { - public function getConsumer(bool $autoCommit, AbstractConfigManager $configManager): ConsumerInterface + public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface { - $conf = $this->getConf($configManager); + $conf = $this->getConf($configOptions); - $conf->set('group.id', $configManager->get('consumer_group')); - $conf->set('auto.offset.reset', $configManager->get('offset_reset')); + $conf->set('group.id', $configOptions->getConsumerGroup()); + $conf->set('auto.offset.reset', $configOptions->getOffsetReset()); if (!$autoCommit) { $conf->set('enable.auto.commit', 'false'); } $consumer = app(KafkaConsumer::class, ['conf' => $conf]); - $consumer->subscribe([$configManager->get('topic_id')]); - $timeout = $configManager->get('timeout'); + $consumer->subscribe([$configOptions->getTopicId()]); + $timeout = $configOptions->getTimeout(); return app(HighLevelConsumer::class, compact('consumer', 'timeout')); } - protected function getConf(AbstractConfigManager $configManager): Conf + protected function getConf(ConfigOptions $configOptions): Conf { $conf = resolve(Conf::class); - Factory::authenticate($conf, $configManager); + $broker = $configOptions->getBroker(); + Factory::authenticate($conf, $broker->getAuth()); - $conf->set('metadata.broker.list', $configManager->get('connections')); + $conf->set('metadata.broker.list', $broker->getConnections()); return $conf; } diff --git a/src/Connectors/Consumer/LowLevel.php b/src/Connectors/Consumer/LowLevel.php index 6c4d3993..9f47da26 100644 --- a/src/Connectors/Consumer/LowLevel.php +++ b/src/Connectors/Consumer/LowLevel.php @@ -6,50 +6,43 @@ use Metamorphosis\Authentication\Factory; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Consumers\LowLevel as LowLevelConsumer; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\Conf; use RdKafka\Consumer; use RdKafka\TopicConf; class LowLevel implements ConnectorInterface { - public function getConsumer(bool $autoCommit, AbstractConfigManager $configManager): ConsumerInterface + public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface { $conf = $this->getConf(); - $conf->set('group.id', $configManager->get('consumer_group')); + $conf->set('group.id', $configOptions->getConsumerGroup()); if (!$autoCommit) { $conf->set('enable.auto.commit', 'false'); } - Factory::authenticate($conf, $configManager); + $broker = $configOptions->getBroker(); + Factory::authenticate($conf, $broker->getAuth()); $consumer = new Consumer($conf); - $consumer->addBrokers($configManager->get('connections')); + $consumer->addBrokers($broker->getConnections()); - $topicConf = $this->getTopicConfigs($configManager); - $topicConsumer = $consumer->newTopic( - $configManager->get('topic_id'), - $topicConf - ); + $topicConf = $this->getTopicConfigs($configOptions); + $topicConsumer = $consumer->newTopic($configOptions->getTopicId(), $topicConf); - $topicConsumer->consumeStart( - $configManager->get('partition'), - $configManager->get('offset') - ); + $topicConsumer->consumeStart($configOptions->getPartition(), $configOptions->getOffset()); - return new LowLevelConsumer($topicConsumer, $configManager); + return new LowLevelConsumer($topicConsumer, $configOptions); } - protected function getTopicConfigs(AbstractConfigManager $configManager) + protected function getTopicConfigs(ConfigOptions $configOptions) { $topicConfig = new TopicConf(); // Set where to start consuming messages when there is no initial offset in // offset store or the desired offset is out of range. // 'smallest': start from the beginning - $topicConfig->set( - 'auto.offset.reset', - $configManager->get('offset_reset') - ); + $topicConfig->set('auto.offset.reset', $configOptions->getOffsetReset()); return $topicConfig; } diff --git a/src/Connectors/Producer/Connector.php b/src/Connectors/Producer/Connector.php index ebe696ea..68eafb07 100644 --- a/src/Connectors/Producer/Connector.php +++ b/src/Connectors/Producer/Connector.php @@ -2,8 +2,8 @@ namespace Metamorphosis\Connectors\Producer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Authentication\Factory; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; use Metamorphosis\TopicHandler\Producer\HandleableResponseInterface; use Metamorphosis\TopicHandler\Producer\HandlerInterface; use RdKafka\Conf; @@ -12,7 +12,7 @@ class Connector { - public function getProducerTopic(HandlerInterface $handler, AbstractConfigManager $configManager): KafkaProducer + public function getProducerTopic(HandlerInterface $handler, ConfigOptions $configOptions): KafkaProducer { $conf = resolve(Conf::class); @@ -29,9 +29,10 @@ function ($kafka, Message $message) use ($handler) { ); } - $conf->set('metadata.broker.list', $configManager->get('connections')); + $broker = $configOptions->getBroker(); + $conf->set('metadata.broker.list', $broker->getConnections()); - Factory::authenticate($conf, $configManager); + Factory::authenticate($conf, $broker->getAuth()); return app(KafkaProducer::class, compact('conf')); } diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index 37dab14c..4d37240b 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -6,6 +6,7 @@ use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; class ConfigOptionsCommand extends BaseCommand { @@ -22,25 +23,29 @@ class ConfigOptionsCommand extends BaseCommand /** * @var {inheritdoc} */ - protected $signature = 'kafka:consume-config-class {handler : handler.}'; + protected $signature = 'kafka:consume-config-class + {handler : handler.} + {--times= : Amount of messages to be consumed.}'; public function handle(Config $config) { - $configManager = $config->makeWithConfigOptions($this->argument()['handler']); + $consumerHandler = app($this->argument('handler')); - $this->writeStartingConsumer($configManager); + $configOptions = $consumerHandler->getConfigOptions(); - $manager = Factory::make($configManager); + $this->writeStartingConsumer($configOptions); + + $manager = Factory::make($configOptions); $runner = app(Runner::class, compact('manager')); - $runner->run(2); + $runner->run($this->option('times')); } - private function writeStartingConsumer(AbstractConfigManager $configManager) + private function writeStartingConsumer(ConfigOptions $configOptions) { - $text = 'Starting consumer for topic: '.$configManager->get('topic').PHP_EOL; - $text .= ' on consumer group: '.$configManager->get('consumer_group').PHP_EOL; - $text .= 'Connecting in '.$configManager->get('connections').PHP_EOL; + $text = 'Starting consumer for topic: '.$configOptions->getTopicId().PHP_EOL; + $text .= ' on consumer group: '.$configOptions->getConsumerGroup().PHP_EOL; + $text .= 'Connecting in '.$configOptions->getBroker()->getConnections().PHP_EOL; $text .= 'Running consumer..'; $this->output->writeln($text); diff --git a/src/Consumers/LowLevel.php b/src/Consumers/LowLevel.php index 0a1d762b..e4a31ea0 100644 --- a/src/Consumers/LowLevel.php +++ b/src/Consumers/LowLevel.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Consumers; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\ConsumerTopic; use RdKafka\Message; @@ -14,12 +14,12 @@ class LowLevel implements ConsumerInterface private ?int $timeout; - public function __construct(ConsumerTopic $consumer, AbstractConfigManager $configManager) + public function __construct(ConsumerTopic $consumer, ConfigOptions $configOptions) { $this->consumer = $consumer; - $this->partition = $configManager->get('partition'); - $this->timeout = $configManager->get('timeout'); + $this->partition = $configOptions->getPartition(); + $this->timeout = $configOptions->getTimeout(); } public function consume(): ?Message diff --git a/src/Producer.php b/src/Producer.php index 61c155a3..e1187cf0 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -7,6 +7,7 @@ use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\Middlewares\Handler\Producer as ProducerMiddleware; use Metamorphosis\Producer\Poll; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Metamorphosis\TopicHandler\Producer\HandlerInterface; @@ -31,32 +32,21 @@ public function produce(HandlerInterface $producerHandler): void public function build(HandlerInterface $producerHandler): Dispatcher { - $configManager = $this->getConfigManager($producerHandler); + $configOptions = $producerHandler->getConfigOptions(); - $middlewares = $configManager->middlewares(); - $middlewares[] = $this->getProducerMiddleware( - $producerHandler, - $configManager - ); + $middlewares = $configOptions->getMiddlewares(); + $middlewares[] = $this->getProducerMiddleware($producerHandler, $configOptions); return new Dispatcher($middlewares); } - public function getProducerMiddleware( - HandlerInterface $producerHandler, - AbstractConfigManager $configManager - ): ProducerMiddleware { - $producer = $this->connector->getProducerTopic( - $producerHandler, - $configManager - ); + public function getProducerMiddleware(HandlerInterface $producerHandler, ConfigOptions $configOptions): ProducerMiddleware + { + $producer = $this->connector->getProducerTopic($producerHandler, $configOptions); - $topic = $producer->newTopic($configManager->get('topic_id')); - $poll = app( - Poll::class, - ['producer' => $producer, 'configManager' => $configManager] - ); - $partition = $configManager->get('partition'); + $topic = $producer->newTopic($configOptions->getTopicId()); + $poll = app(Poll::class, ['producer' => $producer, 'configOptions' => $configOptions]); + $partition = $configOptions->getPartition(); return app( ProducerMiddleware::class, diff --git a/src/Producer/Poll.php b/src/Producer/Poll.php index afb8d081..d6a64626 100644 --- a/src/Producer/Poll.php +++ b/src/Producer/Poll.php @@ -3,6 +3,7 @@ namespace Metamorphosis\Producer; use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; use RdKafka\Producer; use RuntimeException; @@ -24,15 +25,13 @@ class Poll private Producer $producer; - public function __construct(Producer $producer, AbstractConfigManager $configManager) + public function __construct(Producer $producer, ConfigOptions $configOptions) { - $this->isAsync = $configManager->get('is_async'); - $this->maxPollRecords = $configManager->get('max_poll_records'); - $this->requiredAcknowledgment = $configManager->get( - 'required_acknowledgment' - ); - $this->maxFlushAttempts = $configManager->get('flush_attempts'); - $this->timeout = $configManager->get('timeout'); + $this->isAsync = $configOptions->isAsync(); + $this->maxPollRecords = $configOptions->getMaxPollRecords(); + $this->requiredAcknowledgment = $configOptions->isRequiredAcknowledgment(); + $this->maxFlushAttempts = $configOptions->getFlushAttempts(); + $this->timeout = $configOptions->getTimeout(); $this->producer = $producer; } diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 4a45d8e0..4e256a42 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -45,6 +45,7 @@ protected function runTheConsumer(): void 'kafka:consume-config-class', [ 'handler' => '\\App\\Kafka\\Consumers\\ConsumerOverride', + '--times' => 2, ] ); } From a57b4deb908865492efb683cfa516f385aa1a902 Mon Sep 17 00:00:00 2001 From: David Franca Date: Tue, 1 Feb 2022 09:46:24 -0300 Subject: [PATCH 06/72] chore: migrate Avro to use ConfigOptions\AvroSchema --- src/Avro/ClientFactory.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Avro/ClientFactory.php b/src/Avro/ClientFactory.php index 0e537e4c..b30d0c1b 100644 --- a/src/Avro/ClientFactory.php +++ b/src/Avro/ClientFactory.php @@ -3,29 +3,31 @@ namespace Metamorphosis\Avro; use GuzzleHttp\Client as GuzzleClient; -use Metamorphosis\AbstractConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema; class ClientFactory { - public function make(AbstractConfigManager $configManager): CachedSchemaRegistryClient + const REQUEST_TIMEOUT = 2000; + + public function make(AvroSchema $avroSchema): CachedSchemaRegistryClient { - $guzzleHttp = $this->getGuzzleHttpClient($configManager); + $guzzleHttp = $this->getGuzzleHttpClient($avroSchema); $client = app(Client::class, ['client' => $guzzleHttp]); return app(CachedSchemaRegistryClient::class, compact('client')); } - private function getGuzzleHttpClient(AbstractConfigManager $configManager): GuzzleClient + private function getGuzzleHttpClient(AvroSchema $avroSchema): GuzzleClient { - $config = $configManager->get('request_options') ?: []; - $config['timeout'] = $configManager->get('timeout'); - $config['base_uri'] = $configManager->get('url'); + $config = $avroSchema->getRequestOptions(); + $config['timeout'] = self::REQUEST_TIMEOUT; + $config['base_uri'] = $avroSchema->getUrl(); $config['headers'] = array_merge( $this->getDefaultHeaders(), $config['headers'] ?? [] ); - $config['verify'] = $configManager->get('ssl_verify') ?? false; + $config['verify'] = $avroSchema->getRequestOptions()['ssl_verify'] ?? false; return app(GuzzleClient::class, compact('config')); } From 14046edf1d1f473d2a83b8026e97ae9d8453ccd4 Mon Sep 17 00:00:00 2001 From: David Franca Date: Tue, 1 Feb 2022 10:00:06 -0300 Subject: [PATCH 07/72] chore: migrate Aith factory to use config options --- src/Authentication/Factory.php | 2 -- src/Authentication/SASLAuthentication.php | 17 +++++---- src/Authentication/SSLAuthentication.php | 17 +++++---- src/Connectors/Consumer/Config.php | 5 ++- src/Console/ConsumerCommand.php | 1 + src/Middlewares/AvroSchemaDecoder.php | 19 +++++----- tests/Unit/Authentication/FactoryTest.php | 43 ++++++++++------------- 7 files changed, 52 insertions(+), 52 deletions(-) diff --git a/src/Authentication/Factory.php b/src/Authentication/Factory.php index c679209a..c8a8f38a 100644 --- a/src/Authentication/Factory.php +++ b/src/Authentication/Factory.php @@ -2,10 +2,8 @@ namespace Metamorphosis\Authentication; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Exceptions\AuthenticationException; use Metamorphosis\TopicHandler\ConfigOptions\Auth\AuthInterface; -use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use RdKafka\Conf; class Factory diff --git a/src/Authentication/SASLAuthentication.php b/src/Authentication/SASLAuthentication.php index 9b2fb0d0..3918558d 100644 --- a/src/Authentication/SASLAuthentication.php +++ b/src/Authentication/SASLAuthentication.php @@ -9,25 +9,28 @@ class SASLAuthentication implements AuthenticationInterface { private Conf $conf; - private AbstractConfigManager $configManager; + /** + * @var SaslSsl + */ + private $configOptions; - public function __construct(Conf $conf, SaslSsl $config) + public function __construct(Conf $conf, SaslSsl $configOptions) { $this->conf = $conf; - $this->config = $config; + $this->configOptions = $configOptions; $this->authenticate(); } private function authenticate(): void { - $this->conf->set('security.protocol', $this->config->getType()); + $this->conf->set('security.protocol', $this->configOptions->getType()); // The mechanisms key is optional when configuring this kind of authentication // If the user does not specify the mechanism, the default will be 'PLAIN'. // But, to make config more clear, we are asking the user every time. - $this->conf->set('sasl.mechanisms', $this->config->getMechanisms()); - $this->conf->set('sasl.username', $this->config->getUsername()); - $this->conf->set('sasl.password', $this->config->getPassword()); + $this->conf->set('sasl.mechanisms', $this->configOptions->getMechanisms()); + $this->conf->set('sasl.username', $this->configOptions->getUsername()); + $this->conf->set('sasl.password', $this->configOptions->getPassword()); } } diff --git a/src/Authentication/SSLAuthentication.php b/src/Authentication/SSLAuthentication.php index 750552e8..5dfc06e4 100644 --- a/src/Authentication/SSLAuthentication.php +++ b/src/Authentication/SSLAuthentication.php @@ -9,21 +9,24 @@ class SSLAuthentication implements AuthenticationInterface { private Conf $conf; - private AbstractConfigManager $configManager; + /** + * @var Ssl + */ + private $configOptions; - public function __construct(Conf $conf, Ssl $configSsl) + public function __construct(Conf $conf, Ssl $configOptions) { $this->conf = $conf; - $this->configSsl = $configSsl; + $this->configOptions = $configOptions; $this->authenticate(); } private function authenticate(): void { - $this->conf->set('security.protocol', $this->configSsl->getType()); - $this->conf->set('ssl.ca.location', $this->configSsl->getCa()); - $this->conf->set('ssl.certificate.location', $this->configSsl->getCertificate()); - $this->conf->set('ssl.key.location', $this->configSsl->getKey()); + $this->conf->set('security.protocol', $this->configOptions->getType()); + $this->conf->set('ssl.ca.location', $this->configOptions->getCa()); + $this->conf->set('ssl.certificate.location', $this->configOptions->getCertificate()); + $this->conf->set('ssl.key.location', $this->configOptions->getKey()); } } diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 9216cf18..fa6d1398 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -6,6 +6,7 @@ use Metamorphosis\Connectors\AbstractConfig; use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\ConfigurationException; +use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; /** * This class is responsible for handling all configuration made on the @@ -75,10 +76,8 @@ public function make(array $options, array $arguments): AbstractConfigManager ); $this->validate(array_merge($config, $override)); - $configManager = app(ConsumerConfigManager::class); - $configManager->set($config, $override); - return $configManager; + return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); } /** diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index c4bdab76..bd80175e 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -7,6 +7,7 @@ use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; +use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; class ConsumerCommand extends BaseCommand { diff --git a/src/Middlewares/AvroSchemaDecoder.php b/src/Middlewares/AvroSchemaDecoder.php index 4ae626c9..8d185fdd 100644 --- a/src/Middlewares/AvroSchemaDecoder.php +++ b/src/Middlewares/AvroSchemaDecoder.php @@ -3,28 +3,29 @@ namespace Metamorphosis\Middlewares; use Closure; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Avro\ClientFactory; use Metamorphosis\Avro\Serializer\MessageDecoder; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema; class AvroSchemaDecoder implements MiddlewareInterface { private MessageDecoder $decoder; - private AbstractConfigManager $configManager; + /** + * @var AvroSchema + */ + private $avroSchema; - public function __construct(AbstractConfigManager $configManager, ClientFactory $factory) + public function __construct(AvroSchema $avroSchema, ClientFactory $factory) { - $this->configManager = $configManager; - if (!$this->configManager->get('url')) { - throw new ConfigurationException( - "Avro schema url not found, it's required to use AvroSchemaDecoder Middleware" - ); + $this->avroSchema = $avroSchema; + if (!$this->avroSchema->getUrl()) { + throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaDecoder Middleware"); } - $this->decoder = new MessageDecoder($factory->make($configManager)); + $this->decoder = new MessageDecoder($factory->make($avroSchema)); } public function process(RecordInterface $record, Closure $next) diff --git a/tests/Unit/Authentication/FactoryTest.php b/tests/Unit/Authentication/FactoryTest.php index 53999f8a..717abe16 100644 --- a/tests/Unit/Authentication/FactoryTest.php +++ b/tests/Unit/Authentication/FactoryTest.php @@ -3,8 +3,11 @@ namespace Tests\Unit\Authentication; use Metamorphosis\Authentication\Factory; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\AuthenticationException; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\AuthInterface; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\SaslSsl; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\Ssl; +use Mockery as m; use RdKafka\Conf; use Tests\LaravelTestCase; @@ -13,15 +16,7 @@ class FactoryTest extends LaravelTestCase public function testItMakesSslAuthenticationClass(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'auth' => [ - 'type' => 'ssl', - 'ca' => 'path/to/ca', - 'certificate' => 'path/to/certificate', - 'key' => 'path/to/key', - ], - ]); + $configOptionsSsl = new Ssl('path/to/ca', 'path/to/certificate', 'path/to/key'); $conf = new Conf(); $expected = [ 'security.protocol' => 'ssl', @@ -31,7 +26,7 @@ public function testItMakesSslAuthenticationClass(): void ]; // Actions - Factory::authenticate($conf, $configManager); + Factory::authenticate($conf, $configOptionsSsl); // Assertions $this->assertArraySubset($expected, $conf->dump()); @@ -40,15 +35,11 @@ public function testItMakesSslAuthenticationClass(): void public function testItMakesSASLAuthenticationClass(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'auth' => [ - 'type' => 'sasl_ssl', - 'mechanisms' => 'PLAIN', - 'username' => 'some-username', - 'password' => 'some-password', - ], - ]); + $configOptionsSaslSsl = new SaslSsl( + 'PLAIN', + 'some-username', + 'some-password' + ); $conf = new Conf(); $expected = [ 'security.protocol' => 'sasl_ssl', @@ -58,7 +49,7 @@ public function testItMakesSASLAuthenticationClass(): void ]; // Actions - Factory::authenticate($conf, $configManager); + Factory::authenticate($conf, $configOptionsSaslSsl); // Assertions $this->assertArraySubset($expected, $conf->dump()); @@ -67,13 +58,17 @@ public function testItMakesSASLAuthenticationClass(): void public function testItThrowsExceptionWhenInvalidProtocolIsPassed(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set(['auth' => ['type' => 'some-invalid-type']]); + $invalidAuth = m::mock(AuthInterface::class); $conf = new Conf(); + // Expectations + $invalidAuth->expects() + ->getType() + ->andReturn('some-invalid-type'); + $this->expectException(AuthenticationException::class); // Actions - Factory::authenticate($conf, $configManager); + Factory::authenticate($conf, $invalidAuth); } } From 9b3c02884909ccbac041f1d474bcb99efedd5d20 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Tue, 1 Feb 2022 10:28:02 -0300 Subject: [PATCH 08/72] chore: remove configManager and add Sasl config class --- .../Authentication/SASLAuthenticationTest.php | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/Unit/Authentication/SASLAuthenticationTest.php b/tests/Unit/Authentication/SASLAuthenticationTest.php index ce1078b3..f9e9a474 100644 --- a/tests/Unit/Authentication/SASLAuthenticationTest.php +++ b/tests/Unit/Authentication/SASLAuthenticationTest.php @@ -3,7 +3,7 @@ namespace Tests\Unit\Authentication; use Metamorphosis\Authentication\SASLAuthentication; -use Metamorphosis\ConsumerConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\SaslSsl; use RdKafka\Conf; use Tests\LaravelTestCase; @@ -12,16 +12,9 @@ class SASLAuthenticationTest extends LaravelTestCase public function testItShouldValidateAuthenticationConfigurations(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'auth' => [ - 'type' => 'sasl_ssl', - 'mechanisms' => 'PLAIN', - 'username' => 'some-username', - 'password' => 'some-password', - ], - ]); + $configSaslSsl = new SaslSsl('PLAIN', 'some-username', 'some-password'); $conf = new Conf(); + $expected = [ 'security.protocol' => 'sasl_ssl', 'sasl.username' => 'some-username', @@ -30,7 +23,7 @@ public function testItShouldValidateAuthenticationConfigurations(): void ]; // Actions - new SASLAuthentication($conf, $configManager); + new SASLAuthentication($conf, $configSaslSsl); // Assertions $this->assertArraySubset($expected, $conf->dump()); From 2e937f631017064e4bd06df54c187736d11cd29a Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Tue, 1 Feb 2022 10:31:05 -0300 Subject: [PATCH 09/72] chore: remove configManager and add SSL config class --- .../Unit/Authentication/SSLAuthenticationTest.php | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/Unit/Authentication/SSLAuthenticationTest.php b/tests/Unit/Authentication/SSLAuthenticationTest.php index 11c93691..35a9168e 100644 --- a/tests/Unit/Authentication/SSLAuthenticationTest.php +++ b/tests/Unit/Authentication/SSLAuthenticationTest.php @@ -3,7 +3,7 @@ namespace Tests\Unit\Authentication; use Metamorphosis\Authentication\SSLAuthentication; -use Metamorphosis\ConsumerConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\Ssl; use RdKafka\Conf; use Tests\LaravelTestCase; @@ -12,16 +12,8 @@ class SSLAuthenticationTest extends LaravelTestCase public function testItShouldValidateAuthenticationConfigurations(): void { // Set - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'auth' => [ - 'type' => 'ssl', - 'ca' => 'path/to/ca', - 'certificate' => 'path/to/certificate', - 'key' => 'path/to/key', - ], - ]); $conf = new Conf(); + $configSsl = new Ssl('path/to/ca', 'path/to/certificate', 'path/to/key'); $expected = [ 'security.protocol' => 'ssl', 'ssl.ca.location' => 'path/to/ca', @@ -30,7 +22,7 @@ public function testItShouldValidateAuthenticationConfigurations(): void ]; // Actions - new SSLAuthentication($conf, $configManager); + new SSLAuthentication($conf, $configSsl); // Assertions $this->assertArraySubset($expected, $conf->dump()); From 96bfd526718c66caeb59db6fb10a7898a35b1214 Mon Sep 17 00:00:00 2001 From: Hugo Ferreira Date: Tue, 1 Feb 2022 12:04:03 -0300 Subject: [PATCH 10/72] chore: fix code standard with phpcbf --- src/Connectors/Consumer/ConnectorInterface.php | 2 +- src/Connectors/Consumer/Factory.php | 1 - src/Connectors/Consumer/LowLevel.php | 1 - src/Console/ConfigOptionsCommand.php | 1 - src/Console/ConsumerCommand.php | 1 - src/Producer/Poll.php | 1 - 6 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Connectors/Consumer/ConnectorInterface.php b/src/Connectors/Consumer/ConnectorInterface.php index 738a6d0e..f9a7934d 100644 --- a/src/Connectors/Consumer/ConnectorInterface.php +++ b/src/Connectors/Consumer/ConnectorInterface.php @@ -2,8 +2,8 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; use Metamorphosis\Consumers\ConsumerInterface; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; interface ConnectorInterface { diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index f48fe56d..893e9030 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; diff --git a/src/Connectors/Consumer/LowLevel.php b/src/Connectors/Consumer/LowLevel.php index 9f47da26..e897d467 100644 --- a/src/Connectors/Consumer/LowLevel.php +++ b/src/Connectors/Consumer/LowLevel.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Connectors\Consumer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Authentication\Factory; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Consumers\LowLevel as LowLevelConsumer; diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index 4d37240b..eb4c1071 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Console; use Illuminate\Console\Command as BaseCommand; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index bd80175e..c4bdab76 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -7,7 +7,6 @@ use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; -use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; class ConsumerCommand extends BaseCommand { diff --git a/src/Producer/Poll.php b/src/Producer/Poll.php index d6a64626..4c3b9c81 100644 --- a/src/Producer/Poll.php +++ b/src/Producer/Poll.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Producer; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; use RdKafka\Producer; use RuntimeException; From 865e5438a525581099c0715ff33ad76185b142c4 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 7 Mar 2022 18:11:01 -0300 Subject: [PATCH 11/72] chore: debug config options --- src/Connectors/Consumer/Config.php | 3 +- src/Console/ConsumerCommand.php | 24 ++++------- src/Consumer.php | 4 +- .../Producer/AbstractProducer.php | 15 ++++--- tests/Integration/ConsumerTest.php | 11 ++--- tests/Integration/Dummies/MessageProducer.php | 14 +------ tests/Integration/ProducerTest.php | 41 ++++++++++++------- tests/Unit/ProducerTest.php | 16 ++++---- 8 files changed, 64 insertions(+), 64 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index fa6d1398..d14770fb 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -6,6 +6,7 @@ use Metamorphosis\Connectors\AbstractConfig; use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\ConfigurationException; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer; use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; /** @@ -48,7 +49,7 @@ public function makeWithConfigOptions(string $handlerClass, ?int $times = null): return $configManager; } - public function make(array $options, array $arguments): AbstractConfigManager + public function make(array $options, array $arguments): Consumer { $configName = $options['config_name'] ?? 'kafka'; $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index c4bdab76..5c233db4 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -3,10 +3,10 @@ namespace Metamorphosis\Console; use Illuminate\Console\Command as BaseCommand; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer; class ConsumerCommand extends BaseCommand { @@ -38,27 +38,21 @@ class ConsumerCommand extends BaseCommand public function handle(Config $config) { - $configManager = $config->make($this->option(), $this->argument()); + $consumer = $config->make($this->option(), $this->argument()); - $this->writeStartingConsumer($configManager); + $this->writeStartingConsumer($consumer); - $manager = Factory::make($configManager); + $manager = Factory::make($consumer); $runner = app(Runner::class, compact('manager')); - $runner->run($configManager->get('times')); + $runner->run($this->option('times')); } - private function writeStartingConsumer(AbstractConfigManager $configManager) + private function writeStartingConsumer(Consumer $consumer) { - $text = 'Starting consumer for topic: ' . $configManager->get( - 'topic' - ) . PHP_EOL; - $text .= ' on consumer group: ' . $configManager->get( - 'consumer_group' - ) . PHP_EOL; - $text .= 'Connecting in ' . $configManager->get( - 'connections' - ) . PHP_EOL; + $text = 'Starting consumer for topic: '.$consumer->getTopicId().PHP_EOL; + $text .= ' on consumer group: '.$consumer->getConsumerGroup().PHP_EOL; + $text .= 'Connecting in '.$consumer->getBroker()->getConnections().PHP_EOL; $text .= 'Running consumer..'; $this->output->writeln($text); diff --git a/src/Consumer.php b/src/Consumer.php index ba04c783..1d35a0c1 100644 --- a/src/Consumer.php +++ b/src/Consumer.php @@ -19,8 +19,8 @@ public function __construct(ConsumerConfigManager $configManager, ConsumerConfig { $configManager->set($configOptions->toArray()); - $this->consumer = Factory::getConsumer(true, $configManager); - $this->dispatcher = new Dispatcher($configManager->middlewares()); + $this->consumer = Factory::getConsumer(true, $configOptions); + $this->dispatcher = new Dispatcher($configOptions->getMiddlewares()); } public function consume(): ?RecordInterface diff --git a/src/TopicHandler/Producer/AbstractProducer.php b/src/TopicHandler/Producer/AbstractProducer.php index e95a6f63..946db3ce 100644 --- a/src/TopicHandler/Producer/AbstractProducer.php +++ b/src/TopicHandler/Producer/AbstractProducer.php @@ -4,7 +4,7 @@ use Metamorphosis\Exceptions\JsonException; use Metamorphosis\Record\ProducerRecord; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptionsProducer; +use Metamorphosis\TopicHandler\ConfigOptions\Producer; class AbstractProducer implements HandlerInterface { @@ -15,18 +15,21 @@ class AbstractProducer implements HandlerInterface protected ?string $key; - private ConfigOptionsProducer $configOptions; + /** + * @var Producer + */ + protected $producer; - public function __construct($record, ConfigOptionsProducer $configOptions, ?string $key = null) + public function __construct($record, Producer $producer, string $key = null) { $this->record = $record; $this->key = $key; - $this->configOptions = $configOptions; + $this->producer = $producer; } - public function getConfigOptions(): ConfigOptionsProducer + public function getConfigOptions(): Producer { - return $this->configOptions; + return $this->producer; } public function getRecord() diff --git a/tests/Integration/ConsumerTest.php b/tests/Integration/ConsumerTest.php index 8c107499..1d4397e8 100644 --- a/tests/Integration/ConsumerTest.php +++ b/tests/Integration/ConsumerTest.php @@ -59,16 +59,13 @@ public function testItShouldSetup(): void $saleOrderDispatcher = Metamorphosis::build($messageProducer); $saleOrderDispatcher->handle($messageProducer->createRecord()); - $consumer = $this->app->make( - Consumer::class, - ['configOptions' => $consumerConfigOptions] - ); - $expected = ['id' => 'MESSAGE_ID']; + $consumer = $this->app->make(Consumer::class, ['configOptions' => $consumerConfigOptions]); + $expected = '{"id":"MESSAGE_ID"}'; // Actions - $result = $consumer->consume(); + $result = $consumer->consume()->getPayload(); // Assertions - $this->assertSame($expected, $result->getPayload()); + $this->assertSame($expected, $result); } } diff --git a/tests/Integration/Dummies/MessageProducer.php b/tests/Integration/Dummies/MessageProducer.php index 22aac6f8..38fb8090 100644 --- a/tests/Integration/Dummies/MessageProducer.php +++ b/tests/Integration/Dummies/MessageProducer.php @@ -3,22 +3,12 @@ namespace Tests\Integration\Dummies; use Illuminate\Support\Facades\Log; -use Metamorphosis\TopicHandler\Producer\AbstractHandler; +use Metamorphosis\TopicHandler\Producer\AbstractProducer; use RdKafka\Message; use RuntimeException; -class MessageProducer extends AbstractHandler +class MessageProducer extends AbstractProducer { - public string $topic = 'default'; - - public function __construct($record, string $topic, ?string $key = null, ?int $partition = null) - { - $this->record = $record; - $this->topic = $topic; - $this->key = $key ?? 'recordId123'; - $this->partition = $partition; - } - public function success(Message $message): void { Log::info('Record successfully sent to broker.', [ diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index f7d13966..1689fe87 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -5,6 +5,9 @@ use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; use Metamorphosis\Facades\Metamorphosis; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Tests\Integration\Dummies\MessageConsumer; use Tests\Integration\Dummies\MessageProducer; use Tests\LaravelTestCase; @@ -17,11 +20,13 @@ class ProducerTest extends LaravelTestCase protected string $secondLowLevelMessage; + /** + * @group runProducer + */ public function testShouldRunAProducerAndReceiveMessagesWithAHighLevelConsumer(): void { // Given That I $this->haveAConsumerHandlerConfigured(); - $this->haveNoPartitionConfigured(); $this->haveSomeRandomMessagesProduced(); // I Expect That @@ -128,13 +133,9 @@ protected function runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingT private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $producer = app( - MessageProducer::class, - [ - 'record' => $this->highLevelMessage, - 'topic' => 'default', - ] - ); + $configOptionsProducer = $this->createConfigOptionsProducer('kafka-test'); + //$producer = app(MessageProducer::class, ['record' => $this->highLevelMessage, 'configOptions'=> $configOptionsProducer]); + $producer = new MessageProducer($this->highLevelMessage, $configOptionsProducer, 'recordId123'); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -142,8 +143,10 @@ private function haveSomeRandomMessagesProduced(): void private function produceRecordMessage(string $record): string { - $topic = 'low_level'; - $producer = app(MessageProducer::class, compact('record', 'topic')); + $configOptionsProducer = $this->createConfigOptionsProducer('low_level'); + $producer = new MessageProducer($record, $configOptionsProducer, 'recordId123'); + //$producer = app(MessageProducer::class, ['record'=>$record, 'configOptions' => $a]); + //$producer->topic = 'low_level'; Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -183,10 +186,20 @@ private function haveFourProducedMessages(): void $this->produceRecordMessage($this->secondLowLevelMessage); } - private function haveNoPartitionConfigured(): void - { - config( - ['kafka.topics.default.consumer.consumer_groups.test-consumer-group.partition' => -1] + private function createConfigOptionsProducer(string $topicId = 'kafka-test'): ProducerConfigOptions + { + $brokerOptions = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( + $topicId, + $brokerOptions, + null, + null, + [], + 20000, + false, + true, + 10, + 500 ); } } diff --git a/tests/Unit/ProducerTest.php b/tests/Unit/ProducerTest.php index 5a93dc8c..31650aef 100644 --- a/tests/Unit/ProducerTest.php +++ b/tests/Unit/ProducerTest.php @@ -12,7 +12,6 @@ use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; use Metamorphosis\TopicHandler\ConfigOptions\Broker; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; -use Metamorphosis\TopicHandler\Producer\AbstractHandler; use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Mockery as m; use RdKafka\Producer as KafkaProducer; @@ -38,7 +37,7 @@ public function testItShouldProduceRecordAsArrayThroughMiddlewareQueue(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class ($record, $topic) extends AbstractHandler { + $producerHandler = new class($record, $topic) extends AbstractProducer { }; // Expectations @@ -95,7 +94,7 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class ($record, $topic) extends AbstractHandler { + $producerHandler = new class($record, $topic) extends AbstractProducer { }; // Expectations @@ -128,7 +127,10 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void ->withAnyArgs(); // Actions - $producer->produce($producerHandler); + $result = $producer->produce($producerHandler); + + // Assertions + $this->assertNull($result); } public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): void @@ -147,7 +149,7 @@ public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): vo $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class ($record, $topic) extends AbstractHandler { + $producerHandler = new class($record, $topic) extends AbstractProducer { }; // Expectations @@ -223,7 +225,7 @@ public function testShouldBuildDispatcher(): void $producerTopic = m::mock(ProducerTopic::class); $configManager = m::mock(ProducerConfigManager::class); - $producerHandler = new class ($record, $topic) extends AbstractHandler { + $producerHandler = new class($record, $topic) extends AbstractProducer { }; // Expectations @@ -298,7 +300,7 @@ public function testShouldBuildDispatcherWithConfigOptions(): void $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); $broker = new Broker($connections, new None()); $configOptions = new ProducerConfigOptions($topicId, $broker); - $producerHandler = new class ($record, $configOptions) extends AbstractProducer { + $producerHandler = new class($record, $configOptions) extends AbstractProducer { }; // Expectations From a89f497b38110d3722e7a91568eb0bb98c9ac3a7 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 9 Mar 2022 09:45:27 -0300 Subject: [PATCH 12/72] chore: debug config options --- src/Connectors/Consumer/Config.php | 19 ++++-- src/ConsumerConfigManager.php | 7 +-- tests/Unit/AbstractConfigManagerTest.php | 75 ---------------------- tests/Unit/ConsumerConfigManagerTest.php | 79 ------------------------ 4 files changed, 16 insertions(+), 164 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index d14770fb..d2c97dd3 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -2,12 +2,14 @@ namespace Metamorphosis\Connectors\Consumer; +use InvalidArgumentException; use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\AbstractConfig; use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\TopicHandler\ConfigOptions\Consumer; use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; +use Metamorphosis\TopicHandler\Consumer\AbstractHandler; /** * This class is responsible for handling all configuration made on the @@ -41,12 +43,21 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; - public function makeWithConfigOptions(string $handlerClass, ?int $times = null): AbstractConfigManager + /** + * @param string $handlerClass + * @param int|null $times + * @return Consumer + */ + public function makeWithConfigOptions(string $handlerClass, ?int $times = null): ?Consumer { - $configManager = app(ConsumerConfigManager::class); - $configManager->set(['handler' => $handlerClass], ['times' => $times]); + /** @var AbstractHandler */ + $handler = app($handlerClass); + $configOptions = $handler->getConfigOptions(); + if(is_null($configOptions)){ + throw new InvalidArgumentException('Handler class cannot be null'); + } - return $configManager; + return $configOptions; } public function make(array $options, array $arguments): Consumer diff --git a/src/ConsumerConfigManager.php b/src/ConsumerConfigManager.php index 36d31c40..88207c97 100644 --- a/src/ConsumerConfigManager.php +++ b/src/ConsumerConfigManager.php @@ -22,12 +22,7 @@ public function set(array $config, ?array $commandConfig = null): void $this->remove('middlewares'); foreach ($middlewares as $middleware) { - $this->middlewares[] = is_string($middleware) - ? app( - $middleware, - ['configManager' => $this] - ) - : $middleware; + $this->middlewares[] = is_string($middleware) ? app($middleware) : $middleware; } if (!$consumerHandler) { diff --git a/tests/Unit/AbstractConfigManagerTest.php b/tests/Unit/AbstractConfigManagerTest.php index 7cf86812..e69de29b 100644 --- a/tests/Unit/AbstractConfigManagerTest.php +++ b/tests/Unit/AbstractConfigManagerTest.php @@ -1,75 +0,0 @@ -instance( - AbstractHandler::class, - m::mock(AbstractHandler::class) - ); - $config = [ - 'middlewares' => [], - 'handler' => AbstractHandler::class, - 'broker' => [ - 'default' => [ - 'connections' => 'kafka:9092', - ], - ], - 'topic_id' => 'kafka-test', - ]; - $broker = new Broker('kafka:9092', new None()); - $configOptions = new ConsumerConfigOptions( - 'kafka-override', - $broker, - '\App\Kafka\Consumers\ConsumerExample', - null, - null, - 'default', - null, - [MiddlewareDummy::class], - 200, - false, - true - ); - - $expected = [ - 'topic_id' => 'kafka-override', - 'connections' => 'kafka:9092', - 'auth' => null, - 'timeout' => 200, - 'handler' => '\App\Kafka\Consumers\ConsumerExample', - 'partition' => -1, - 'offset' => null, - 'consumer_group' => 'default', - 'auto_commit' => false, - 'commit_async' => true, - 'offset_reset' => 'smallest', - ]; - - $configManager = new ConsumerConfigManager(); - - // Expectations - $handler->expects() - ->getConfigOptions() - ->andReturn($configOptions); - - // Actions - $configManager->set($config); - - // Expectations - $this->assertEquals($expected, $configManager->get()); - } -} diff --git a/tests/Unit/ConsumerConfigManagerTest.php b/tests/Unit/ConsumerConfigManagerTest.php index 347aea6c..e69de29b 100644 --- a/tests/Unit/ConsumerConfigManagerTest.php +++ b/tests/Unit/ConsumerConfigManagerTest.php @@ -1,79 +0,0 @@ -instance( - AbstractHandler::class, - m::mock(AbstractHandler::class) - ); - $config = [ - 'middlewares' => [], - 'handler' => AbstractHandler::class, - 'broker' => [ - 'default' => [ - 'connections' => 'kafka:9092', - ], - ], - 'topic_id' => 'kafka-test', - ]; - $broker = new Broker('kafka:9092', new None()); - $configOptions = new ConsumerConfigOptions( - 'kafka-override', - $broker, - '\App\Kafka\Consumers\ConsumerExample', - null, - null, - 'default', - null, - [MiddlewareDummy::class], - 200, - false - ); - - $expected = [ - 'topic_id' => 'kafka-override', - 'connections' => 'kafka:9092', - 'auth' => null, - 'timeout' => 1000, - 'handler' => '\App\Kafka\Consumers\ConsumerExample', - 'partition' => -1, - 'offset' => null, - 'consumer_group' => 'default', - 'auto_commit' => false, - 'commit_async' => true, - 'offset_reset' => 'smallest', - 'times' => 2, - ]; - - $configManager = new ConsumerConfigManager(); - - $commandConfig = [ - 'timeout' => 1000, - 'times' => 2, - ]; - - // Expectations - $handler->expects() - ->getConfigOptions() - ->andReturn($configOptions); - - // Actions - $configManager->set($config, $commandConfig); - - // Expectations - $this->assertEquals($expected, $configManager->get()); - } -} From de86b27060d65c36a6a9e214a9f35fbbab74bf2d Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 10 Mar 2022 17:23:57 -0300 Subject: [PATCH 13/72] chore: create interfaces for classes --- src/Avro/CachedSchemaRegistryClient.php | 4 ++-- src/Avro/Client.php | 2 +- src/Avro/Serializer/Encoders/SchemaId.php | 8 +++++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Avro/CachedSchemaRegistryClient.php b/src/Avro/CachedSchemaRegistryClient.php index c91e643c..97706cd5 100644 --- a/src/Avro/CachedSchemaRegistryClient.php +++ b/src/Avro/CachedSchemaRegistryClient.php @@ -5,7 +5,7 @@ use AvroSchemaParseException; use RuntimeException; -class CachedSchemaRegistryClient +class CachedSchemaRegistryClient implements CachedSchemaRegistryClientInterface { private Client $client; @@ -20,7 +20,7 @@ class CachedSchemaRegistryClient */ private array $subjectVersionToSchema = []; - public function __construct(Client $client) + public function __construct(AvroClientInterface $client) { $this->client = $client; } diff --git a/src/Avro/Client.php b/src/Avro/Client.php index e0b714ac..02516b70 100644 --- a/src/Avro/Client.php +++ b/src/Avro/Client.php @@ -5,7 +5,7 @@ use GuzzleHttp\Client as GuzzleHttp; use Psr\Http\Message\ResponseInterface; -class Client +class Client implements AvroClientInterface { private GuzzleHttp $client; diff --git a/src/Avro/Serializer/Encoders/SchemaId.php b/src/Avro/Serializer/Encoders/SchemaId.php index ea428472..fd796333 100644 --- a/src/Avro/Serializer/Encoders/SchemaId.php +++ b/src/Avro/Serializer/Encoders/SchemaId.php @@ -6,6 +6,7 @@ use AvroIODatumWriter; use AvroStringIO; use Metamorphosis\Avro\CachedSchemaRegistryClient; +use Metamorphosis\Avro\CachedSchemaRegistryClientInterface; use Metamorphosis\Avro\Schema; use Metamorphosis\Avro\Serializer\SchemaFormats; @@ -13,7 +14,7 @@ class SchemaId implements EncoderInterface { private CachedSchemaRegistryClient $registry; - public function __construct(CachedSchemaRegistryClient $registry) + public function __construct(CachedSchemaRegistryClientInterface $registry) { $this->registry = $registry; } @@ -41,4 +42,9 @@ public function encode(Schema $schema, $message): string return $io->string(); } + + public function getRegistry() + { + return $this->registry; + } } From 7294e95d5527adb3f5c5edbd3c914c61bd00c8e4 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 10 Mar 2022 17:25:28 -0300 Subject: [PATCH 14/72] chore: update calls from configManager to configOptions --- src/Middlewares/AvroSchemaMixedEncoder.php | 29 ++++++++++------------ 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/Middlewares/AvroSchemaMixedEncoder.php b/src/Middlewares/AvroSchemaMixedEncoder.php index 4df0beb7..11534359 100644 --- a/src/Middlewares/AvroSchemaMixedEncoder.php +++ b/src/Middlewares/AvroSchemaMixedEncoder.php @@ -3,12 +3,12 @@ namespace Metamorphosis\Middlewares; use Closure; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Avro\CachedSchemaRegistryClient; use Metamorphosis\Avro\ClientFactory; use Metamorphosis\Avro\Serializer\Encoders\SchemaId; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptionsProducer; /** * Fetches a schema for a topic by subject and version (currently only 'latest') @@ -21,30 +21,27 @@ class AvroSchemaMixedEncoder implements MiddlewareInterface private CachedSchemaRegistryClient $schemaRegistry; - private AbstractConfigManager $configManager; + /** + * @var ConfigOptionsProducer + */ + private $configOptionsProducer; - public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, AbstractConfigManager $configManager) + public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, ConfigOptionsProducer $configOptionsProducer) { - if (!$configManager->get('url')) { - throw new ConfigurationException( - "Avro schema url not found, it's required to use AvroSchemaEncoder Middleware" - ); + if (!$configOptionsProducer->getAvroSchema()->getUrl()) { + throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaEncoder Middleware"); } - $schemaRegistry = $factory->make($configManager); +// $schemaRegistry = $factory->make($configOptionsProducer->getAvroSchema()); $this->schemaIdEncoder = $schemaIdEncoder; - $this->schemaRegistry = $schemaRegistry; - $this->configManager = $configManager; + $this->schemaRegistry = $schemaIdEncoder->getRegistry(); + $this->configOptionsProducer = $configOptionsProducer; } public function process(RecordInterface $record, Closure $next) { - $topic = $this->configManager->get('topic_id'); - $schema = $this->schemaRegistry->getBySubjectAndVersion( - "{$topic}-value", - 'latest' - ); - + $topic = $this->configOptionsProducer->getTopicId(); + $schema = $this->schemaRegistry->getBySubjectAndVersion("{$topic}-value", 'latest'); $arrayPayload = json_decode($record->getPayload(), true); $encodedPayload = $this->schemaIdEncoder->encode( $schema, From 3af47115ba4cbd8a519f6e31906165d7e50fd7b8 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 10 Mar 2022 17:26:51 -0300 Subject: [PATCH 15/72] chore: add interfaces --- src/Avro/AvroClientInterface.php | 6 ++++++ src/Avro/CachedSchemaRegistryClientInterface.php | 7 +++++++ 2 files changed, 13 insertions(+) create mode 100644 src/Avro/AvroClientInterface.php create mode 100644 src/Avro/CachedSchemaRegistryClientInterface.php diff --git a/src/Avro/AvroClientInterface.php b/src/Avro/AvroClientInterface.php new file mode 100644 index 00000000..d7617023 --- /dev/null +++ b/src/Avro/AvroClientInterface.php @@ -0,0 +1,6 @@ + Date: Thu, 10 Mar 2022 17:30:37 -0300 Subject: [PATCH 16/72] chore: add AvroSchemaMixedEncoderTest --- .../AvroSchemaMixedEncoderTest.php | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php diff --git a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php new file mode 100644 index 00000000..9412bd4b --- /dev/null +++ b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php @@ -0,0 +1,69 @@ +getSchemaTest(); + $configOptionsProducer = $this->createProducer(); + + $record = new ProducerRecord($schemaTest, 'kafka-test'); + $cachedSchemaRegistryClient = m::mock(CachedSchemaRegistryClient::class); + + $schema = (new Schema())->parse($schemaTest, '123'); + $clientFactory = new ClientFactory(); + $schemaIdEncoder = m::mock(SchemaId::class, [$cachedSchemaRegistryClient])->makePartial(); + + $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder($schemaIdEncoder, $clientFactory, $configOptionsProducer); + + //expect + $cachedSchemaRegistryClient->shouldReceive('getBySubjectAndVersion')->andReturn($schema); + $schemaIdEncoder->shouldReceive('encode')->andReturn('string'); + + //act + $avroSchemaMixedEncoder->process($record, $closure); + + //assert + $this->assertInstanceOf(AvroSchemaMixedEncoder::class, $avroSchemaMixedEncoder); + } + + private function createProducer() + { + $brokerOptions = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( + 'kafka-test', + $brokerOptions, + null, + new AvroSchema('http://url.teste', []), + [], + 20000, + false, + true, + 10, + 500 + ); + } + + private function getSchemaTest(): string + { + return file_get_contents(__DIR__.'/../fixtures/schemas/sales_price.avsc'); + } +} From 61dfbab8f5ae578a2c37d2c1510e14c3d77b5db1 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 21 Mar 2022 15:25:33 -0300 Subject: [PATCH 17/72] chore: remove inutilized interfaces --- src/Avro/CachedSchemaRegistryClient.php | 4 ++-- src/Avro/Client.php | 2 +- src/Avro/Serializer/Encoders/SchemaId.php | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Avro/CachedSchemaRegistryClient.php b/src/Avro/CachedSchemaRegistryClient.php index 97706cd5..c91e643c 100644 --- a/src/Avro/CachedSchemaRegistryClient.php +++ b/src/Avro/CachedSchemaRegistryClient.php @@ -5,7 +5,7 @@ use AvroSchemaParseException; use RuntimeException; -class CachedSchemaRegistryClient implements CachedSchemaRegistryClientInterface +class CachedSchemaRegistryClient { private Client $client; @@ -20,7 +20,7 @@ class CachedSchemaRegistryClient implements CachedSchemaRegistryClientInterface */ private array $subjectVersionToSchema = []; - public function __construct(AvroClientInterface $client) + public function __construct(Client $client) { $this->client = $client; } diff --git a/src/Avro/Client.php b/src/Avro/Client.php index 02516b70..e0b714ac 100644 --- a/src/Avro/Client.php +++ b/src/Avro/Client.php @@ -5,7 +5,7 @@ use GuzzleHttp\Client as GuzzleHttp; use Psr\Http\Message\ResponseInterface; -class Client implements AvroClientInterface +class Client { private GuzzleHttp $client; diff --git a/src/Avro/Serializer/Encoders/SchemaId.php b/src/Avro/Serializer/Encoders/SchemaId.php index fd796333..75d08216 100644 --- a/src/Avro/Serializer/Encoders/SchemaId.php +++ b/src/Avro/Serializer/Encoders/SchemaId.php @@ -6,7 +6,6 @@ use AvroIODatumWriter; use AvroStringIO; use Metamorphosis\Avro\CachedSchemaRegistryClient; -use Metamorphosis\Avro\CachedSchemaRegistryClientInterface; use Metamorphosis\Avro\Schema; use Metamorphosis\Avro\Serializer\SchemaFormats; @@ -14,7 +13,7 @@ class SchemaId implements EncoderInterface { private CachedSchemaRegistryClient $registry; - public function __construct(CachedSchemaRegistryClientInterface $registry) + public function __construct(CachedSchemaRegistryClient $registry) { $this->registry = $registry; } From ca175afbdaaba6ddb6fafff77a88d5298e2cf16c Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 21 Mar 2022 15:37:01 -0300 Subject: [PATCH 18/72] chore: add method to get consumerGroupId --- src/Connectors/Consumer/Config.php | 54 +++++++++++++----------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index d2c97dd3..aace08a9 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -64,22 +64,11 @@ public function make(array $options, array $arguments): Consumer { $configName = $options['config_name'] ?? 'kafka'; $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); - $consumerConfig = $this->getConsumerConfig( - $topicConfig, - $arguments['consumer_group'] - ); - $brokerConfig = $this->getBrokerConfig( - $configName, - $topicConfig['broker'] - ); - $schemaConfig = $this->getSchemaConfig( - $configName, - $arguments['topic'] - ); - $override = array_merge( - $this->filterValues($options), - $this->filterValues($arguments) - ); + $consumerGroupId = $this->getConsumerGroup($topicConfig, $arguments['consumer_group']); + $consumerConfig = $this->getConsumerConfig($topicConfig, $arguments, $consumerGroupId); + $brokerConfig = $this->getBrokerConfig($configName, $topicConfig['broker']); + $schemaConfig = $this->getSchemaConfig($configName, $arguments['topic']); + $override = array_merge($this->filterValues($options), $this->filterValues($arguments)); $config = array_merge( $topicConfig, $brokerConfig, @@ -89,6 +78,9 @@ public function make(array $options, array $arguments): Consumer $this->validate(array_merge($config, $override)); + $topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'] = $consumerConfig['partition']; + $topicConfig['consumer_group'] = $consumerGroupId; + return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); } @@ -110,31 +102,31 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } - private function getConsumerConfig(array $topicConfig, ?string $consumerGroupId = null): array + private function getConsumerConfig(array $topicConfig, array $arguments, string $consumerGroupId): array { - if ( - !$consumerGroupId && 1 === count( - $topicConfig['consumer']['consumer_groups'] - ) - ) { - $consumerGroupId = current( - array_keys($topicConfig['consumer']['consumer_groups']) - ); + $consumerConfig = $topicConfig['consumer']['consumer_groups'][$consumerGroupId] ?? null; + if (!$consumerConfig) { + throw new ConfigurationException("Consumer group '{$consumerGroupId}' not found"); } - $consumerGroupId = $consumerGroupId ?? 'default'; - $consumerConfig = $topicConfig['consumer']['consumer_groups'][$consumerGroupId] ?? null; $consumerConfig['consumer_group'] = $consumerGroupId; - if (!$consumerConfig) { - throw new ConfigurationException( - "Consumer group '{$consumerGroupId}' not found" - ); + if(isset($arguments['partition'])){ + $consumerConfig['partition'] = $arguments['partition']; } return $consumerConfig; } + private function getConsumerGroup(array $topicConfig, ?string $consumerGroupId): string + { + if (!$consumerGroupId && 1 === count($topicConfig['consumer']['consumer_groups'])) { + $consumerGroupId = current(array_keys($topicConfig['consumer']['consumer_groups'])); + } + + return $consumerGroupId ?? 'default'; + } + private function getMiddlewares(string $configName, array $topicConfig): array { return array_merge( From 8bddb6b634a9046be3cfadeeb34837731c559b7c Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 21 Mar 2022 15:58:27 -0300 Subject: [PATCH 19/72] chore: replace configManager by configOptions --- src/Connectors/Consumer/Config.php | 17 +- src/Connectors/Consumer/Factory.php | 12 +- src/Connectors/Producer/Connector.php | 6 +- src/Consumers/LowLevel.php | 8 +- src/Middlewares/AvroSchemaDecoder.php | 8 +- src/Middlewares/AvroSchemaMixedEncoder.php | 18 +- src/Producer.php | 23 +- src/Producer/Poll.php | 14 +- .../Factories/ConsumerFactory.php | 14 +- tests/Integration/ConsumerTest.php | 2 +- tests/Integration/ProducerTest.php | 31 ++- tests/Unit/Connectors/Consumer/ConfigTest.php | 8 +- .../Unit/Connectors/Consumer/FactoryTest.php | 75 +++--- .../Connectors/Consumer/HighLevelTest.php | 26 +- .../Unit/Connectors/Consumer/LowLevelTest.php | 27 +- .../Connectors/Producer/ConnectorTest.php | 47 ++-- tests/Unit/Consumers/LowLevelTest.php | 19 +- .../Middlewares/AvroSchemaDecoderTest.php | 70 ++++++ .../AvroSchemaMixedEncoderTest.php | 97 ++++--- tests/Unit/Producer/PollTest.php | 69 +++-- tests/Unit/ProducerTest.php | 238 +++++------------- .../Factories/ConsumerFactoryTest.php | 1 + 22 files changed, 430 insertions(+), 400 deletions(-) create mode 100644 tests/Unit/Middlewares/AvroSchemaDecoderTest.php diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index aace08a9..4db8f76a 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -3,13 +3,10 @@ namespace Metamorphosis\Connectors\Consumer; use InvalidArgumentException; -use Metamorphosis\AbstractConfigManager; use Metamorphosis\Connectors\AbstractConfig; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\TopicHandler\ConfigOptions\Consumer; use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; -use Metamorphosis\TopicHandler\Consumer\AbstractHandler; /** * This class is responsible for handling all configuration made on the @@ -43,17 +40,11 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; - /** - * @param string $handlerClass - * @param int|null $times - * @return Consumer - */ public function makeWithConfigOptions(string $handlerClass, ?int $times = null): ?Consumer { - /** @var AbstractHandler */ $handler = app($handlerClass); $configOptions = $handler->getConfigOptions(); - if(is_null($configOptions)){ + if (is_null($configOptions)) { throw new InvalidArgumentException('Handler class cannot be null'); } @@ -77,8 +68,10 @@ public function make(array $options, array $arguments): Consumer ); $this->validate(array_merge($config, $override)); + if (isset($topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'])) { + $topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'] = $consumerConfig['partition']; + } - $topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'] = $consumerConfig['partition']; $topicConfig['consumer_group'] = $consumerGroupId; return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); @@ -111,7 +104,7 @@ private function getConsumerConfig(array $topicConfig, array $arguments, string $consumerConfig['consumer_group'] = $consumerGroupId; - if(isset($arguments['partition'])){ + if (isset($arguments['partition'])) { $consumerConfig['partition'] = $arguments['partition']; } diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index 893e9030..6a7c61aa 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -4,16 +4,20 @@ use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Middlewares\Handler\Dispatcher; -use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; /** * This factory will determine what kind of connector will be used. * Basically, if the user pass --partition and --offset as argument * means that we will use the low level approach. */ + +/** + * TODO Rename this class to ConsumerFactory, it will improve semantics + */ class Factory { - public static function make(ConfigOptions $configOptions): Manager + public static function make(ConsumerConfigOptions $configOptions): Manager { $autoCommit = $configOptions->isAutoCommit(); $commitAsync = $configOptions->isCommitASync(); @@ -26,14 +30,14 @@ public static function make(ConfigOptions $configOptions): Manager return new Manager($consumer, $handler, $dispatcher, $autoCommit, $commitAsync); } - protected static function requiresPartition(ConfigOptions $configOptions): bool + protected static function requiresPartition(ConsumerConfigOptions $configOptions): bool { $partition = $configOptions->getPartition(); return !is_null($partition) && $partition >= 0; } - public static function getConsumer(bool $autoCommit, ConfigOptions $configOptions): ConsumerInterface + public static function getConsumer(bool $autoCommit, ConsumerConfigOptions $configOptions): ConsumerInterface { if (self::requiresPartition($configOptions)) { return app(LowLevel::class)->getConsumer($autoCommit, $configOptions); diff --git a/src/Connectors/Producer/Connector.php b/src/Connectors/Producer/Connector.php index 68eafb07..1109a598 100644 --- a/src/Connectors/Producer/Connector.php +++ b/src/Connectors/Producer/Connector.php @@ -3,7 +3,7 @@ namespace Metamorphosis\Connectors\Producer; use Metamorphosis\Authentication\Factory; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Metamorphosis\TopicHandler\Producer\HandleableResponseInterface; use Metamorphosis\TopicHandler\Producer\HandlerInterface; use RdKafka\Conf; @@ -12,7 +12,7 @@ class Connector { - public function getProducerTopic(HandlerInterface $handler, ConfigOptions $configOptions): KafkaProducer + public function getProducerTopic(HandlerInterface $handler, ProducerConfigOptions $producerConfigOptions): KafkaProducer { $conf = resolve(Conf::class); @@ -29,7 +29,7 @@ function ($kafka, Message $message) use ($handler) { ); } - $broker = $configOptions->getBroker(); + $broker = $producerConfigOptions->getBroker(); $conf->set('metadata.broker.list', $broker->getConnections()); Factory::authenticate($conf, $broker->getAuth()); diff --git a/src/Consumers/LowLevel.php b/src/Consumers/LowLevel.php index e4a31ea0..e9fee56c 100644 --- a/src/Consumers/LowLevel.php +++ b/src/Consumers/LowLevel.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Consumers; -use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; use RdKafka\ConsumerTopic; use RdKafka\Message; @@ -14,12 +14,12 @@ class LowLevel implements ConsumerInterface private ?int $timeout; - public function __construct(ConsumerTopic $consumer, ConfigOptions $configOptions) + public function __construct(ConsumerTopic $consumer, ConsumerConfigOptions $consumerConfigOptions) { $this->consumer = $consumer; - $this->partition = $configOptions->getPartition(); - $this->timeout = $configOptions->getTimeout(); + $this->partition = $consumerConfigOptions->getPartition(); + $this->timeout = $consumerConfigOptions->getTimeout(); } public function consume(): ?Message diff --git a/src/Middlewares/AvroSchemaDecoder.php b/src/Middlewares/AvroSchemaDecoder.php index 8d185fdd..1aeffcff 100644 --- a/src/Middlewares/AvroSchemaDecoder.php +++ b/src/Middlewares/AvroSchemaDecoder.php @@ -8,6 +8,7 @@ use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; class AvroSchemaDecoder implements MiddlewareInterface { @@ -18,14 +19,13 @@ class AvroSchemaDecoder implements MiddlewareInterface */ private $avroSchema; - public function __construct(AvroSchema $avroSchema, ClientFactory $factory) + public function __construct(ClientFactory $factory, ConsumerConfigOptions $consumerConfigOptions) { - $this->avroSchema = $avroSchema; - if (!$this->avroSchema->getUrl()) { + if (!$consumerConfigOptions->getAvroSchema()->getUrl()) { throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaDecoder Middleware"); } - $this->decoder = new MessageDecoder($factory->make($avroSchema)); + $this->decoder = new MessageDecoder($factory->make($consumerConfigOptions->getAvroSchema())); } public function process(RecordInterface $record, Closure $next) diff --git a/src/Middlewares/AvroSchemaMixedEncoder.php b/src/Middlewares/AvroSchemaMixedEncoder.php index 11534359..6969a80e 100644 --- a/src/Middlewares/AvroSchemaMixedEncoder.php +++ b/src/Middlewares/AvroSchemaMixedEncoder.php @@ -8,7 +8,7 @@ use Metamorphosis\Avro\Serializer\Encoders\SchemaId; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptionsProducer; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; /** * Fetches a schema for a topic by subject and version (currently only 'latest') @@ -22,25 +22,25 @@ class AvroSchemaMixedEncoder implements MiddlewareInterface private CachedSchemaRegistryClient $schemaRegistry; /** - * @var ConfigOptionsProducer + * @var ProducerConfigOptions */ - private $configOptionsProducer; + private $producerConfigOptions; - public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, ConfigOptionsProducer $configOptionsProducer) + public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, ProducerConfigOptions $producerConfigOptions) { - if (!$configOptionsProducer->getAvroSchema()->getUrl()) { + if (!$producerConfigOptions->getAvroSchema()->getUrl()) { throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaEncoder Middleware"); } -// $schemaRegistry = $factory->make($configOptionsProducer->getAvroSchema()); + $schemaRegistry = $factory->make($producerConfigOptions->getAvroSchema()); $this->schemaIdEncoder = $schemaIdEncoder; - $this->schemaRegistry = $schemaIdEncoder->getRegistry(); - $this->configOptionsProducer = $configOptionsProducer; + $this->schemaRegistry = $schemaRegistry; + $this->producerConfigOptions = $producerConfigOptions; } public function process(RecordInterface $record, Closure $next) { - $topic = $this->configOptionsProducer->getTopicId(); + $topic = $this->producerConfigOptions->getTopicId(); $schema = $this->schemaRegistry->getBySubjectAndVersion("{$topic}-value", 'latest'); $arrayPayload = json_decode($record->getPayload(), true); $encodedPayload = $this->schemaIdEncoder->encode( diff --git a/src/Producer.php b/src/Producer.php index e1187cf0..ee277fdf 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -7,7 +7,7 @@ use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\Middlewares\Handler\Producer as ProducerMiddleware; use Metamorphosis\Producer\Poll; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Metamorphosis\TopicHandler\Producer\HandlerInterface; @@ -32,21 +32,26 @@ public function produce(HandlerInterface $producerHandler): void public function build(HandlerInterface $producerHandler): Dispatcher { - $configOptions = $producerHandler->getConfigOptions(); + $producerConfigOptions = $producerHandler->getConfigOptions(); - $middlewares = $configOptions->getMiddlewares(); - $middlewares[] = $this->getProducerMiddleware($producerHandler, $configOptions); + $middlewares = $producerConfigOptions->getMiddlewares(); + + foreach ($middlewares as &$middleware) { + $middleware = is_string($middleware) ? app($middleware, ['producerConfigOptions' => $producerConfigOptions]) : $middleware; + } + + $middlewares[] = $this->getProducerMiddleware($producerHandler, $producerConfigOptions); return new Dispatcher($middlewares); } - public function getProducerMiddleware(HandlerInterface $producerHandler, ConfigOptions $configOptions): ProducerMiddleware + public function getProducerMiddleware(HandlerInterface $producerHandler, ProducerConfigOptions $producerConfigOptions): ProducerMiddleware { - $producer = $this->connector->getProducerTopic($producerHandler, $configOptions); + $producer = $this->connector->getProducerTopic($producerHandler, $producerConfigOptions); - $topic = $producer->newTopic($configOptions->getTopicId()); - $poll = app(Poll::class, ['producer' => $producer, 'configOptions' => $configOptions]); - $partition = $configOptions->getPartition(); + $topic = $producer->newTopic($producerConfigOptions->getTopicId()); + $poll = app(Poll::class, ['producer' => $producer, 'producerConfigOptions' => $producerConfigOptions]); + $partition = $producerConfigOptions->getPartition(); return app( ProducerMiddleware::class, diff --git a/src/Producer/Poll.php b/src/Producer/Poll.php index 4c3b9c81..c07508ec 100644 --- a/src/Producer/Poll.php +++ b/src/Producer/Poll.php @@ -2,7 +2,7 @@ namespace Metamorphosis\Producer; -use Metamorphosis\TopicHandler\ConfigOptions\Producer as ConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use RdKafka\Producer; use RuntimeException; @@ -24,13 +24,13 @@ class Poll private Producer $producer; - public function __construct(Producer $producer, ConfigOptions $configOptions) + public function __construct(Producer $producer, ProducerConfigOptions $producerConfigOptions) { - $this->isAsync = $configOptions->isAsync(); - $this->maxPollRecords = $configOptions->getMaxPollRecords(); - $this->requiredAcknowledgment = $configOptions->isRequiredAcknowledgment(); - $this->maxFlushAttempts = $configOptions->getFlushAttempts(); - $this->timeout = $configOptions->getTimeout(); + $this->isAsync = $producerConfigOptions->isAsync(); + $this->maxPollRecords = $producerConfigOptions->getMaxPollRecords(); + $this->requiredAcknowledgment = $producerConfigOptions->isRequiredAcknowledgment(); + $this->maxFlushAttempts = $producerConfigOptions->getFlushAttempts(); + $this->timeout = $producerConfigOptions->getTimeout(); $this->producer = $producer; } diff --git a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php index 103b882e..7bd58c93 100644 --- a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php +++ b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php @@ -22,20 +22,16 @@ public static function make( private static function getConsumerGroupConfig(array $topicData): array { $topicData['topicId'] = $topicData['topic_id']; + $topicData['consumerGroup'] = $topicData['consumer_group']; - $consumer = current($topicData['consumer']); - $topicData['consumerGroup'] = key($consumer); + $consumerGroup = $topicData['consumerGroup']; + $consumer = current($topicData['consumer'])[$consumerGroup]; - return array_merge( - $topicData, - self::convertConfigAttributes($consumer) - ); + return array_merge_recursive($topicData, self::convertConfigAttributes($consumer)); } - private static function convertConfigAttributes(array $topic): array + private static function convertConfigAttributes(array $consumerConfig): array { - $consumerConfig = current($topic); - if (isset($consumerConfig['auto_commit'])) { $consumerConfig['autoCommit'] = $consumerConfig['auto_commit']; } diff --git a/tests/Integration/ConsumerTest.php b/tests/Integration/ConsumerTest.php index 1d4397e8..7088e46b 100644 --- a/tests/Integration/ConsumerTest.php +++ b/tests/Integration/ConsumerTest.php @@ -51,7 +51,7 @@ public function testItShouldSetup(): void MessageProducerWithConfigOptions::class, [ 'record' => ['id' => 'MESSAGE_ID'], - 'configOptions' => $producerConfigOptions, + 'producer' => $producerConfigOptions, 'key' => 1, ] ); diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 1689fe87..9780d74c 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -27,6 +27,7 @@ public function testShouldRunAProducerAndReceiveMessagesWithAHighLevelConsumer() { // Given That I $this->haveAConsumerHandlerConfigured(); + $this->haveAConsumerPartitionConfigured(); $this->haveSomeRandomMessagesProduced(); // I Expect That @@ -69,6 +70,11 @@ protected function haveAConsumerHandlerConfigured(): void ); } + protected function haveAConsumerPartitionConfigured(): void + { + config(['kafka.topics.default.consumer.consumer_groups.test-consumer-group.partition' => -1]); + } + protected function runTheConsumer(): void { $this->artisan( @@ -133,9 +139,9 @@ protected function runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingT private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $configOptionsProducer = $this->createConfigOptionsProducer('kafka-test'); - //$producer = app(MessageProducer::class, ['record' => $this->highLevelMessage, 'configOptions'=> $configOptionsProducer]); - $producer = new MessageProducer($this->highLevelMessage, $configOptionsProducer, 'recordId123'); + $producerConfigOptions = $this->createProducerConfigOptions('kafka-test'); +// $producer = app(MessageProducer::class, ['record' => $this->highLevelMessage, 'producer'=> $producerConfigOptions]); + $producer = new MessageProducer($this->highLevelMessage, $producerConfigOptions, 'recordId123'); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -143,10 +149,8 @@ private function haveSomeRandomMessagesProduced(): void private function produceRecordMessage(string $record): string { - $configOptionsProducer = $this->createConfigOptionsProducer('low_level'); - $producer = new MessageProducer($record, $configOptionsProducer, 'recordId123'); - //$producer = app(MessageProducer::class, ['record'=>$record, 'configOptions' => $a]); - //$producer->topic = 'low_level'; + $producerConfigOptions = $this->createProducerConfigOptions('low_level'); + $producer = new MessageProducer($record, $producerConfigOptions, 'recordId123'); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -186,20 +190,15 @@ private function haveFourProducedMessages(): void $this->produceRecordMessage($this->secondLowLevelMessage); } - private function createConfigOptionsProducer(string $topicId = 'kafka-test'): ProducerConfigOptions + private function createProducerConfigOptions(string $topicId = 'kafka-test'): ProducerConfigOptions { - $brokerOptions = new Broker('kafka:9092', new None()); + $broker = new Broker('kafka:9092', new None()); return new ProducerConfigOptions( $topicId, - $brokerOptions, + $broker, null, null, - [], - 20000, - false, - true, - 10, - 500 + [] ); } } diff --git a/tests/Unit/Connectors/Consumer/ConfigTest.php b/tests/Unit/Connectors/Consumer/ConfigTest.php index 9e1c26f9..539b6ea0 100644 --- a/tests/Unit/Connectors/Consumer/ConfigTest.php +++ b/tests/Unit/Connectors/Consumer/ConfigTest.php @@ -84,10 +84,10 @@ public function testShouldValidateConsumerConfig(): void ]); // Actions - $configManager = $config->make($options, $arguments); + $configManager = $config->makeWithConfigOptions(ConsumerHandlerDummy::class); // Assertions - $this->assertArraySubset($expected, $configManager->get()); + $this->assertArraySubset($expected, $configManager->toArray()); } public function testShouldNotSetRuntimeConfigWhenOptionsIsInvalid(): void @@ -109,7 +109,7 @@ public function testShouldNotSetRuntimeConfigWhenOptionsIsInvalid(): void $configManager = $config->make($options, $arguments); // Assertions - $this->assertEmpty($configManager->get()); + $this->assertEmpty($configManager->toArray()); } public function testShouldNotSetRuntimeConfigWhenKafkaConfigIsInvalid(): void @@ -132,6 +132,6 @@ public function testShouldNotSetRuntimeConfigWhenKafkaConfigIsInvalid(): void $configManager = $config->make($options, $arguments); // Assertions - $this->assertEmpty($configManager->get()); + $this->assertEmpty($configManager->toArray()); } } diff --git a/tests/Unit/Connectors/Consumer/FactoryTest.php b/tests/Unit/Connectors/Consumer/FactoryTest.php index a90471d6..50a158f2 100644 --- a/tests/Unit/Connectors/Consumer/FactoryTest.php +++ b/tests/Unit/Connectors/Consumer/FactoryTest.php @@ -11,48 +11,6 @@ class FactoryTest extends LaravelTestCase { - public function testItMakesManagerWithLowLevelConsumer(): void - { - // Set - $config = new Config(); - $configManager = $config->make( - ['timeout' => 61], - ['topic' => 'topic_key', 'consumer_group' => 'with-partition'] - ); - $manager = Factory::make($configManager); - - // Assertions - $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); - } - - public function testItMakesManagerWithLowLevelConsumerWhenPartitionIsNotValid(): void - { - // Set - $config = new Config(); - $configManager = $config->make( - ['timeout' => 61], - ['topic' => 'topic_key', 'consumer_group' => 'with-partition', 'partition' => -1] - ); - $manager = Factory::make($configManager); - - // Assertions - $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); - } - - public function testItMakesHighLevelClass(): void - { - // Set - $config = new Config(); - $configManager = $config->make( - ['timeout' => 61], - ['topic' => 'topic_key', 'consumer_group' => 'without-partition'] - ); - $manager = Factory::make($configManager); - - // Assertions - $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); - } - protected function setUp(): void { parent::setUp(); @@ -88,4 +46,37 @@ protected function setUp(): void ], ]); } + + public function testItMakesManagerWithLowLevelConsumer(): void + { + // Set + $config = new Config(); + $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); + } + + public function testItMakesManagerWithLowLevelConsumerWhenPartitionIsNotValid(): void + { + // Set + $config = new Config(); + $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition', 'partition' => -1]); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); + } + + public function testItMakesHighLevelClass(): void + { + // Set + $config = new Config(); + $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'without-partition']); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); + } } diff --git a/tests/Unit/Connectors/Consumer/HighLevelTest.php b/tests/Unit/Connectors/Consumer/HighLevelTest.php index 52d58fc4..97ab8e91 100644 --- a/tests/Unit/Connectors/Consumer/HighLevelTest.php +++ b/tests/Unit/Connectors/Consumer/HighLevelTest.php @@ -3,8 +3,11 @@ namespace Tests\Unit\Connectors\Consumer; use Metamorphosis\Connectors\Consumer\HighLevel; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\HighLevel as HighLevelConsumer; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; use Tests\LaravelTestCase; class HighLevelTest extends LaravelTestCase @@ -12,19 +15,20 @@ class HighLevelTest extends LaravelTestCase public function testItShouldMakeConnectorSetup(): void { // Set - $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'connections' => $connections, - 'consumer_group' => 'some-group', - 'topic_id' => 'some_topic', - 'offset_reset' => 'earliest', - 'timeout' => 1000, - ]); $connector = new HighLevel(); + $brokerOptions = new Broker('kafka:9092', new None()); + $consumerConfigOptions = new ConsumerConfigOptions( + 'kafka-test', + $brokerOptions, + null, + 1, + 0, + 'some-group', + new AvroSchemaConfigOptions('http://url.teste') + ); // Actions - $result = $connector->getConsumer(false, $configManager); + $result = $connector->getConsumer(false, $consumerConfigOptions); // Assertions $this->assertInstanceOf(HighLevelConsumer::class, $result); diff --git a/tests/Unit/Connectors/Consumer/LowLevelTest.php b/tests/Unit/Connectors/Consumer/LowLevelTest.php index 039d096c..21c12291 100644 --- a/tests/Unit/Connectors/Consumer/LowLevelTest.php +++ b/tests/Unit/Connectors/Consumer/LowLevelTest.php @@ -3,8 +3,11 @@ namespace Tests\Unit\Connectors\Consumer; use Metamorphosis\Connectors\Consumer\LowLevel; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\LowLevel as LowLevelConsumer; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; use Tests\LaravelTestCase; class LowLevelTest extends LaravelTestCase @@ -12,20 +15,20 @@ class LowLevelTest extends LaravelTestCase public function testItShouldMakeConnectorSetup(): void { // Set - $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'connections' => $connections, - 'consumer_group' => 'some-group', - 'topic' => 'some_topic', - 'offset_reset' => 'earliest', - 'offset' => 0, - 'partition' => 1, - ]); $connector = new LowLevel(); + $brokerOptions = new Broker('kafka:9092', new None()); + $consumerConfigOptions = new ConsumerConfigOptions( + 'kafka-test', + $brokerOptions, + null, + 1, + 0, + 'some-group', + new AvroSchemaConfigOptions('http://url.teste') + ); // Actions - $result = $connector->getConsumer(true, $configManager); + $result = $connector->getConsumer(true, $consumerConfigOptions); // Assertions $this->assertInstanceOf(LowLevelConsumer::class, $result); diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index e0cb6123..f456dfb7 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -3,7 +3,8 @@ namespace Tests\Unit\Connectors\Producer; use Metamorphosis\Connectors\Producer\Connector; -use Metamorphosis\ProducerConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Metamorphosis\TopicHandler\Producer\AbstractProducer; use Metamorphosis\TopicHandler\Producer\HandleableResponseInterface; @@ -27,12 +28,12 @@ public function testItShouldMakeSetup(): void KafkaProducer::class, m::mock(KafkaProducer::class) ); - $configManager = m::mock(ProducerConfigManager::class); - $configOptions = m::mock(ProducerConfigOptions::class); + + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); - $handler = new class ('record', $configOptions) extends AbstractProducer implements HandleableResponseInterface { - /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */ + $handler = new class('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface { public function success(Message $message): void { } @@ -49,18 +50,14 @@ public function failed(Message $message): void ->withAnyArgs(); $conf->expects() - ->set('metadata.broker.list', 'kafka:9092'); - - $configManager->expects() - ->get('connections') - ->andReturn('kafka:9092'); + ->set('metadata.broker.list', 0); - $configManager->expects() - ->get('auth.type') - ->andReturn('none'); + $producerConfigOptions->expects() + ->getBroker() + ->andReturn($broker); // Actions - $result = $connector->getProducerTopic($handler, $configManager); + $result = $connector->getProducerTopic($handler, $producerConfigOptions); // Assertions $this->assertInstanceOf(KafkaProducer::class, $result); @@ -77,12 +74,12 @@ public function testItShouldMakeSetupWithoutHandleResponse(): void KafkaProducer::class, m::mock(KafkaProducer::class) ); - $configManager = m::mock(ProducerConfigManager::class); - $configOptions = m::mock(ProducerConfigOptions::class); + + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); - $handler = new class ('record', $configOptions) extends AbstractProducer implements HandlerInterface { - /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */ + $handler = new class('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface { public function success(Message $message): void { } @@ -98,18 +95,14 @@ public function failed(Message $message): void ->never(); $conf->expects() - ->set('metadata.broker.list', 'kafka:9092'); - - $configManager->expects() - ->get('connections') - ->andReturn('kafka:9092'); + ->set('metadata.broker.list', 0); - $configManager->expects() - ->get('auth.type') - ->andReturn('none'); + $producerConfigOptions->expects() + ->getBroker() + ->andReturn($broker); // Actions - $result = $connector->getProducerTopic($handler, $configManager); + $result = $connector->getProducerTopic($handler, $producerConfigOptions); // Assertions $this->assertInstanceOf(KafkaProducer::class, $result); diff --git a/tests/Unit/Consumers/LowLevelTest.php b/tests/Unit/Consumers/LowLevelTest.php index 6b35f993..cc8bb988 100644 --- a/tests/Unit/Consumers/LowLevelTest.php +++ b/tests/Unit/Consumers/LowLevelTest.php @@ -4,6 +4,10 @@ use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\LowLevel; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; use Mockery as m; use RdKafka\ConsumerTopic; use RdKafka\Message; @@ -19,10 +23,23 @@ public function testItShouldConsume(): void $configManager = new ConsumerConfigManager(); $configManager->set(compact('timeout', 'partition')); + $brokerOptions = new Broker('kafka:9092', new None()); + $consumerConfigOptions = new ConsumerConfigOptions( + 'kafka-test', + $brokerOptions, + null, + $partition, + null, + '', + new AvroSchemaConfigOptions('http://url.teste'), + [], + $timeout + ); + $consumerTopic = m::mock(ConsumerTopic::class); $message = new Message(); - $lowLevelConsumer = new LowLevel($consumerTopic, $configManager); + $lowLevelConsumer = new LowLevel($consumerTopic, $consumerConfigOptions); // Expectations $consumerTopic->expects() diff --git a/tests/Unit/Middlewares/AvroSchemaDecoderTest.php b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php new file mode 100644 index 00000000..6c8d31b0 --- /dev/null +++ b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php @@ -0,0 +1,70 @@ +getAvroSchema(); + $avroSchema = new AvroSchema('string'); + $decoder = m::mock(Schema::class); + $clientFactory = m::mock(ClientFactory::class); + $cachedSchemaRegistryClient = m::mock(CachedSchemaRegistryClient::class); + $expected = 'my awesome message'; + + $message = new Message(); + $message->payload = "\x01\x00\x00\x00\fmy-topic-key\x00\x00\x00\x05\$my awesome message"; + $message->err = 0; + + $closure = Closure::fromCallable(function ($producerRecord) { + return $producerRecord; + }); + + $consumerRecord = new ConsumerRecord($message); + + // Expectations + $clientFactory->expects() + ->make($avroSchemaConfigOptions) + ->andReturn($cachedSchemaRegistryClient); + + $cachedSchemaRegistryClient->expects() + ->getBySubjectAndVersion('my-topic-key', 5) + ->andReturn($decoder); + + $decoder->expects() + ->getAvroSchema() + ->andReturn($avroSchema); + + $avroSchemaDecoder = new AvroSchemaDecoder($clientFactory, $consumerConfigOptions); + + $result = $avroSchemaDecoder->process($consumerRecord, $closure); + + $this->assertSame($expected, $result->getPayload()); + } +} diff --git a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php index 9412bd4b..fcff2d6c 100644 --- a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php +++ b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php @@ -1,6 +1,8 @@ getSchemaFixture(); + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + 'kafka-test', + $broker, + null, + new AvroSchemaConfigOptions('subjects/kafka-test-value/versions/latest', []) + ); + $avroSchemaConfigOptions = $producerConfigOptions->getAvroSchema(); - $schemaTest = $this->getSchemaTest(); - $configOptionsProducer = $this->createProducer(); + $clientFactory = m::mock(ClientFactory::class); - $record = new ProducerRecord($schemaTest, 'kafka-test'); $cachedSchemaRegistryClient = m::mock(CachedSchemaRegistryClient::class); + $schemaIdEncoder = m::mock(SchemaId::class, [$cachedSchemaRegistryClient]); + + $schema = new Schema(); + $parsedSchema = $schema->parse($avroSchema, '123', 'kafka-test-value', 'latest'); + $record = $this->getRecord($parsedSchema->getAvroSchema()); + $producerRecord = new ProducerRecord($record, 'kafka-test'); + + $closure = Closure::fromCallable(function ($producerRecord) { + return $producerRecord; + }); + + $payload = json_decode($producerRecord->getPayload(), true); + $encodedMessage = 'binary_message'; - $schema = (new Schema())->parse($schemaTest, '123'); - $clientFactory = new ClientFactory(); - $schemaIdEncoder = m::mock(SchemaId::class, [$cachedSchemaRegistryClient])->makePartial(); + // Expectations + $clientFactory->expects() + ->make($avroSchemaConfigOptions) + ->andReturn($cachedSchemaRegistryClient); - $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder($schemaIdEncoder, $clientFactory, $configOptionsProducer); + $cachedSchemaRegistryClient->expects() + ->getBySubjectAndVersion('kafka-test-value', 'latest') + ->andReturn($schema); - //expect - $cachedSchemaRegistryClient->shouldReceive('getBySubjectAndVersion')->andReturn($schema); - $schemaIdEncoder->shouldReceive('encode')->andReturn('string'); + $schemaIdEncoder->expects() + ->encode($schema, $payload) + ->andReturn($encodedMessage); - //act - $avroSchemaMixedEncoder->process($record, $closure); + // Actions + $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder($schemaIdEncoder, $clientFactory, $producerConfigOptions); + $result = $avroSchemaMixedEncoder->process($producerRecord, $closure); - //assert - $this->assertInstanceOf(AvroSchemaMixedEncoder::class, $avroSchemaMixedEncoder); + // Assertions + $this->assertSame($record, $result->getOriginal()); + $this->assertSame($encodedMessage, $result->getPayload()); } - private function createProducer() + private function getRecord(AvroSchema $avroSchema): string { - $brokerOptions = new Broker('kafka:9092', new None()); - return new ProducerConfigOptions( - 'kafka-test', - $brokerOptions, - null, - new AvroSchema('http://url.teste', []), - [], - 20000, - false, - true, - 10, - 500 - ); + $defaultValues = [ + 'null' => null, + 'boolean' => true, + 'string' => 'abc', + 'int' => 1, + 'long' => 1.0, + 'float' => 1.0, + 'double' => 1.0, + 'array' => [], + ]; + + $result = []; + foreach ($avroSchema->fields() as $field) { + $result[$field->name()] = $defaultValues[$field->type->type]; + } + + return json_encode($result); } - private function getSchemaTest(): string + private function getSchemaFixture(): string { return file_get_contents(__DIR__.'/../fixtures/schemas/sales_price.avsc'); } diff --git a/tests/Unit/Producer/PollTest.php b/tests/Unit/Producer/PollTest.php index 8bda848d..3ef21c96 100644 --- a/tests/Unit/Producer/PollTest.php +++ b/tests/Unit/Producer/PollTest.php @@ -4,6 +4,10 @@ use Metamorphosis\Producer\Poll; use Metamorphosis\ProducerConfigManager; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Mockery as m; use RdKafka\Producer as KafkaProducer; use RuntimeException; @@ -14,17 +18,21 @@ class PollTest extends LaravelTestCase public function testItShouldHandleMessageWithoutAcknowledgment(): void { // Set - $configManager = new ProducerConfigManager(); - $configManager->set([ - 'topic_id' => 'topic_name', - 'timeout' => 4000, - 'is_async' => true, - 'max_poll_records' => 500, - 'flush_attempts' => 10, - 'required_acknowledgment' => false, - ]); + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + 'topic_name', + $broker, + null, + new AvroSchemaConfigOptions('string', []), + [], + 4000, + true, + false, + 500, + 10 + ); $kafkaProducer = m::mock(KafkaProducer::class); - $poll = new Poll($kafkaProducer, $configManager); + $poll = new Poll($kafkaProducer, $producerConfigOptions); // Expectations $kafkaProducer->expects() @@ -51,8 +59,21 @@ public function testShouldThrowExceptionWhenFlushFailed(): void 'partition' => 0, ]); + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + 'topic_name', + $broker, + null, + new AvroSchemaConfigOptions('string', []), + [], + 100, + false, + true, + 500, + 10 + ); $kafkaProducer = m::mock(KafkaProducer::class); - $poll = new Poll($kafkaProducer, $configManager); + $poll = new Poll($kafkaProducer, $producerConfigOptions); // Expectations $kafkaProducer->expects() @@ -72,18 +93,22 @@ public function testShouldThrowExceptionWhenFlushFailed(): void public function testItShouldHandleResponseEveryTimeWhenAsyncModeIsTrue(): void { // Set - $configManager = new ProducerConfigManager(); - $configManager->set([ - 'topic_id' => 'topic_name', - 'timeout' => 4000, - 'is_async' => false, - 'max_poll_records' => 500, - 'flush_attempts' => 10, - 'required_acknowledgment' => true, - 'partition' => 0, - ]); + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + 'topic_name', + $broker, + null, + new AvroSchemaConfigOptions('string', []), + [], + 4000, + false, + true, + 10, + 500 + ); + $kafkaProducer = m::mock(KafkaProducer::class); - $poll = new Poll($kafkaProducer, $configManager); + $poll = new Poll($kafkaProducer, $producerConfigOptions); // Expectations $kafkaProducer->expects() diff --git a/tests/Unit/ProducerTest.php b/tests/Unit/ProducerTest.php index 31650aef..02808390 100644 --- a/tests/Unit/ProducerTest.php +++ b/tests/Unit/ProducerTest.php @@ -8,8 +8,8 @@ use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\Middlewares\Handler\Producer as ProducerMiddleware; use Metamorphosis\Producer; -use Metamorphosis\ProducerConfigManager; use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; use Metamorphosis\TopicHandler\ConfigOptions\Broker; use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Metamorphosis\TopicHandler\Producer\AbstractProducer; @@ -31,38 +31,22 @@ public function testItShouldProduceRecordAsArrayThroughMiddlewareQueue(): void ); $config = m::mock(Config::class); $connector = m::mock(Connector::class); - $configManager = m::mock(ProducerConfigManager::class)->makePartial(); $producer = new Producer($config, $connector); $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class($record, $topic) extends AbstractProducer { + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker + ); + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $config->expects() - ->makeByTopic($topic) - ->andReturn($configManager); - - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topic); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - - $configManager->expects() - ->get('timeout') - ->andReturn(1000); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() @@ -86,36 +70,27 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void ProducerMiddleware::class, m::mock(ProducerMiddleware::class) ); + $config = m::mock(Config::class); $connector = m::mock(Connector::class); - $configManager = m::mock(ProducerConfigManager::class)->makePartial(); + + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker + ); + $producer = new Producer($config, $connector); $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class($record, $topic) extends AbstractProducer { + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $config->expects() - ->makeByTopic($topic) - ->andReturn($configManager); - - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topic); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() @@ -127,10 +102,7 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void ->withAnyArgs(); // Actions - $result = $producer->produce($producerHandler); - - // Assertions - $this->assertNull($result); + $producer->produce($producerHandler); } public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): void @@ -145,52 +117,29 @@ public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): vo $config = m::mock(Config::class); $connector = m::mock(Connector::class); $producer = new Producer($config, $connector); - $configManager = m::mock(ProducerConfigManager::class); + $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class($record, $topic) extends AbstractProducer { + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker, + 0, + new AvroSchemaConfigOptions('string'), + [], + 1000, + false, + true, + 500, + 1 + ); + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topic); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - - $configManager->expects() - ->get('max_poll_records') - ->andReturn(500); - - $configManager->expects() - ->get('required_acknowledgment') - ->andReturn(true); - - $configManager->expects() - ->get('flush_attempts') - ->andReturn(1); - - $configManager->expects() - ->get('timeout') - ->andReturn(1000); - - $configManager->expects() - ->get('is_async') - ->andReturn(false); - - $config->expects() - ->makeByTopic($topic) - ->andReturn($configManager); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() @@ -223,50 +172,27 @@ public function testShouldBuildDispatcher(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $configManager = m::mock(ProducerConfigManager::class); - $producerHandler = new class($record, $topic) extends AbstractProducer { + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker, + null, + new AvroSchemaConfigOptions('string'), + [], + 1000, + true, + true, + 500, + 1 + ); + + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $config->expects() - ->makeByTopic($topic) - ->andReturn($configManager); - - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topic); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - - $configManager->expects() - ->get('max_poll_records') - ->andReturn(500); - - $configManager->expects() - ->get('is_async') - ->andReturn(true); - - $configManager->expects() - ->get('required_acknowledgment') - ->andReturn(true); - - $configManager->expects() - ->get('flush_attempts') - ->andReturn(1); - - $configManager->expects() - ->get('timeout') - ->andReturn(1000); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() @@ -288,7 +214,7 @@ public function testShouldBuildDispatcherWithConfigOptions(): void { // Set $record = json_encode(['message' => 'some message']); - $topicId = 'TOPIC-ID'; + $topic = 'TOPIC-ID'; $config = m::mock(Config::class); $connector = m::mock(Connector::class); @@ -296,56 +222,30 @@ public function testShouldBuildDispatcherWithConfigOptions(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $configManager = m::mock(ProducerConfigManager::class); - $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); - $broker = new Broker($connections, new None()); - $configOptions = new ProducerConfigOptions($topicId, $broker); - $producerHandler = new class($record, $configOptions) extends AbstractProducer { + + $broker = new Broker('kafka:9092', new None()); + $producerConfigOptions = new ProducerConfigOptions( + $topic, + $broker, + null, + new AvroSchemaConfigOptions('string'), + [], + 1000, + true, + true, + 500, + 1 + ); + $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations - $config->expects() - ->make($configOptions) - ->andReturn($configManager); - - $configManager->expects() - ->middlewares() - ->andReturn([]); - - $configManager->expects() - ->get('topic_id') - ->andReturn($topicId); - - $configManager->expects() - ->get('partition') - ->andReturn(0); - - $configManager->expects() - ->get('max_poll_records') - ->andReturn(500); - - $configManager->expects() - ->get('is_async') - ->andReturn(true); - - $configManager->expects() - ->get('required_acknowledgment') - ->andReturn(true); - - $configManager->expects() - ->get('flush_attempts') - ->andReturn(1); - - $configManager->expects() - ->get('timeout') - ->andReturn(1000); - $connector->expects() - ->getProducerTopic($producerHandler, $configManager) + ->getProducerTopic($producerHandler, $producerConfigOptions) ->andReturn($kafkaProducer); $kafkaProducer->expects() - ->newTopic($topicId) + ->newTopic($topic) ->andReturn($producerTopic); $kafkaProducer->expects() diff --git a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php index cb78b0f7..f5c07f80 100644 --- a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php +++ b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php @@ -33,6 +33,7 @@ public function testShouldMakeConfigOptionWithAvroSchema(): void ]; $topicData = [ 'topic_id' => 'kafka-test', + 'consumer_group' => 'test-consumer-group', 'consumer' => [ 'consumer_groups' => [ 'test-consumer-group' => [ From d33816e2ebb00fa1c7be5c0eea48356dc669c785 Mon Sep 17 00:00:00 2001 From: David Franca Date: Mon, 21 Mar 2022 16:47:14 -0300 Subject: [PATCH 20/72] fix: update consumer middlewares --- src/Connectors/Consumer/Factory.php | 11 ++++++++++- src/Console/ConsumerCommand.php | 5 +++++ src/Facades/Metamorphosis.php | 1 + src/Producer.php | 1 - tests/Integration/ProducerTest.php | 27 ++++++++++++++++++++------- 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index 6a7c61aa..0da13cac 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -3,6 +3,7 @@ namespace Metamorphosis\Connectors\Consumer; use Metamorphosis\Consumers\ConsumerInterface; +use Metamorphosis\Middlewares\Handler\Consumer as ConsumerMiddleware; use Metamorphosis\Middlewares\Handler\Dispatcher; use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; @@ -23,9 +24,17 @@ public static function make(ConsumerConfigOptions $configOptions): Manager $commitAsync = $configOptions->isCommitASync(); $consumer = self::getConsumer($autoCommit, $configOptions); + $handler = app($configOptions->getHandler()); - $dispatcher = self::getMiddlewareDispatcher($configOptions->getMiddlewares()); + $middlewares = $configOptions->getMiddlewares(); + foreach ($middlewares as &$middleware) { + $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $configOptions]) : $middleware; + } + + $middlewares[] = app(ConsumerMiddleware::class, ['consumerTopicHandler' => $handler]); + + $dispatcher = self::getMiddlewareDispatcher($middlewares); return new Manager($consumer, $handler, $dispatcher, $autoCommit, $commitAsync); } diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index 5c233db4..bd4272e1 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -40,6 +40,11 @@ public function handle(Config $config) { $consumer = $config->make($this->option(), $this->argument()); + $middlewares = $consumer->getMiddlewares(); + foreach ($middlewares as &$middleware) { + $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $consumer]) : $middleware; + } + $this->writeStartingConsumer($consumer); $manager = Factory::make($consumer); diff --git a/src/Facades/Metamorphosis.php b/src/Facades/Metamorphosis.php index 70f903c8..c009a5f5 100644 --- a/src/Facades/Metamorphosis.php +++ b/src/Facades/Metamorphosis.php @@ -3,6 +3,7 @@ namespace Metamorphosis\Facades; use Illuminate\Support\Facades\Facade; +use Metamorphosis\TopicHandler\Producer\HandlerInterface; /** * @method static void produce(HandlerInterface $producerHandler) diff --git a/src/Producer.php b/src/Producer.php index ee277fdf..4fa9ee5b 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -35,7 +35,6 @@ public function build(HandlerInterface $producerHandler): Dispatcher $producerConfigOptions = $producerHandler->getConfigOptions(); $middlewares = $producerConfigOptions->getMiddlewares(); - foreach ($middlewares as &$middleware) { $middleware = is_string($middleware) ? app($middleware, ['producerConfigOptions' => $producerConfigOptions]) : $middleware; } diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 9780d74c..ec43989b 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -77,6 +77,8 @@ protected function haveAConsumerPartitionConfigured(): void protected function runTheConsumer(): void { + config(['kafka.topics.default.topic_id' => $this->topicId]); + $this->artisan( 'kafka:consume', [ @@ -139,9 +141,13 @@ protected function runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingT private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $producerConfigOptions = $this->createProducerConfigOptions('kafka-test'); -// $producer = app(MessageProducer::class, ['record' => $this->highLevelMessage, 'producer'=> $producerConfigOptions]); - $producer = new MessageProducer($this->highLevelMessage, $producerConfigOptions, 'recordId123'); + $this->topicId = 'kafka-test-'.Str::random(5); + $producerConfigOptions = $this->createProducerConfigOptions(); + $producer = app(MessageProducer::class, [ + 'record' => $this->highLevelMessage, + 'producer' => $producerConfigOptions, + 'key' => 'recordId123', + ]); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -150,7 +156,11 @@ private function haveSomeRandomMessagesProduced(): void private function produceRecordMessage(string $record): string { $producerConfigOptions = $this->createProducerConfigOptions('low_level'); - $producer = new MessageProducer($record, $producerConfigOptions, 'recordId123'); + $producer = app(MessageProducer::class, [ + 'record' => $record, + 'producer' => $producerConfigOptions, + 'key' => 'recordId123' + ]); Metamorphosis::produce($producer); Metamorphosis::produce($producer); @@ -190,15 +200,18 @@ private function haveFourProducedMessages(): void $this->produceRecordMessage($this->secondLowLevelMessage); } - private function createProducerConfigOptions(string $topicId = 'kafka-test'): ProducerConfigOptions + private function createProducerConfigOptions(): ProducerConfigOptions { $broker = new Broker('kafka:9092', new None()); return new ProducerConfigOptions( - $topicId, + $this->topicId, $broker, null, null, - [] + [], + 2000, + false, + true ); } } From 7b7407baa56cd9d698e2ac83db521572cf73f71a Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:21:58 -0300 Subject: [PATCH 21/72] fix: get and set options from consumer command --- src/Connectors/Consumer/Config.php | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 4db8f76a..1b0b5c6e 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -56,7 +56,7 @@ public function make(array $options, array $arguments): Consumer $configName = $options['config_name'] ?? 'kafka'; $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); $consumerGroupId = $this->getConsumerGroup($topicConfig, $arguments['consumer_group']); - $consumerConfig = $this->getConsumerConfig($topicConfig, $arguments, $consumerGroupId); + $consumerConfig = $this->getConsumerConfig($topicConfig, $options, $consumerGroupId); $brokerConfig = $this->getBrokerConfig($configName, $topicConfig['broker']); $schemaConfig = $this->getSchemaConfig($configName, $arguments['topic']); $override = array_merge($this->filterValues($options), $this->filterValues($arguments)); @@ -68,8 +68,19 @@ public function make(array $options, array $arguments): Consumer ); $this->validate(array_merge($config, $override)); - if (isset($topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'])) { - $topicConfig['consumer']['consumer_groups'][$consumerConfig['consumer_group']]['partition'] = $consumerConfig['partition']; + + if (isset($topicConfig['consumer']['consumer_groups'][$consumerGroupId])) { + if (isset($options['partition'])) { + $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['partition'] = $options['partition']; + } + + if (isset($options['offset'])) { + $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['offset'] = $options['offset']; + } + + if (isset($options['timeout'])) { + $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['timeout'] = $options['timeout']; + } } $topicConfig['consumer_group'] = $consumerGroupId; @@ -95,7 +106,7 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } - private function getConsumerConfig(array $topicConfig, array $arguments, string $consumerGroupId): array + private function getConsumerConfig(array $topicConfig, array $options, string $consumerGroupId): array { $consumerConfig = $topicConfig['consumer']['consumer_groups'][$consumerGroupId] ?? null; if (!$consumerConfig) { @@ -104,10 +115,6 @@ private function getConsumerConfig(array $topicConfig, array $arguments, string $consumerConfig['consumer_group'] = $consumerGroupId; - if (isset($arguments['partition'])) { - $consumerConfig['partition'] = $arguments['partition']; - } - return $consumerConfig; } From b8059b38d5f004627e53754a8933a8e2ea4a6946 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:23:16 -0300 Subject: [PATCH 22/72] chore: remove inutilized code --- src/Console/ConsumerCommand.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index bd4272e1..70f151e0 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -39,12 +39,6 @@ class ConsumerCommand extends BaseCommand public function handle(Config $config) { $consumer = $config->make($this->option(), $this->argument()); - - $middlewares = $consumer->getMiddlewares(); - foreach ($middlewares as &$middleware) { - $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $consumer]) : $middleware; - } - $this->writeStartingConsumer($consumer); $manager = Factory::make($consumer); From a8291f4332b6061788968bdec13fd0db40117738 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:26:34 -0300 Subject: [PATCH 23/72] chore: create public variable for message --- tests/Integration/ProducerTest.php | 34 ++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index ec43989b..5a456a71 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -18,7 +18,22 @@ class ProducerTest extends LaravelTestCase protected string $firstLowLevelMessage; - protected string $secondLowLevelMessage; + /** + * @var string + */ + protected $secondLowLevelMessage; + + /** + * @var string + */ + protected $topicId; + + protected function setUp(): void + { + parent::setUp(); + $this->withoutAuthentication(); + $this->topicId = 'kafka-test-'.Str::random(5); + } /** * @group runProducer @@ -51,13 +66,6 @@ public function testShouldRunAProducerAndReceiveMessagesWithALowLevelConsumer(): $this->runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingToTwoMessagesConsumed(); } - protected function setUp(): void - { - parent::setUp(); - - $this->withoutAuthentication(); - } - protected function withoutAuthentication(): void { config(['kafka.brokers.default.auth' => []]); @@ -141,8 +149,8 @@ protected function runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingT private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $this->topicId = 'kafka-test-'.Str::random(5); - $producerConfigOptions = $this->createProducerConfigOptions(); + + $producerConfigOptions = $this->createProducerConfigOptions($this->topicId); $producer = app(MessageProducer::class, [ 'record' => $this->highLevelMessage, 'producer' => $producerConfigOptions, @@ -159,7 +167,7 @@ private function produceRecordMessage(string $record): string $producer = app(MessageProducer::class, [ 'record' => $record, 'producer' => $producerConfigOptions, - 'key' => 'recordId123' + 'key' => 'recordId123', ]); Metamorphosis::produce($producer); @@ -200,11 +208,11 @@ private function haveFourProducedMessages(): void $this->produceRecordMessage($this->secondLowLevelMessage); } - private function createProducerConfigOptions(): ProducerConfigOptions + private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { $broker = new Broker('kafka:9092', new None()); return new ProducerConfigOptions( - $this->topicId, + $topicId, $broker, null, null, From 1d677b091eaa4f5ccd9adb79758ec9383c4033f2 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:34:21 -0300 Subject: [PATCH 24/72] chore: create public variable for message --- tests/Integration/ProducerWithAvroTest.php | 65 ++++++++++++++++++---- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index 449370da..014100ef 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -6,15 +6,30 @@ use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\HandlerStack; use GuzzleHttp\Psr7\Response; +use Illuminate\Support\Facades\Log; use Metamorphosis\Facades\Metamorphosis; use Metamorphosis\Middlewares\AvroSchemaDecoder; use Metamorphosis\Middlewares\AvroSchemaMixedEncoder; +use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; +use Metamorphosis\TopicHandler\ConfigOptions\Broker; +use Metamorphosis\TopicHandler\ConfigOptions\Producer as ProducerConfigOptions; use Tests\Integration\Dummies\MessageConsumer; use Tests\Integration\Dummies\MessageProducer; use Tests\LaravelTestCase; class ProducerWithAvroTest extends LaravelTestCase { + /** + * @var string[] + */ + protected $records; + + public function setUp(): void + { + parent::setUp(); + $this->records = ['saleOrderId' => 'SALE_ORDER_ID', 'productId' => 'PRODUCT_ID']; + } + public function testShouldRunAProducerMessagesWithAAvroSchema(): void { // Given That I @@ -22,7 +37,9 @@ public function testShouldRunAProducerMessagesWithAAvroSchema(): void // When I $this->haveSomeRandomMessagesProduced(); - $this->expectNotToPerformAssertions(); + + // I expect that + $this->myMessagesHaveBeenLogged(); } protected function haveAHandlerConfigured(): void @@ -93,14 +110,17 @@ protected function haveAHandlerConfigured(): void private function haveSomeRandomMessagesProduced(): void { - $saleOrderProducer = app( - MessageProducer::class, - ['record' => ['saleOrderId' => 'SALE_ORDER_ID'], 'topic' => 'sale_order'] - ); - $productProducer = app( - MessageProducer::class, - ['record' => ['productId' => 'PRODUCT_ID'], 'topic' => 'product'] - ); + $producerConfigOptionsSale = $this->createProducerConfigOptions('sale_order'); + $producerConfigOptionsProduct = $this->createProducerConfigOptions('product'); + + $saleOrderProducer = app(MessageProducer::class, [ + 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], + 'producer' => $producerConfigOptionsSale, + ]); + $productProducer = app(MessageProducer::class, [ + 'record' => ['productId' => 'PRODUCT_ID'], + 'producer' => $producerConfigOptionsProduct, + ]); $saleOrderSchemaResponse = '{ "subject":"sale_order-value", @@ -129,7 +149,32 @@ private function haveSomeRandomMessagesProduced(): void $productDispatcher = Metamorphosis::build($productProducer); $productDispatcher->handle($productProducer->createRecord()); + } - $saleOrderDispatcher->handle($saleOrderProducer->createRecord()); + private function myMessagesHaveBeenLogged(): void + { + $this->setLogExpectationsFor($this->records['saleOrderId']); + $this->setLogExpectationsFor($this->records['productId']); + } + + private function setLogExpectationsFor(string $message): void + { + Log::shouldReceive('info') + ->with($message); + } + + private function createProducerConfigOptions(string $topicId): ProducerConfigOptions + { + $broker = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( + $topicId, + $broker, + null, + null, + [], + 2000, + false, + true + ); } } From 881960655c4b7374cbd16b990c24007d7a05c496 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:37:24 -0300 Subject: [PATCH 25/72] chore: create public variable for message --- tests/Integration/ProducerWithConfigOptionsTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 4e256a42..f4f58b94 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -88,7 +88,7 @@ private function haveSomeRandomMessageProduced(): void MessageProducerWithConfigOptions::class, [ 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], - 'configOptions' => $this->producerConfigOptions, + 'producer' => $this->producerConfigOptions, 'key' => 1, ] ); From 5b718ec10f3170dc888ac486770e0fb54df639ba Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 10:38:44 -0300 Subject: [PATCH 26/72] fix: change parameter order --- tests/Unit/Connectors/Consumer/FactoryTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Unit/Connectors/Consumer/FactoryTest.php b/tests/Unit/Connectors/Consumer/FactoryTest.php index 50a158f2..ae3dedf6 100644 --- a/tests/Unit/Connectors/Consumer/FactoryTest.php +++ b/tests/Unit/Connectors/Consumer/FactoryTest.php @@ -58,11 +58,11 @@ public function testItMakesManagerWithLowLevelConsumer(): void $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); } - public function testItMakesManagerWithLowLevelConsumerWhenPartitionIsNotValid(): void + public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid(): void { // Set $config = new Config(); - $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition', 'partition' => -1]); + $configConsumer = $config->make(['timeout' => 61, 'partition' => -1], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); $manager = Factory::make($configConsumer); // Assertions From 62a193847abb8c193e8ac9ce6b8cdfe94396b546 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 16:44:23 -0300 Subject: [PATCH 27/72] docs: add CHANGELOG file --- CHANGELOG.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..0ecf5eae --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,40 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- + +### Fixed + +- + +### Removed +- Remove deprecated class AbstractHandler + +## [4.1.0] - 2022-03-23 + +### Added +- Added AvroSchemaMixedEncoderTest +- Added AvroSchemaDecoderTest +- Added ProducerWithConfigOptionsTest +- Added ConfigOptionsCommand to run commands with ConfigOptions class +- Added pt_BR contributing section +- Added setup-dev script on composer +- Added grumphp commit validation + +### Fixed +- Fixed parameters and options override on Consumer\Config class +- Update instructions on contribute section +- Update project install section + +### Changed +- Updated class from ConfigManager to ConfigOptions on unit tests +- Updated class from ConfigManager to ConfigOptions where any config request was made +- Consumer and Producer middlewares resolution From d8bf597e650b2d6689a64809852edf707562857c Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 23 Mar 2022 17:12:50 -0300 Subject: [PATCH 28/72] docs: add CHANGELOG file --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ecf5eae..32abee8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - +### Changed + +- + ### Removed - Remove deprecated class AbstractHandler From 8ddde21470997c2133e3f45990e18995c889b8f4 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 11:50:50 -0300 Subject: [PATCH 29/72] chore: rename parameter from 'producer' to 'configOptions' This would be a breaking change, so it's better keep as 'configOptions' by now. --- src/TopicHandler/Producer/AbstractProducer.php | 4 ++-- tests/Integration/ConsumerTest.php | 2 +- tests/Integration/ProducerTest.php | 4 ++-- tests/Integration/ProducerWithAvroTest.php | 4 ++-- tests/Integration/ProducerWithConfigOptionsTest.php | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/TopicHandler/Producer/AbstractProducer.php b/src/TopicHandler/Producer/AbstractProducer.php index 946db3ce..694aedd0 100644 --- a/src/TopicHandler/Producer/AbstractProducer.php +++ b/src/TopicHandler/Producer/AbstractProducer.php @@ -20,11 +20,11 @@ class AbstractProducer implements HandlerInterface */ protected $producer; - public function __construct($record, Producer $producer, string $key = null) + public function __construct($record, Producer $configOptions, string $key = null) { $this->record = $record; $this->key = $key; - $this->producer = $producer; + $this->producer = $configOptions; } public function getConfigOptions(): Producer diff --git a/tests/Integration/ConsumerTest.php b/tests/Integration/ConsumerTest.php index 7088e46b..1d4397e8 100644 --- a/tests/Integration/ConsumerTest.php +++ b/tests/Integration/ConsumerTest.php @@ -51,7 +51,7 @@ public function testItShouldSetup(): void MessageProducerWithConfigOptions::class, [ 'record' => ['id' => 'MESSAGE_ID'], - 'producer' => $producerConfigOptions, + 'configOptions' => $producerConfigOptions, 'key' => 1, ] ); diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 5a456a71..73852c4c 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -153,7 +153,7 @@ private function haveSomeRandomMessagesProduced(): void $producerConfigOptions = $this->createProducerConfigOptions($this->topicId); $producer = app(MessageProducer::class, [ 'record' => $this->highLevelMessage, - 'producer' => $producerConfigOptions, + 'configOptions' => $producerConfigOptions, 'key' => 'recordId123', ]); @@ -166,7 +166,7 @@ private function produceRecordMessage(string $record): string $producerConfigOptions = $this->createProducerConfigOptions('low_level'); $producer = app(MessageProducer::class, [ 'record' => $record, - 'producer' => $producerConfigOptions, + 'configOptions' => $producerConfigOptions, 'key' => 'recordId123', ]); diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index 014100ef..b0c801ad 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -115,11 +115,11 @@ private function haveSomeRandomMessagesProduced(): void $saleOrderProducer = app(MessageProducer::class, [ 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], - 'producer' => $producerConfigOptionsSale, + 'configOptions' => $producerConfigOptionsSale, ]); $productProducer = app(MessageProducer::class, [ 'record' => ['productId' => 'PRODUCT_ID'], - 'producer' => $producerConfigOptionsProduct, + 'configOptions' => $producerConfigOptionsProduct, ]); $saleOrderSchemaResponse = '{ diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index f4f58b94..4e256a42 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -88,7 +88,7 @@ private function haveSomeRandomMessageProduced(): void MessageProducerWithConfigOptions::class, [ 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], - 'producer' => $this->producerConfigOptions, + 'configOptions' => $this->producerConfigOptions, 'key' => 1, ] ); From f61dd2dacc5275888d233a6e537423e7e6e422a8 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 14:44:51 -0300 Subject: [PATCH 30/72] chore: remove annotation --- src/Facades/Metamorphosis.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Facades/Metamorphosis.php b/src/Facades/Metamorphosis.php index c009a5f5..b3c934ca 100644 --- a/src/Facades/Metamorphosis.php +++ b/src/Facades/Metamorphosis.php @@ -3,11 +3,7 @@ namespace Metamorphosis\Facades; use Illuminate\Support\Facades\Facade; -use Metamorphosis\TopicHandler\Producer\HandlerInterface; -/** - * @method static void produce(HandlerInterface $producerHandler) - */ class Metamorphosis extends Facade { protected static function getFacadeAccessor() From 4b83d91a7cfc091b6cda980ab1a1e481753cd55c Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 20:42:10 -0300 Subject: [PATCH 31/72] chore: update middleware resolution on consumerCommand --- src/Console/ConsumerCommand.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index 70f151e0..bd4272e1 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -39,6 +39,12 @@ class ConsumerCommand extends BaseCommand public function handle(Config $config) { $consumer = $config->make($this->option(), $this->argument()); + + $middlewares = $consumer->getMiddlewares(); + foreach ($middlewares as &$middleware) { + $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $consumer]) : $middleware; + } + $this->writeStartingConsumer($consumer); $manager = Factory::make($consumer); From 2a8149c508eda5b6bb5f45ccc948ddc0059e86e1 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 21:43:32 -0300 Subject: [PATCH 32/72] chore: remove ManageTest --- tests/Unit/ManagerTest.php | 66 -------------------------------------- 1 file changed, 66 deletions(-) diff --git a/tests/Unit/ManagerTest.php b/tests/Unit/ManagerTest.php index 846db3f5..e69de29b 100644 --- a/tests/Unit/ManagerTest.php +++ b/tests/Unit/ManagerTest.php @@ -1,66 +0,0 @@ -set([ - 'topic' => 'products', - 'middlewares' => [ - Log::class, - app(JsonDecode::class), - ], - 'other' => 'config', - ]); - - // Actions - $other = $manager->get('other'); - $middlewares = $manager->middlewares(); - - // Assertions - $this->assertSame('config', $other); - $this->assertInstanceOf(Log::class, $middlewares[0]); - $this->assertInstanceOf(JsonDecode::class, $middlewares[1]); - } - - public function testShouldRemoveOldMiddlewareBeforeAddOthers(): void - { - // Set - $manager = new ConsumerConfigManager(); - $firstConfig = [ - 'topic' => 'products', - 'middlewares' => [ - Log::class, - app(JsonDecode::class), - ], - 'other' => 'config', - ]; - $secondConfig = [ - 'topic' => 'products', - 'middlewares' => [ - JsonDecode::class, - ], - 'other' => 'config', - ]; - - // Actions - $manager->set($firstConfig); - $manager->set($secondConfig); - $other = $manager->get('other'); - $middlewares = $manager->middlewares(); - - // Assertions - $this->assertSame('config', $other); - $this->assertInstanceOf(JsonDecode::class, $middlewares[0]); - $this->assertCount(1, $middlewares); - } -} From 16118fd77c51f2b5f41b78312fcf39338f82c527 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 21:47:10 -0300 Subject: [PATCH 33/72] chore: remove occurrences of ConsumerConfigManager --- src/Consumer.php | 4 +--- tests/Unit/Connectors/Consumer/ManagerTest.php | 2 -- tests/Unit/Consumers/LowLevelTest.php | 3 --- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Consumer.php b/src/Consumer.php index 1d35a0c1..367556fa 100644 --- a/src/Consumer.php +++ b/src/Consumer.php @@ -15,10 +15,8 @@ class Consumer private Dispatcher $dispatcher; - public function __construct(ConsumerConfigManager $configManager, ConsumerConfigOptions $configOptions) + public function __construct(ConsumerConfigOptions $configOptions) { - $configManager->set($configOptions->toArray()); - $this->consumer = Factory::getConsumer(true, $configOptions); $this->dispatcher = new Dispatcher($configOptions->getMiddlewares()); } diff --git a/tests/Unit/Connectors/Consumer/ManagerTest.php b/tests/Unit/Connectors/Consumer/ManagerTest.php index d69d72eb..230513b2 100644 --- a/tests/Unit/Connectors/Consumer/ManagerTest.php +++ b/tests/Unit/Connectors/Consumer/ManagerTest.php @@ -4,7 +4,6 @@ use Exception; use Metamorphosis\Connectors\Consumer\Manager; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\ConsumerInterface; use Metamorphosis\Exceptions\ResponseTimeoutException; use Metamorphosis\Exceptions\ResponseWarningException; @@ -14,7 +13,6 @@ use Mockery as m; use RdKafka\Message as KafkaMessage; use Tests\LaravelTestCase; -use Tests\Unit\Dummies\ConsumerHandlerDummy; class ManagerTest extends LaravelTestCase { diff --git a/tests/Unit/Consumers/LowLevelTest.php b/tests/Unit/Consumers/LowLevelTest.php index cc8bb988..b39ac2aa 100644 --- a/tests/Unit/Consumers/LowLevelTest.php +++ b/tests/Unit/Consumers/LowLevelTest.php @@ -2,7 +2,6 @@ namespace Tests\Unit\Consumers; -use Metamorphosis\ConsumerConfigManager; use Metamorphosis\Consumers\LowLevel; use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; @@ -20,8 +19,6 @@ public function testItShouldConsume(): void // Set $timeout = 2; $partition = 3; - $configManager = new ConsumerConfigManager(); - $configManager->set(compact('timeout', 'partition')); $brokerOptions = new Broker('kafka:9092', new None()); $consumerConfigOptions = new ConsumerConfigOptions( From 3df64261c55e57aafc7f4904f936e4284f839097 Mon Sep 17 00:00:00 2001 From: hcdias Date: Thu, 24 Mar 2022 21:47:56 -0300 Subject: [PATCH 34/72] chore: remove TODO comment --- src/Connectors/Consumer/Factory.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index 0da13cac..c82c4ad2 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -13,9 +13,6 @@ * means that we will use the low level approach. */ -/** - * TODO Rename this class to ConsumerFactory, it will improve semantics - */ class Factory { public static function make(ConsumerConfigOptions $configOptions): Manager From c0d23785958f153dd7fe90f606054d0327da736f Mon Sep 17 00:00:00 2001 From: hcdias Date: Fri, 25 Mar 2022 10:20:06 -0300 Subject: [PATCH 35/72] chore: remove ProducerConfigManager occurences --- src/ConsumerConfigManager.php | 54 ------------------- .../Unit/Middlewares/Handler/ProducerTest.php | 1 - tests/Unit/Producer/PollTest.php | 12 ----- 3 files changed, 67 deletions(-) delete mode 100644 src/ConsumerConfigManager.php diff --git a/src/ConsumerConfigManager.php b/src/ConsumerConfigManager.php deleted file mode 100644 index 88207c97..00000000 --- a/src/ConsumerConfigManager.php +++ /dev/null @@ -1,54 +0,0 @@ -setConfig($config, $consumerHandler); - $this->setCommandConfig($commandConfig); - - $middlewares = $this->get('middlewares', []); - $this->middlewares = []; - $this->remove('middlewares'); - - foreach ($middlewares as $middleware) { - $this->middlewares[] = is_string($middleware) ? app($middleware) : $middleware; - } - - if (!$consumerHandler) { - return; - } - - $this->middlewares[] = new ConsumerMiddleware($consumerHandler); - } - - private function setCommandConfig(?array $commandConfig): void - { - if (!$commandConfig) { - return; - } - - $this->setting = array_merge($this->setting, $commandConfig); - } - - private function setConfig(array $config, ?AbstractHandler $handler): void - { - if (!$handler || !$overrideConfig = $handler->getConfigOptions()) { - $this->setting = $config; - - return; - } - - $this->setting = $overrideConfig->toArray(); - } -} diff --git a/tests/Unit/Middlewares/Handler/ProducerTest.php b/tests/Unit/Middlewares/Handler/ProducerTest.php index 3a7bbc70..df2750e6 100644 --- a/tests/Unit/Middlewares/Handler/ProducerTest.php +++ b/tests/Unit/Middlewares/Handler/ProducerTest.php @@ -5,7 +5,6 @@ use Closure; use Metamorphosis\Middlewares\Handler\Producer; use Metamorphosis\Producer\Poll; -use Metamorphosis\ProducerConfigManager; use Metamorphosis\Record\ProducerRecord; use Mockery as m; use RdKafka\ProducerTopic as KafkaTopicProducer; diff --git a/tests/Unit/Producer/PollTest.php b/tests/Unit/Producer/PollTest.php index 3ef21c96..0576cd6c 100644 --- a/tests/Unit/Producer/PollTest.php +++ b/tests/Unit/Producer/PollTest.php @@ -3,7 +3,6 @@ namespace Tests\Unit\Producer; use Metamorphosis\Producer\Poll; -use Metamorphosis\ProducerConfigManager; use Metamorphosis\TopicHandler\ConfigOptions\Auth\None; use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema as AvroSchemaConfigOptions; use Metamorphosis\TopicHandler\ConfigOptions\Broker; @@ -48,17 +47,6 @@ public function testItShouldHandleMessageWithoutAcknowledgment(): void public function testShouldThrowExceptionWhenFlushFailed(): void { // Set - $configManager = new ProducerConfigManager(); - $configManager->set([ - 'topic_id' => 'topic_name', - 'timeout' => 1000, - 'is_async' => false, - 'max_poll_records' => 500, - 'flush_attempts' => 3, - 'required_acknowledgment' => true, - 'partition' => 0, - ]); - $broker = new Broker('kafka:9092', new None()); $producerConfigOptions = new ProducerConfigOptions( 'topic_name', From 353a22852367c83267d8fe574c31e99ca33be26f Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 19 Apr 2022 16:10:10 -0300 Subject: [PATCH 36/72] chore: remove unecessary interface --- src/Avro/AvroClientInterface.php | 6 ------ src/Avro/CachedSchemaRegistryClientInterface.php | 7 ------- 2 files changed, 13 deletions(-) delete mode 100644 src/Avro/AvroClientInterface.php delete mode 100644 src/Avro/CachedSchemaRegistryClientInterface.php diff --git a/src/Avro/AvroClientInterface.php b/src/Avro/AvroClientInterface.php deleted file mode 100644 index d7617023..00000000 --- a/src/Avro/AvroClientInterface.php +++ /dev/null @@ -1,6 +0,0 @@ - Date: Tue, 19 Apr 2022 16:12:26 -0300 Subject: [PATCH 37/72] chore: remove unecessary code block --- src/Console/ConsumerCommand.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index bd4272e1..5c233db4 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -40,11 +40,6 @@ public function handle(Config $config) { $consumer = $config->make($this->option(), $this->argument()); - $middlewares = $consumer->getMiddlewares(); - foreach ($middlewares as &$middleware) { - $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $consumer]) : $middleware; - } - $this->writeStartingConsumer($consumer); $manager = Factory::make($consumer); From 9342da49c15b91e74560696be498be0873619228 Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 19 Apr 2022 16:13:45 -0300 Subject: [PATCH 38/72] chore: define constant visibility --- src/Avro/ClientFactory.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Avro/ClientFactory.php b/src/Avro/ClientFactory.php index b30d0c1b..388a8fed 100644 --- a/src/Avro/ClientFactory.php +++ b/src/Avro/ClientFactory.php @@ -7,7 +7,7 @@ class ClientFactory { - const REQUEST_TIMEOUT = 2000; + protected const REQUEST_TIMEOUT = 2000; public function make(AvroSchema $avroSchema): CachedSchemaRegistryClient { From 9fa3113faef7df4965c1a81edc47780f6c5ed038 Mon Sep 17 00:00:00 2001 From: hcdias Date: Fri, 1 Apr 2022 15:59:18 -0300 Subject: [PATCH 39/72] chore: change config file format --- config/kafka.php | 168 ++++-------------- config/service.php | 28 +++ src/Connectors/AbstractConfig.php | 24 +-- src/Connectors/Consumer/Config.php | 56 +----- src/Connectors/Producer/Config.php | 12 +- src/MetamorphosisServiceProvider.php | 6 +- .../Factories/ConsumerFactory.php | 6 +- 7 files changed, 86 insertions(+), 214 deletions(-) create mode 100644 config/service.php diff --git a/config/kafka.php b/config/kafka.php index 9f9ae370..9e74693b 100644 --- a/config/kafka.php +++ b/config/kafka.php @@ -1,124 +1,46 @@ [ - 'default' => [ - 'url' => '', - // Disable SSL verification on schema request. - 'ssl_verify' => true, - // This option will be put directly into a Guzzle http request - // Use this to do authorizations or send any headers you want. - // Here is a example of basic authentication on AVRO schema. - 'request_options' => [ - 'headers' => [ - 'Authorization' => [ - 'Basic ' . base64_encode( - env('AVRO_SCHEMA_USERNAME') - . ':' - . env('AVRO_SCHEMA_PASSWORD') - ), - ], - ], - ], - ], - ], - - /* - |-------------------------------------------------------------------------- - | Brokers - |-------------------------------------------------------------------------- - | - | Here you may specify the connections details for each broker configured - | on topic's broker key. - | - */ - - 'brokers' => [ - 'default' => [ - 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), - - // If your broker doest not have authentication, you can - // remove this configuration, or set as empty. - // The Authentication types may be "ssl" or "none" - 'auth' => [ - 'type' => 'ssl', // ssl and none - 'ca' => storage_path('ca.pem'), - 'certificate' => storage_path('kafka.cert'), - 'key' => storage_path('kafka.key'), - ], - ], - ], - 'topics' => [ - // This is your topic "keyword" where you will put all configurations needed - // on this specific topic. 'default' => [ - // The topic id is where you want to send or consume - // your messages from kafka. 'topic_id' => 'kafka-test', - - // Here you may point the key of the broker configured above. - 'broker' => 'default', - - // Configurations specific for consumer 'consumer' => [ - // You may define more than one consumer group per topic. - // If there is just one defined, it will be used by default, - // otherwise, you may pass which consumer group should be used - // when using the consumer command. - 'consumer_groups' => [ - 'test-consumer-group' => [ - - // Action to take when there is no initial - // offset in offset store or the desired offset is out of range. - // This config will be passed to 'auto.offset.reset'. - // The valid options are: smallest, earliest, beginning, largest, latest, end, error. - 'offset_reset' => 'earliest', - - // The offset at which to start consumption. This only applies if partition is set. - // You can use a positive integer or any of the constants: RD_KAFKA_OFFSET_BEGINNING, - // RD_KAFKA_OFFSET_END, RD_KAFKA_OFFSET_STORED. - 'offset' => 0, - - // The partition to consume. It can be null, - // if you don't wish do specify one. - 'partition' => 0, - - // A consumer class that implements ConsumerTopicHandler - 'handler' => '\App\Kafka\Consumers\ConsumerExample', - - // A Timeout to listen to a message. That means: how much - // time we need to wait until receiving a message? - 'timeout' => 20000, - - // Once you've enabled this, the Kafka consumer will commit the - // offset of the last message received in response to its poll() call - 'auto_commit' => true, - - // If commit_async is false process block until offsets are committed or the commit fails. - // Only works when auto_commit is false - 'commit_async' => false, - - // An array of middlewares applied only for this consumer_group - 'middlewares' => [], - ], - ], + 'consumer_group' => 'test-consumer-group', + // Action to take when there is no initial + // offset in offset store or the desired offset is out of range. + // This config will be passed to 'auto.offset.reset'. + // The valid options are: smallest, earliest, beginning, largest, latest, end, error. + 'offset_reset' => 'earliest', + + // The offset at which to start consumption. This only applies if partition is set. + // You can use a positive integer or any of the constants: RD_KAFKA_OFFSET_BEGINNING, + // RD_KAFKA_OFFSET_END, RD_KAFKA_OFFSET_STORED. + 'offset' => 0, + + // The partition to consume. It can be null, + // if you don't wish do specify one. + 'partition' => 0, + + // A consumer class that implements ConsumerTopicHandler + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + + // A Timeout to listen to a message. That means: how much + // time we need to wait until receiving a message? + 'timeout' => 20000, + + // Once you've enabled this, the Kafka consumer will commit the + // offset of the last message received in response to its poll() call + 'auto_commit' => true, + + // If commit_async is false process block until offsets are committed or the commit fails. + // Only works when auto_commit is false + 'commit_async' => false, + + // An array of middlewares applied only for this consumer_group + 'middlewares' => [], ], - // Configurations specific for producer 'producer' => [ - // Sets to true if you want to know if a message was successfully posted. 'required_acknowledgment' => true, @@ -149,28 +71,4 @@ ], ], ], - - /* - |-------------------------------------------------------------------------- - | Global Middlewares - |-------------------------------------------------------------------------- - | - | Here you may specify the global middlewares that will be applied for every - | consumed topic. Middlewares work between the received data from broker and - | before being passed into consumers. - | Available middlewares: log, avro-decode - | - */ - - 'middlewares' => [ - 'consumer' => [ - \Metamorphosis\Middlewares\Log::class, - ], - 'producer' => [ - \Metamorphosis\Middlewares\Log::class, - ], - 'global' => [ - \Metamorphosis\Middlewares\Log::class, - ], - ], ]; diff --git a/config/service.php b/config/service.php new file mode 100644 index 00000000..82cb1299 --- /dev/null +++ b/config/service.php @@ -0,0 +1,28 @@ + [ + 'url' => '', + 'request_options' => [ + 'headers' => [ + 'Authorization' => [ + 'Basic '.base64_encode( + env('AVRO_SCHEMA_USERNAME').':'.env('AVRO_SCHEMA_PASSWORD') + ), + ], + ], + ], + 'ssl_verify' => true, + 'username' => 'USERNAME', + 'password' => 'PASSWORD', + ], + 'broker' => [ + 'connections' => 'kafka:9092', + 'auth' => [ + 'type' => 'ssl', // ssl and none + 'ca' => storage_path('ca.pem'), + 'certificate' => storage_path('kafka.cert'), + 'key' => storage_path('kafka.key'), + ], + ], +]; diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index 9942dc2e..fa8b8ac1 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -7,20 +7,17 @@ abstract class AbstractConfig { - /** - * @var mixed[] - */ - protected array $rules = []; - - protected function getBrokerConfig(string $configName, string $brokerId): array + protected function getBrokerConfig(string $servicesFile): array { - if (!$brokerConfig = config("{$configName}.brokers.{$brokerId}")) { - throw new ConfigurationException( - "Broker '{$brokerId}' configuration not found" - ); + if (!$brokerConfig = config($servicesFile.'.broker')) { + throw new ConfigurationException("Broker configuration not found on '{$servicesFile}'"); } + return $brokerConfig; + } - return (array) $brokerConfig; + protected function getSchemaConfig(string $servicesFile): array + { + return config($servicesFile.'.avro_schema', []); } protected function validate(array $config): void @@ -31,9 +28,4 @@ protected function validate(array $config): void throw new ConfigurationException($validator->errors()->toJson()); } } - - protected function getSchemaConfig(string $configName, string $topicId): array - { - return config($configName . '.avro_schemas.' . $topicId, []); - } } diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 1b0b5c6e..aabcfd8b 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -54,37 +54,26 @@ public function makeWithConfigOptions(string $handlerClass, ?int $times = null): public function make(array $options, array $arguments): Consumer { $configName = $options['config_name'] ?? 'kafka'; - $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); - $consumerGroupId = $this->getConsumerGroup($topicConfig, $arguments['consumer_group']); - $consumerConfig = $this->getConsumerConfig($topicConfig, $options, $consumerGroupId); - $brokerConfig = $this->getBrokerConfig($configName, $topicConfig['broker']); - $schemaConfig = $this->getSchemaConfig($configName, $arguments['topic']); - $override = array_merge($this->filterValues($options), $this->filterValues($arguments)); - $config = array_merge( - $topicConfig, - $brokerConfig, - $consumerConfig, - $schemaConfig - ); + $service = $options['service'] ?? 'service'; - $this->validate(array_merge($config, $override)); + $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); + $brokerConfig = $this->getBrokerConfig($service); + $schemaConfig = $this->getSchemaConfig($service); - if (isset($topicConfig['consumer']['consumer_groups'][$consumerGroupId])) { + if (isset($topicConfig['consumer'])) { if (isset($options['partition'])) { - $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['partition'] = $options['partition']; + $topicConfig['consumer']['partition'] = $options['partition']; } if (isset($options['offset'])) { - $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['offset'] = $options['offset']; + $topicConfig['consumer']['offset'] = $options['offset']; } if (isset($options['timeout'])) { - $topicConfig['consumer']['consumer_groups'][$consumerGroupId]['timeout'] = $options['timeout']; + $topicConfig['consumer']['timeout'] = $options['timeout']; } } - $topicConfig['consumer_group'] = $consumerGroupId; - return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); } @@ -106,35 +95,6 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } - private function getConsumerConfig(array $topicConfig, array $options, string $consumerGroupId): array - { - $consumerConfig = $topicConfig['consumer']['consumer_groups'][$consumerGroupId] ?? null; - if (!$consumerConfig) { - throw new ConfigurationException("Consumer group '{$consumerGroupId}' not found"); - } - - $consumerConfig['consumer_group'] = $consumerGroupId; - - return $consumerConfig; - } - - private function getConsumerGroup(array $topicConfig, ?string $consumerGroupId): string - { - if (!$consumerGroupId && 1 === count($topicConfig['consumer']['consumer_groups'])) { - $consumerGroupId = current(array_keys($topicConfig['consumer']['consumer_groups'])); - } - - return $consumerGroupId ?? 'default'; - } - - private function getMiddlewares(string $configName, array $topicConfig): array - { - return array_merge( - config($configName . '.middlewares.consumer', []), - $topicConfig['consumer']['middlewares'] ?? [] - ); - } - /** * Sometimes that user may pass `--partition=0` as argument. * So if we just use array_filter here, this option will diff --git a/src/Connectors/Producer/Config.php b/src/Connectors/Producer/Config.php index 90a9cd1d..e3cf8e80 100644 --- a/src/Connectors/Producer/Config.php +++ b/src/Connectors/Producer/Config.php @@ -50,15 +50,9 @@ public function make(ProducerConfigOptions $configOptions): AbstractConfigManage public function makeByTopic(string $topicId): AbstractConfigManager { $topicConfig = $this->getTopicConfig($topicId); - $topicConfig['middlewares'] = array_merge( - config('kafka.middlewares.producer', []), - $topicConfig['producer']['middlewares'] ?? [] - ); - $brokerConfig = $this->getBrokerConfig( - 'kafka', - $topicConfig['broker'] - ); - $schemaConfig = $this->getSchemaConfig('kafka', $topicId); + $topicConfig['middlewares'] = $topicConfig['producer']['middlewares'] ?? []; + $brokerConfig = $this->getBrokerConfig('service'); + $schemaConfig = $this->getSchemaConfig('service'); $config = array_merge($topicConfig, $brokerConfig, $schemaConfig); $this->validate($config); diff --git a/src/MetamorphosisServiceProvider.php b/src/MetamorphosisServiceProvider.php index 4938fbac..36dffdf5 100644 --- a/src/MetamorphosisServiceProvider.php +++ b/src/MetamorphosisServiceProvider.php @@ -14,10 +14,12 @@ class MetamorphosisServiceProvider extends ServiceProvider public function boot() { $this->publishes([ - __DIR__ . '/../config/kafka.php' => config_path('kafka.php'), + __DIR__.'/../config/kafka.php' => config_path('kafka.php'), + __DIR__.'/../config/service.php' => config_path('service.php'), ], 'config'); - $this->mergeConfigFrom(__DIR__ . '/../config/kafka.php', 'kafka'); + $this->mergeConfigFrom(__DIR__.'/../config/kafka.php', 'kafka'); + $this->mergeConfigFrom(__DIR__.'/../config/service.php', 'service'); } public function register() diff --git a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php index 7bd58c93..4dd62736 100644 --- a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php +++ b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php @@ -22,10 +22,8 @@ public static function make( private static function getConsumerGroupConfig(array $topicData): array { $topicData['topicId'] = $topicData['topic_id']; - $topicData['consumerGroup'] = $topicData['consumer_group']; - - $consumerGroup = $topicData['consumerGroup']; - $consumer = current($topicData['consumer'])[$consumerGroup]; + $consumer = $topicData['consumer']; + $topicData['consumerGroup'] = $consumer['consumer_group']; return array_merge_recursive($topicData, self::convertConfigAttributes($consumer)); } From ddc454c5391f7c2043610b4fd42f936f8bc3625c Mon Sep 17 00:00:00 2001 From: hcdias Date: Fri, 1 Apr 2022 16:09:05 -0300 Subject: [PATCH 40/72] chore: adjust tests to match new config file format --- tests/Integration/ProducerTest.php | 23 +++---- tests/Unit/Connectors/Consumer/ConfigTest.php | 7 ++- .../Unit/Connectors/Consumer/FactoryTest.php | 52 ++++++++++++++++ tests/Unit/Connectors/Producer/ConfigTest.php | 2 - tests/Unit/Console/ConsumerCommandTest.php | 61 ++++++++----------- .../Factories/ConsumerFactoryTest.php | 22 +++---- 6 files changed, 102 insertions(+), 65 deletions(-) diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 73852c4c..94ab5626 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -68,19 +68,17 @@ public function testShouldRunAProducerAndReceiveMessagesWithALowLevelConsumer(): protected function withoutAuthentication(): void { - config(['kafka.brokers.default.auth' => []]); + config(['service.broker.auth' => []]); } protected function haveAConsumerHandlerConfigured(): void { - config( - ['kafka.topics.default.consumer.consumer_groups.test-consumer-group.handler' => MessageConsumer::class] - ); + config(['kafka.topics.default.consumer.handler' => MessageConsumer::class]); } protected function haveAConsumerPartitionConfigured(): void { - config(['kafka.topics.default.consumer.consumer_groups.test-consumer-group.partition' => -1]); + config(['kafka.topics.default.consumer.partition' => -1]); } protected function runTheConsumer(): void @@ -107,15 +105,12 @@ protected function haveALowLevelConsumerConfigured(): void 'topic_id' => 'low_level', 'broker' => 'default', 'consumer' => [ - 'consumer_groups' => [ - 'test-consumer-group' => [ - 'offset_reset' => 'earliest', - 'offset' => 0, - 'handler' => MessageConsumer::class, - 'timeout' => 20000, - 'middlewares' => [], - ], - ], + 'consumer_group' => 'test-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'handler' => MessageConsumer::class, + 'timeout' => 20000, + 'middlewares' => [], ], 'producer' => [ 'required_acknowledgment' => true, diff --git a/tests/Unit/Connectors/Consumer/ConfigTest.php b/tests/Unit/Connectors/Consumer/ConfigTest.php index 539b6ea0..ffeaf5cd 100644 --- a/tests/Unit/Connectors/Consumer/ConfigTest.php +++ b/tests/Unit/Connectors/Consumer/ConfigTest.php @@ -8,6 +8,7 @@ use Mockery as m; use Tests\LaravelTestCase; use Tests\Unit\Dummies\ConsumerHandlerDummy; +use TypeError; class ConfigTest extends LaravelTestCase { @@ -104,8 +105,10 @@ public function testShouldNotSetRuntimeConfigWhenOptionsIsInvalid(): void 'consumer_group' => 'default', ]; + // Expectations + $this->expectException(TypeError::class); + // Actions - $this->expectException(ConfigurationException::class); $configManager = $config->make($options, $arguments); // Assertions @@ -115,7 +118,7 @@ public function testShouldNotSetRuntimeConfigWhenOptionsIsInvalid(): void public function testShouldNotSetRuntimeConfigWhenKafkaConfigIsInvalid(): void { // Set - config(['kafka.brokers.default.connections' => null]); + config(['service.broker' => null]); $config = new Config(); $options = [ 'partition' => 0, diff --git a/tests/Unit/Connectors/Consumer/FactoryTest.php b/tests/Unit/Connectors/Consumer/FactoryTest.php index ae3dedf6..2ccb9cd9 100644 --- a/tests/Unit/Connectors/Consumer/FactoryTest.php +++ b/tests/Unit/Connectors/Consumer/FactoryTest.php @@ -50,6 +50,8 @@ protected function setUp(): void public function testItMakesManagerWithLowLevelConsumer(): void { // Set + $this->haveAConsumerWithPartitionConfigured(); + $config = new Config(); $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); $manager = Factory::make($configConsumer); @@ -61,6 +63,7 @@ public function testItMakesManagerWithLowLevelConsumer(): void public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid(): void { // Set + $this->haveAConsumerWithoutPartitionConfigured(); $config = new Config(); $configConsumer = $config->make(['timeout' => 61, 'partition' => -1], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); $manager = Factory::make($configConsumer); @@ -72,6 +75,7 @@ public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid() public function testItMakesHighLevelClass(): void { // Set + $this->haveAConsumerWithoutPartitionConfigured(); $config = new Config(); $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'without-partition']); $manager = Factory::make($configConsumer); @@ -79,4 +83,52 @@ public function testItMakesHighLevelClass(): void // Assertions $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); } + + private function haveAConsumerWithPartitionConfigured() + { + config([ + 'kafka' => [ + 'topics' => [ + 'topic_key' => [ + 'topic_id' => 'topic_name', + 'consumer' => [ + 'consumer_group' => 'with-partition', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => ConsumerHandlerDummy::class, + ], + ], + ], + ], + 'service' => [ + 'broker' => [ + 'connections' => 'kafka:123', + ], + ], + ]); + } + + private function haveAConsumerWithoutPartitionConfigured() + { + config([ + 'kafka' => [ + 'topics' => [ + 'topic_key' => [ + 'topic_id' => 'topic_name', + 'consumer' => [ + 'consumer_group' => 'without-partition', + 'offset_reset' => 'earliest', + 'handler' => ConsumerHandlerDummy::class, + ], + ], + ], + ], + 'service' => [ + 'broker' => [ + 'connections' => 'kafka:123', + ], + ], + ]); + } } diff --git a/tests/Unit/Connectors/Producer/ConfigTest.php b/tests/Unit/Connectors/Producer/ConfigTest.php index 82adae08..d60567e1 100644 --- a/tests/Unit/Connectors/Producer/ConfigTest.php +++ b/tests/Unit/Connectors/Producer/ConfigTest.php @@ -24,7 +24,6 @@ public function testShouldValidateProducerConfig(): void 'max_poll_records' => 500, 'flush_attempts' => 10, 'partition' => -1, - 'broker' => 'default', 'topic' => 'default', 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), 'auth' => [ @@ -70,7 +69,6 @@ public function testShouldNotOverrideDefaultParametersWhenConfigIsSet(): void 'required_acknowledgment' => true, 'max_poll_records' => 3000, 'flush_attempts' => 10, - 'broker' => 'default', 'topic' => 'default', 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), 'auth' => [ diff --git a/tests/Unit/Console/ConsumerCommandTest.php b/tests/Unit/Console/ConsumerCommandTest.php index 26132745..5a5a8fcf 100644 --- a/tests/Unit/Console/ConsumerCommandTest.php +++ b/tests/Unit/Console/ConsumerCommandTest.php @@ -10,6 +10,33 @@ class ConsumerCommandTest extends LaravelTestCase { + protected function setUp(): void + { + parent::setUp(); + + config([ + 'kafka' => [ + 'topics' => [ + 'topic_key' => [ + 'topic_id' => 'topic_name', + 'consumer' => [ + 'consumer_group' => 'default', + 'offset_reset' => 'earliest', + 'handler' => ConsumerHandlerDummy::class, + 'timeout' => 123, + ], + ], + ], + ], + 'service' => [ + 'broker' => [ + 'connections' => 'test_kafka:6680', + 'auth' => [], + ], + ], + ]); + } + public function testItCallsCommandWithInvalidTopic(): void { // Set @@ -131,38 +158,4 @@ public function testItOverridesBrokerConnectionWhenCallingCommand(): void $this->artisan($command, $parameters); } - - protected function setUp(): void - { - parent::setUp(); - - config([ - 'kafka' => [ - 'brokers' => [ - 'default' => [ - 'connections' => env( - 'KAFKA_BROKER_CONNECTIONS', - 'kafka:9092' - ), - 'auth' => [], - ], - ], - 'topics' => [ - 'topic_key' => [ - 'topic_id' => 'topic_name', - 'broker' => 'default', - 'consumer' => [ - 'consumer_groups' => [ - 'default' => [ - 'offset_reset' => 'earliest', - 'handler' => ConsumerHandlerDummy::class, - 'timeout' => 123, - ], - ], - ], - ], - ], - ], - ]); - } } diff --git a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php index f5c07f80..02aa3368 100644 --- a/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php +++ b/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php @@ -33,20 +33,16 @@ public function testShouldMakeConfigOptionWithAvroSchema(): void ]; $topicData = [ 'topic_id' => 'kafka-test', - 'consumer_group' => 'test-consumer-group', 'consumer' => [ - 'consumer_groups' => [ - 'test-consumer-group' => [ - 'middlewares' => [], - 'auto_commit' => true, - 'commit_async' => true, - 'offset_reset' => 'earliest', - 'handler' => '\App\Kafka\Consumers\ConsumerExample', - 'partition' => 0, - 'offset' => 0, - 'timeout' => 20000, - ], - ], + 'consumer_group' => 'test-consumer-group', + 'middlewares' => [], + 'auto_commit' => true, + 'commit_async' => true, + 'offset_reset' => 'earliest', + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'partition' => 0, + 'offset' => 0, + 'timeout' => 20000, ], ]; $expected = [ From b060e90b555b5a2429baaa1109a21ed5fcdf3aa0 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 15:51:49 -0300 Subject: [PATCH 41/72] chore: update docs --- CHANGELOG.md | 25 ++----- docs/advanced.md | 35 ++++++--- docs/quick-usage.md | 179 +++++++++++++++++++++++--------------------- 3 files changed, 124 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32abee8f..009e3b39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,36 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- - -### Fixed - -- - -### Changed - -- - -### Removed -- Remove deprecated class AbstractHandler - -## [4.1.0] - 2022-03-23 - -### Added +- Added file `config/service.php` to configure broker and authentication - Added AvroSchemaMixedEncoderTest - Added AvroSchemaDecoderTest - Added ProducerWithConfigOptionsTest - Added ConfigOptionsCommand to run commands with ConfigOptions class - Added pt_BR contributing section - Added setup-dev script on composer -- Added grumphp commit validation +- Added grumphp commit validation + +### Fixed -### Fixed - Fixed parameters and options override on Consumer\Config class - Update instructions on contribute section - Update project install section ### Changed + - Updated class from ConfigManager to ConfigOptions on unit tests - Updated class from ConfigManager to ConfigOptions where any config request was made - Consumer and Producer middlewares resolution + +### Removed diff --git a/docs/advanced.md b/docs/advanced.md index f290b06c..acca8234 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -275,29 +275,42 @@ stdout_logfile=/var/log/default/kafka-consumer-price-update.log Although you can run this simple command, it provides some options you can pass to make it more flexible to your needs. -- `--broker=` - - Sometimes, you may want to change which broker the consumer should connect to (maybe for testing/debug purposes). - For that, you just nedd to call the `--broker` option with another broker connection key already set in the `config/kafka.php` file. - - `$ php artisan kafka:consume price-update --broker='some-other-broker'` - `--offset=` And if you need to start the consumption of a topic in a specific offset (it can be useful for debug purposes) you can pass the `--offset=` option, but for this, it will be required to specify the partition too. - `$ php artisan kafka:consume price-update --partition=2 --offset=34` + + $ php artisan kafka:consume price-update --partition=2 --offset=34 + - `--partition=` - If you wish do specify in which partition the consumer must be attached, you can set the option `--partition=`. + Set in which partition the consumer must be attached. + + + $ php artisan kafka:consume price-update --partition=2 --offset=34 - `$ php artisan kafka:consume price-update --partition=2 --offset=34` - `--timeout=` - You can specify what would be the timeout for the consumer, by using the `--timeout=` option, the time is in milliseconds. + Set the timeout for the consumer in milliseconds. + + + $ php artisan kafka:consume price-update --timeout=23000 + + +- `--config_name=` + + Specify from what file topics configuration should be read. + + + $ php artisan kafka:consume topic-name --config_name=config.file + + +- `--service_name=` - `$ php artisan kafka:consume price-update --timeout=23000` + Specify from what file services configurations should be read. + `$ php artisan kafka:consume price-update --service_name=config.file` diff --git a/docs/quick-usage.md b/docs/quick-usage.md index 3d003da4..7fd95905 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -8,76 +8,104 @@ - [Produce Message](#produce-message) -### Config file: `config/kafka.php` - -The config file holds all information about brokers, topics, consumer groups and middlewares. - -To quickly start using, we can focus in two sections: -- Brokers - - An array of brokers, with connection and authentication configurations: - - - `connections`: *required*. can be a `string` with multiple connections separated by comma or an `array` of connections (as `string`) - - - `auth`: *optional*. out of the box, the package can connect with SSL Authentication only or without any authentication - - ```php - 'brokers' => [ - 'price_brokers' => [ - 'connections' => 'localhost:8091,localhost:8092', - 'auth' => [ - 'type' => 'ssl', - 'ca' => storage_path('ca.pem'), - 'certificate' => storage_path('kafka.cert'), - 'key' => storage_path('kafka.key'), - ], - ], - 'stock_brokers' => [ - 'connections' => ['localhost:8091', 'localhost:8092'], - 'auth' => [], // can be an empty array or even don't have this key in the broker config - ], - ], - ``` - -- Topics - - An array of topics configuration, such as the topic name, which broker connection should use, consumer groups and middlewares. - - Here we can specify the group consumers, each topic can have multiple groups, - and each group holds the configuration for which consumer, offset_reset (for setting initial offset) and middleware it must use. - - ```php - 'topics' => [ - 'price_update' => [ - 'topic' => 'products.price.update', - 'broker' => 'price_brokers', - 'consumer_groups' => [ - 'default' => [ - 'offset_reset' => 'smallest', - 'handler' => '\App\Kafka\Consumers\PriceUpdateConsumer', - ], - ], - ], - ], - ``` +### Configure using files + +To get started using configuration files, at least two files are needed. A file to keep the topics +configuration and a file to keep the broker and schema configuration. In this example, we will use the files `config/kafka.php` and `config/service.php`. + +### File `config/kafka.php`: + +This file keeps configurations about topics, consumers and producers. +It should return an array of topics containing the topic name, topic_id, consumer, producer and the settings for each one of them: + + +```php + [ + 'this_is_your_topic_name' => [ + 'topic_id' => "this_is_your_topic_id", + 'consumer' => [ + 'consumer_group' => 'your-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'timeout' => 20000, + 'auto_commit' => true, + 'commit_async' => false, + 'middlewares' => [], + ], + + 'producer' => [ + 'required_acknowledgment' => true, + 'is_async' => true, + 'max_poll_records' => 500, + 'flush_attempts' => 10, + 'middlewares' => [], + 'timeout' => 10000, + 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1, + ], + ] + ], +]; +``` + +### File `config/service.php` + +This file keeps configurations about **broker** and **schema** utilized. + + +```php + [ + 'url' => '', + 'request_options' => [ + 'headers' => [ + 'Authorization' => [ + 'Basic ' . base64_encode( + env('AVRO_SCHEMA_USERNAME').':'.env('AVRO_SCHEMA_PASSWORD') + ), + ], + ], + ], + + 'ssl_verify' => true, + 'username' => 'USERNAME', + 'password' => 'PASSWORD', + ], + + 'broker' => [ + 'connections' => 'kafka:9092', + 'auth' => [ + 'type' => 'ssl', + 'ca' => storage_path('ca.pem'), + 'certificate' => storage_path('kafka.cert'), + 'key' => storage_path('kafka.key'), + ], + ], +]; +``` + ### Consumer -After setting up the required configs, you need to create the consumer, which will handle all records received -from the topic specified in the config. +After setting up the required configuration, you must create a consumer to handle records received +from the specified topic in your configuration. -#### Creating Consumer +#### Creating a Consumer -Creating the consumer is easy as running the following command: +To create a consumer run the following command: ```bash $ php artisan make:kafka-consumer PriceUpdateConsumer ``` -This will create a KafkaConsumer class inside the application, on the app/Kafka/Consumers/ directory - -There, you'll have a handler method, which will send all records from the topic to the Consumer, -also, methods will be available for handling exceptions +This will create a KafkaConsumer class on the app/Kafka/Consumers/ directory with the following +content: ```php use App\Kafka\Consumers\PriceUpdateConsumer; @@ -109,20 +137,19 @@ class PriceUpdateConsumer extends AbstractHandler ``` -#### Running consumer +#### Running the consumer -Now you just need to start consuming the topic. +To start consuming the topic, the simplest way to see it working is by running the kafka:consume command along with the topic name, topic configuration file and service configuration file: -The simplest way to see it working is by running the kafka:consume command along with the topic name -declared in the topics config key: ```bash -$ php artisan kafka:consume price-update +$ php artisan kafka:consume this_is_your_topic_name --config_name=config.file --service_name=service.file ``` This command will run in a `while true`, that means, it will never stop running. But, errors can happen, so we strongly advice you to run this command along with [supervisor](http://supervisord.org/running.html), like this example below: + ```bash [program:kafka-consumer-price-update] process_name=%(program_name)s_%(process_num)02d @@ -137,30 +164,10 @@ stdout_logfile=/var/log/default/kafka-consumer-price-update.log That's it. For more information about usage, middlewares, broker authentication, consumer groups and other advanced topics, please have a look at our [Advanced Usage Guide](advanced.md). - -### Producer - -Producer also required configs, which will produce all records using parameters specified in the config. - -```php - 'brokers' => [ - 'local-dev' => [ - 'connections' => 'kafka:9092', - ], - ], - 'topics' => [ - 'product-updated' => [ - 'topic_id' => 'product_updated', - 'broker' => 'local-dev', - ], - ], -``` ### Produce Message -Creating Producer handler. - -The Producer must extends AbstractHandler class and can be empty. +To create a producer handler, create a class that extends `Metamorphosis\TopicHandler\Producer\AbstractHandler` class: ```php Date: Mon, 4 Apr 2022 15:53:44 -0300 Subject: [PATCH 42/72] chore: update console parameters Added a new option to specify the service file --- src/Connectors/Consumer/Config.php | 2 +- src/Console/ConsumerCommand.php | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index aabcfd8b..81ba46a6 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -54,7 +54,7 @@ public function makeWithConfigOptions(string $handlerClass, ?int $times = null): public function make(array $options, array $arguments): Consumer { $configName = $options['config_name'] ?? 'kafka'; - $service = $options['service'] ?? 'service'; + $service = $options['service_name'] ?? 'service'; $topicConfig = $this->getTopicConfig($configName, $arguments['topic']); $brokerConfig = $this->getBrokerConfig($service); diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index 5c233db4..93420418 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -34,7 +34,8 @@ class ConsumerCommand extends BaseCommand {--broker= : Override broker connection from config.} {--timeout= : Sets timeout for consumer.} {--times= : Amount of messages to be consumed.} - {--config_name= : Change default name for laravel config file.}'; + {--config_name= : Change default name for laravel config file.} + {--service_name= : Change default name for services config file.}'; public function handle(Config $config) { From ab85f7e5b271f8e19071e809f02c3f3305324968 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 16:10:57 -0300 Subject: [PATCH 43/72] chore: remove unused method --- src/Connectors/Consumer/Config.php | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 81ba46a6..33673cae 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -94,18 +94,4 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } - - /** - * Sometimes that user may pass `--partition=0` as argument. - * So if we just use array_filter here, this option will - * be removed. - * - * This code makes sure that only null values will be removed. - */ - private function filterValues(array $options = []): array - { - return array_filter($options, function ($value) { - return !is_null($value); - }); - } } From f25959d4523db6459966180086a5611304e9b267 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 16:46:50 -0300 Subject: [PATCH 44/72] chore: add how to use data objects --- docs/quick-usage.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/quick-usage.md b/docs/quick-usage.md index 7fd95905..f306cba2 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -1,6 +1,7 @@ ## Quick Usage Guide -- [Config file](#config) +- [Configure with files](#config) +- [Configure using data objects](#config-dto) - [Consumer](#consumer) - [Creating Consumer](#creating-consumer) - [Running](#running-consumer) @@ -90,7 +91,6 @@ return [ ]; ``` - ### Consumer @@ -144,7 +144,7 @@ To start consuming the topic, the simplest way to see it working is by running t ```bash $ php artisan kafka:consume this_is_your_topic_name --config_name=config.file --service_name=service.file -``` +``` This command will run in a `while true`, that means, it will never stop running. But, errors can happen, so we strongly advice you to run this command along with [supervisor](http://supervisord.org/running.html), @@ -162,6 +162,27 @@ redirect_stderr=true stdout_logfile=/var/log/default/kafka-consumer-price-update.log ``` +### Using data objects + +To configure and consume using classes: + +```php + use Metamorphosis\Consumer; + use Metamorphosis\TopicHandler\ConfigOptions\Factories\ConsumerFactory; + + $topic = config('yourConfig.topics.topic-id'); + $broker = config('yourService.broker'); + $avro = config('yourService.avro_schema'); + + $consumerConfiguration = ConsumerFactory::make($broker, $topic, $avro); + $consumer = app(Consumer::class, ['configOptions' => $consumerConfiguration]); + + $consumer->consume(); +``` + + + + That's it. For more information about usage, middlewares, broker authentication, consumer groups and other advanced topics, please have a look at our [Advanced Usage Guide](advanced.md). From eb023d5f1931b976bc45254c4df51afab9f763b8 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 16:48:54 -0300 Subject: [PATCH 45/72] chore: remove blank spaces --- docs/quick-usage.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/quick-usage.md b/docs/quick-usage.md index f306cba2..5178a312 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -180,9 +180,6 @@ To configure and consume using classes: $consumer->consume(); ``` - - - That's it. For more information about usage, middlewares, broker authentication, consumer groups and other advanced topics, please have a look at our [Advanced Usage Guide](advanced.md). From 708e14e1bce9b374b444f156a087291229b0a21a Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 16:56:07 -0300 Subject: [PATCH 46/72] chore: fix codacy warnings --- CHANGELOG.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 009e3b39..b282caa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added file `config/service.php` to configure broker and authentication -- Added AvroSchemaMixedEncoderTest -- Added AvroSchemaDecoderTest -- Added ProducerWithConfigOptionsTest -- Added ConfigOptionsCommand to run commands with ConfigOptions class -- Added pt_BR contributing section -- Added setup-dev script on composer -- Added grumphp commit validation +- Added file `config/service.php` to configure broker and authentication +- Added AvroSchemaMixedEncoderTest +- Added AvroSchemaDecoderTest +- Added ProducerWithConfigOptionsTest +- Added ConfigOptionsCommand to run commands with ConfigOptions class +- Added pt_BR contributing section +- Added setup-dev script on composer +- Added grumphp commit validation ### Fixed From 1b26ba4702dd546a3efb7c650a12a444bcb7ec8e Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 17:00:33 -0300 Subject: [PATCH 47/72] chore: fix codacy warnings --- CHANGELOG.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b282caa2..b237f25e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added file `config/service.php` to configure broker and authentication -- Added AvroSchemaMixedEncoderTest -- Added AvroSchemaDecoderTest -- Added ProducerWithConfigOptionsTest -- Added ConfigOptionsCommand to run commands with ConfigOptions class -- Added pt_BR contributing section -- Added setup-dev script on composer -- Added grumphp commit validation +- Added AvroSchemaMixedEncoderTest +- Added AvroSchemaDecoderTest +- Added ProducerWithConfigOptionsTest +- Added ConfigOptionsCommand to run commands with ConfigOptions class +- Added pt_BR contributing section +- Added setup-dev script on composer +- Added grumphp commit validation +- Added file `config/service.php` to configure broker and authentication ### Fixed From 84cda5b0b8255ccf81fee1acad76567cf25d85c9 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 17:39:32 -0300 Subject: [PATCH 48/72] docs: add commens explaining parameters --- config/kafka.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/kafka.php b/config/kafka.php index 9e74693b..3e7f0764 100644 --- a/config/kafka.php +++ b/config/kafka.php @@ -2,8 +2,14 @@ return [ 'topics' => [ + // This is your topic "keyword" where you will put all configurations needed + // on this specific topic. 'default' => [ + // The topic id is where you want to send or consume + // your messages from kafka. 'topic_id' => 'kafka-test', + + //your consumer configurations 'consumer' => [ 'consumer_group' => 'test-consumer-group', // Action to take when there is no initial From 4b618a58609d67de1a5c0f38c2674ca282d40a98 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 17:41:06 -0300 Subject: [PATCH 49/72] docs: document finished method --- docs/advanced.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index acca8234..14e40d1e 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -181,10 +181,17 @@ $ php artisan make:kafka-consumer PriceUpdateHandler This will create a KafkaConsumer class inside the application, on the `app/Kafka/Consumers/` directory. There, you'll have a `handler` method, which will send all records from the topic to the Consumer. -Methods will be available for handling exceptions: + +Available methods: + - `warning` method will be call whenever something not critical is received from the topic. Like a message informing that there's no more records to consume. - - `failure` method will be call whenever something critical happens, like an error to decode the record. + + + - `failure` will be call whenever something critical happens, like an error to decode the record. + + + - `finished` will be call when queue finishes ```php use App\Repository; @@ -230,6 +237,11 @@ class PriceUpdateHandler extends AbstractHandler { // handle failure exception } + + public function finished(): void + { + //handle queue end + } } ``` From 9822cf81c6f171cff6839c90204a93d260d2d2d7 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 4 Apr 2022 17:41:52 -0300 Subject: [PATCH 50/72] docs: add anchor to section --- docs/quick-usage.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/quick-usage.md b/docs/quick-usage.md index 5178a312..b2cf9a92 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -162,7 +162,8 @@ redirect_stderr=true stdout_logfile=/var/log/default/kafka-consumer-price-update.log ``` -### Using data objects + +#### Using data objects To configure and consume using classes: From 2ba105064dace665a0644b844578455adbb5da82 Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 5 Apr 2022 14:34:32 -0300 Subject: [PATCH 51/72] chore: remove blank spaces on docs --- CHANGELOG.md | 2 +- docs/advanced.md | 10 ++++------ docs/quick-usage.md | 2 -- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b237f25e..47140732 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added ConfigOptionsCommand to run commands with ConfigOptions class - Added pt_BR contributing section - Added setup-dev script on composer -- Added grumphp commit validation +- Added grumphp commit validation - Added file `config/service.php` to configure broker and authentication ### Fixed diff --git a/docs/advanced.md b/docs/advanced.md index 14e40d1e..686d80ef 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -136,8 +136,7 @@ If you wish, you may set a middleware to run of a topic level or a consumer grou ], ``` -The order matters here, they'll be execute as queue, from the most global scope to the most specific (global scope > topic scope > group_consumers scope). - +The order matters here, they'll execute as queue, from the most global scope to the most specific (global scope > topic scope > group_consumers scope). ### Schemas @@ -245,7 +244,6 @@ class PriceUpdateHandler extends AbstractHandler } ``` - #### Creating Middleware You can create a middleware class, that works between the received data from broker and before being passed into consumers, using the follow command: @@ -287,7 +285,6 @@ stdout_logfile=/var/log/default/kafka-consumer-price-update.log Although you can run this simple command, it provides some options you can pass to make it more flexible to your needs. - - `--offset=` And if you need to start the consumption of a topic in a specific offset (it can be useful for debug purposes) @@ -323,6 +320,7 @@ Although you can run this simple command, it provides some options you can pass - `--service_name=` - Specify from what file services configurations should be read. + Specify from what file services configurations should be read. + - `$ php artisan kafka:consume price-update --service_name=config.file` + $ php artisan kafka:consume price-update --service_name=config.file diff --git a/docs/quick-usage.md b/docs/quick-usage.md index b2cf9a92..88488205 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -19,7 +19,6 @@ configuration and a file to keep the broker and schema configuration. In this ex This file keeps configurations about topics, consumers and producers. It should return an array of topics containing the topic name, topic_id, consumer, producer and the settings for each one of them: - ```php Date: Tue, 5 Apr 2022 14:50:42 -0300 Subject: [PATCH 52/72] chore: fix codacy --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47140732..101df70f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added pt_BR contributing section - Added setup-dev script on composer - Added grumphp commit validation -- Added file `config/service.php` to configure broker and authentication ### Fixed From c6e3f20e1279d3f21922d8d552f0b7efbb420783 Mon Sep 17 00:00:00 2001 From: Hugo Carvalho Date: Thu, 7 Apr 2022 11:03:20 -0300 Subject: [PATCH 53/72] Update docs/quick-usage.md Apply suggested changes Co-authored-by: Diego Felix --- docs/quick-usage.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/quick-usage.md b/docs/quick-usage.md index 88488205..d6a143c8 100644 --- a/docs/quick-usage.md +++ b/docs/quick-usage.md @@ -12,7 +12,8 @@ ### Configure using files To get started using configuration files, at least two files are needed. A file to keep the topics -configuration and a file to keep the broker and schema configuration. In this example, we will use the files `config/kafka.php` and `config/service.php`. +configuration and a file to keep the broker and schema configuration. In this example, we will use the files +`config/kafka.php` and `config/service.php`. ### File `config/kafka.php`: From e51273f110226abe955fdb6e965b513bea1e10d2 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 25 Apr 2022 11:10:53 -0300 Subject: [PATCH 54/72] fix(codacy): fix codacy warnings --- CHANGELOG.md | 30 +++-- docs/quick-usage.pt.md | 154 ++++++++++++++++---------- readme.md | 1 + src/Connectors/Consumer/Config.php | 2 +- src/Console/ConfigOptionsCommand.php | 3 +- src/Middlewares/AvroSchemaDecoder.php | 6 - 6 files changed, 109 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 101df70f..342b1ef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,28 +5,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] - ### Added -- Added AvroSchemaMixedEncoderTest -- Added AvroSchemaDecoderTest -- Added ProducerWithConfigOptionsTest -- Added ConfigOptionsCommand to run commands with ConfigOptions class -- Added pt_BR contributing section -- Added setup-dev script on composer -- Added grumphp commit validation +- Added AvroSchemaMixedEncoderTest +- Added AvroSchemaDecoderTest +- Added ProducerWithConfigOptionsTest +- Added ConfigOptionsCommand to run commands with ConfigOptions class +- Added pt_BR contributing section +- Added setup-dev script on composer +- Added grumphp commit validation ### Fixed -- Fixed parameters and options override on Consumer\Config class -- Update instructions on contribute section -- Update project install section +- Fixed parameters and options override on Consumer\Config class +- Update instructions on contribute section +- Update project install section ### Changed -- Updated class from ConfigManager to ConfigOptions on unit tests -- Updated class from ConfigManager to ConfigOptions where any config request was made -- Consumer and Producer middlewares resolution - -### Removed +- Updated class from ConfigManager to ConfigOptions on unit tests +- Updated class from ConfigManager to ConfigOptions where any config request was made +- Consumer and Producer middlewares resolution diff --git a/docs/quick-usage.pt.md b/docs/quick-usage.pt.md index daf4dbfd..c3e14753 100644 --- a/docs/quick-usage.pt.md +++ b/docs/quick-usage.pt.md @@ -1,64 +1,96 @@ -## Guia rápido +## Quick Usage Guide -- [Arquivo de configuração](#config) -- [Consumer](#consumer) - - [Criando um Consumer](#creating-consumer) - - [Rodando o Consumer](#running-consumer) +- [Configurar usando arquivos](#config) +- [Configurar usando objetos](#config-dto) +- [Consumidor](#consumer) + - [Criando um consumidor](#creating-consumer) + - [Executando um consumidor](#running-consumer) +- [Produtor](#producer) + - [Produzindo mensagens](#produce-message) -### Arquivo de configuração: `config/kafka.php` - -Esse arquivo contém todas as informações sobre *brokers*, tópicos, *consumer groups* e *middlewares*. - -Para começar a usar, podemos focar em duas seções: - -- Brokers - - Uma lista de *brokers*, com configurações de conexão e autenticação. - - - `connections`: *obrigatório*. Pode ser uma `string` com múltiplas conexões separadas por vírgula ou uma `array` de conexões. - - - `auth`: *opcional*. É possivel se conectar sem autenticação ou usando autenticação SSL. - - ```php - 'brokers' => [ - 'price_brokers' => [ - 'connections' => 'localhost:8091,localhost:8092', - 'auth' => [ - 'type' => 'ssl', - 'ca' => storage_path('ca.pem'), - 'certificate' => storage_path('kafka.cert'), - 'key' => storage_path('kafka.key'), - ], - ], - 'stock_brokers' => [ - 'connections' => ['localhost:8091', 'localhost:8092'], - 'auth' => [], // pode ser uma array vazia ou até mesmo não ter essa chave aqui. - ], - ], - ``` - -- Tópicos - - Uma lista de configuração de tópicos, como nome, qual *broker* usar, *consumer group* e *middlewares*. - - Aqui você pode especificar os *consumer groups*. Cada tópico pode ter vários grupos, - e cada grupo tem a sua configuração para cada *consumer*, *offset_reset* (para definir um *offset* inicial) e *middlewares* que devem ser usados. - - ```php - 'topics' => [ - 'price_update' => [ - 'topic' => 'products.price.update', - 'broker' => 'price_brokers', - 'consumer_groups' => [ - 'default' => [ - 'offset_reset' => 'smallest', - 'handler' => '\App\Kafka\Consumers\PriceUpdateConsumer', - ], - ], - ], - ], - ``` +### Configurar usando arquivos + +Para começar a usar arquivos de configuração, são necessários pelo menos dois arquivos. Um arquivo para manter os tópicos +configuração e um arquivo para manter a configuração do broker e do esquema. Neste exemplo, usaremos os arquivos +`config/kafka.php` e `config/service.php`. + + +### Arquivo `config/kafka.php`: + +Este arquivo mantém configurações sobre tópicos, consumidores e produtores. +Deve retornar um array de tópicos contendo o nome do tópico, topic_id, consumidor, produtor e as configurações de cada um deles: + + +```php + [ + 'this_is_your_topic_name' => [ + 'topic_id' => "this_is_your_topic_id", + 'consumer' => [ + 'consumer_group' => 'your-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'timeout' => 20000, + 'auto_commit' => true, + 'commit_async' => false, + 'middlewares' => [], + ], + + 'producer' => [ + 'required_acknowledgment' => true, + 'is_async' => true, + 'max_poll_records' => 500, + 'flush_attempts' => 10, + 'middlewares' => [], + 'timeout' => 10000, + 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1, + ], + ] + ], +]; +``` + +### File `config/service.php` + +Esse arquivo possui as configurações de **broker** e **schema** utilizados. + +```php + [ + 'url' => '', + 'request_options' => [ + 'headers' => [ + 'Authorization' => [ + 'Basic ' . base64_encode( + env('AVRO_SCHEMA_USERNAME').':'.env('AVRO_SCHEMA_PASSWORD') + ), + ], + ], + ], + + 'ssl_verify' => true, + 'username' => 'USERNAME', + 'password' => 'PASSWORD', + ], + + 'broker' => [ + 'connections' => 'kafka:9092', + 'auth' => [ + 'type' => 'ssl', + 'ca' => storage_path('ca.pem'), + 'certificate' => storage_path('kafka.cert'), + 'key' => storage_path('kafka.key'), + ], + ], +]; +``` ### Consumer @@ -113,11 +145,11 @@ class PriceUpdateConsumer extends AbstractHandler Agora é só consumir o tópico. -A forma mais simples de ver tudo isso funcionando é rodando o comando `kafka:consume` com o nome do tópico que foi configurado: +Para começar a consumir o tópico, a maneira mais simples de vê-lo funcionando é executando o comando kafka:consume junto com o nome do tópico, arquivo de configuração do tópico e arquivo de configuração do serviço: ```bash -$ php artisan kafka:consume price-update -``` +$ php artisan kafka:consume this_is_your_topic_name --config_name=config.file --service_name=service.file +``` Esse comando rodará em um laço infinito (while true), isso significa que ele nunca irá parar de rodar por conta própria. Mas erros podem acontecer, então, recomendamos fortemente que você execute este comando com o auxílio de um [supervisor](http://supervisord.org/running.html), como no exemplo abaixo: diff --git a/readme.md b/readme.md index 76b81013..9206ab91 100644 --- a/readme.md +++ b/readme.md @@ -15,6 +15,7 @@ - [Installation](#installation) - [Quick Usage Guide](docs/quick-usage.md) - [Advanced Usage Guide](docs/advanced.md) +- [Upgrade Guide](docs/upgrade.md) - [Contributing](docs/CONTRIBUTING.md) - [License](#license) diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index 33673cae..fd74f86c 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -40,7 +40,7 @@ class Config extends AbstractConfig 'middlewares' => 'array', ]; - public function makeWithConfigOptions(string $handlerClass, ?int $times = null): ?Consumer + public function makeWithConfigOptions(string $handlerClass): ?Consumer { $handler = app($handlerClass); $configOptions = $handler->getConfigOptions(); diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index eb4c1071..c65bf454 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -2,7 +2,6 @@ namespace Metamorphosis\Console; use Illuminate\Console\Command as BaseCommand; -use Metamorphosis\Connectors\Consumer\Config; use Metamorphosis\Connectors\Consumer\Factory; use Metamorphosis\Consumers\Runner; use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConfigOptions; @@ -26,7 +25,7 @@ class ConfigOptionsCommand extends BaseCommand {handler : handler.} {--times= : Amount of messages to be consumed.}'; - public function handle(Config $config) + public function handle() { $consumerHandler = app($this->argument('handler')); diff --git a/src/Middlewares/AvroSchemaDecoder.php b/src/Middlewares/AvroSchemaDecoder.php index 1aeffcff..d7a39f90 100644 --- a/src/Middlewares/AvroSchemaDecoder.php +++ b/src/Middlewares/AvroSchemaDecoder.php @@ -7,18 +7,12 @@ use Metamorphosis\Avro\Serializer\MessageDecoder; use Metamorphosis\Exceptions\ConfigurationException; use Metamorphosis\Record\RecordInterface; -use Metamorphosis\TopicHandler\ConfigOptions\AvroSchema; use Metamorphosis\TopicHandler\ConfigOptions\Consumer as ConsumerConfigOptions; class AvroSchemaDecoder implements MiddlewareInterface { private MessageDecoder $decoder; - /** - * @var AvroSchema - */ - private $avroSchema; - public function __construct(ClientFactory $factory, ConsumerConfigOptions $consumerConfigOptions) { if (!$consumerConfigOptions->getAvroSchema()->getUrl()) { From 5a3fcda7a8553afc9dde8f91a5142d85731d9a5c Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 3 May 2022 16:12:16 -0300 Subject: [PATCH 55/72] docs: add upgrade guide --- docs/upgrade.md | 76 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 docs/upgrade.md diff --git a/docs/upgrade.md b/docs/upgrade.md new file mode 100644 index 00000000..5c4ef6fe --- /dev/null +++ b/docs/upgrade.md @@ -0,0 +1,76 @@ + +## Upgrade guide + +To upgrade from version X.x to version X.y: + +Move your `avroschema` and `broker` section from old `config/kafka.php` file into a new file: + + +```php + [ + 'this_is_your_topic_name' => [ + 'topic_id' => "this_is_your_topic_id", + 'consumer' => [ + 'consumer_group' => 'your-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'timeout' => 20000, + 'auto_commit' => true, + 'commit_async' => false, + 'middlewares' => [], + ], + + 'producer' => [ + 'required_acknowledgment' => true, + 'is_async' => true, + 'max_poll_records' => 500, + 'flush_attempts' => 10, + 'middlewares' => [], + 'timeout' => 10000, + 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1, + ], + ] + ], +]; +``` + +Upgrade your topic configuration files: + +```php + [ + 'this_is_your_topic_name' => [ + 'topic_id' => "this_is_your_topic_id", + 'consumer' => [ + 'consumer_group' => 'your-consumer-group', + 'offset_reset' => 'earliest', + 'offset' => 0, + 'partition' => 0, + 'handler' => '\App\Kafka\Consumers\ConsumerExample', + 'timeout' => 20000, + 'auto_commit' => true, + 'commit_async' => false, + 'middlewares' => [], + ], + + 'producer' => [ + 'required_acknowledgment' => true, + 'is_async' => true, + 'max_poll_records' => 500, + 'flush_attempts' => 10, + 'middlewares' => [], + 'timeout' => 10000, + 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1, + ], + ] + ], +]; +``` + From 066df6da91bb5f35bb49854064be94ea525fc146 Mon Sep 17 00:00:00 2001 From: hcdias Date: Wed, 5 Apr 2023 18:48:45 -0300 Subject: [PATCH 56/72] fix: add handleMiddlewares method --- src/Connectors/Consumer/Config.php | 8 ++++++++ tests/Unit/AbstractConfigManagerTest.php | 0 2 files changed, 8 insertions(+) delete mode 100644 tests/Unit/AbstractConfigManagerTest.php diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index fd74f86c..f868a2cb 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -94,4 +94,12 @@ private function getTopicConfig(string $configName, string $topicId): array return $topicConfig; } + + private function getMiddlewares(string $configName, array $topicConfig): array + { + return array_merge( + config($configName . '.middlewares.consumer', []), + $topicConfig['consumer']['middlewares'] ?? [] + ); + } } diff --git a/tests/Unit/AbstractConfigManagerTest.php b/tests/Unit/AbstractConfigManagerTest.php deleted file mode 100644 index e69de29b..00000000 From 402a59da91e71df409b5859e60c9bd4743976688 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 14:01:28 -0300 Subject: [PATCH 57/72] style: fix coding standard violations Signed-off-by: hcdias --- config/service.php | 6 ++- src/Authentication/Factory.php | 10 ++++- src/Authentication/SASLAuthentication.php | 10 ++--- src/Authentication/SSLAuthentication.php | 10 ++--- src/Connectors/AbstractConfig.php | 9 +++-- src/Connectors/Consumer/Config.php | 6 ++- src/Connectors/Consumer/Factory.php | 39 +++++++++++++------ src/Connectors/Consumer/LowLevel.php | 15 +++++-- src/Connectors/Producer/Connector.php | 6 ++- src/Console/ConfigOptionsCommand.php | 7 ++-- src/Console/ConsumerCommand.php | 6 +-- src/MetamorphosisServiceProvider.php | 8 ++-- src/Middlewares/AvroSchemaDecoder.php | 8 +++- src/Middlewares/AvroSchemaMixedEncoder.php | 25 +++++++----- src/Producer.php | 28 ++++++++++--- .../Factories/ConsumerFactory.php | 5 ++- .../Producer/AbstractProducer.php | 2 +- 17 files changed, 137 insertions(+), 63 deletions(-) diff --git a/config/service.php b/config/service.php index 82cb1299..64f81077 100644 --- a/config/service.php +++ b/config/service.php @@ -6,8 +6,10 @@ 'request_options' => [ 'headers' => [ 'Authorization' => [ - 'Basic '.base64_encode( - env('AVRO_SCHEMA_USERNAME').':'.env('AVRO_SCHEMA_PASSWORD') + 'Basic' . base64_encode( + env('AVRO_SCHEMA_USERNAME') . ':' . env( + 'AVRO_SCHEMA_PASSWORD' + ) ), ], ], diff --git a/src/Authentication/Factory.php b/src/Authentication/Factory.php index c8a8f38a..55fb7053 100644 --- a/src/Authentication/Factory.php +++ b/src/Authentication/Factory.php @@ -24,11 +24,17 @@ public static function authenticate(Conf $conf, AuthInterface $configOptions): v break; case self::TYPE_SSL: - app(SSLAuthentication::class, compact('conf', 'configOptions')); + app( + SSLAuthentication::class, + compact('conf', 'configOptions') + ); break; case self::TYPE_SASL_SSL: - app(SASLAuthentication::class, compact('conf', 'configOptions')); + app( + SASLAuthentication::class, + compact('conf', 'configOptions') + ); break; default: diff --git a/src/Authentication/SASLAuthentication.php b/src/Authentication/SASLAuthentication.php index 3918558d..f4a0e3b0 100644 --- a/src/Authentication/SASLAuthentication.php +++ b/src/Authentication/SASLAuthentication.php @@ -9,10 +9,7 @@ class SASLAuthentication implements AuthenticationInterface { private Conf $conf; - /** - * @var SaslSsl - */ - private $configOptions; + private SaslSsl $configOptions; public function __construct(Conf $conf, SaslSsl $configOptions) { @@ -29,7 +26,10 @@ private function authenticate(): void // The mechanisms key is optional when configuring this kind of authentication // If the user does not specify the mechanism, the default will be 'PLAIN'. // But, to make config more clear, we are asking the user every time. - $this->conf->set('sasl.mechanisms', $this->configOptions->getMechanisms()); + $this->conf->set( + 'sasl.mechanisms', + $this->configOptions->getMechanisms() + ); $this->conf->set('sasl.username', $this->configOptions->getUsername()); $this->conf->set('sasl.password', $this->configOptions->getPassword()); } diff --git a/src/Authentication/SSLAuthentication.php b/src/Authentication/SSLAuthentication.php index 5dfc06e4..e698dfe5 100644 --- a/src/Authentication/SSLAuthentication.php +++ b/src/Authentication/SSLAuthentication.php @@ -9,10 +9,7 @@ class SSLAuthentication implements AuthenticationInterface { private Conf $conf; - /** - * @var Ssl - */ - private $configOptions; + private Ssl $configOptions; public function __construct(Conf $conf, Ssl $configOptions) { @@ -26,7 +23,10 @@ private function authenticate(): void { $this->conf->set('security.protocol', $this->configOptions->getType()); $this->conf->set('ssl.ca.location', $this->configOptions->getCa()); - $this->conf->set('ssl.certificate.location', $this->configOptions->getCertificate()); + $this->conf->set( + 'ssl.certificate.location', + $this->configOptions->getCertificate() + ); $this->conf->set('ssl.key.location', $this->configOptions->getKey()); } } diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index fa8b8ac1..b71c683c 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -9,15 +9,18 @@ abstract class AbstractConfig { protected function getBrokerConfig(string $servicesFile): array { - if (!$brokerConfig = config($servicesFile.'.broker')) { - throw new ConfigurationException("Broker configuration not found on '{$servicesFile}'"); + if (!$brokerConfig = config($servicesFile . '.broker')) { + throw new ConfigurationException( + "Broker configuration not found on '{$servicesFile}'" + ); } + return $brokerConfig; } protected function getSchemaConfig(string $servicesFile): array { - return config($servicesFile.'.avro_schema', []); + return config($servicesFile . '.avro_schema', []); } protected function validate(array $config): void diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index f868a2cb..d13976be 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -74,7 +74,11 @@ public function make(array $options, array $arguments): Consumer } } - return ConsumerFactory::make($brokerConfig, $topicConfig, $schemaConfig); + return ConsumerFactory::make( + $brokerConfig, + $topicConfig, + $schemaConfig + ); } /** diff --git a/src/Connectors/Consumer/Factory.php b/src/Connectors/Consumer/Factory.php index c82c4ad2..923fb817 100644 --- a/src/Connectors/Consumer/Factory.php +++ b/src/Connectors/Consumer/Factory.php @@ -26,32 +26,49 @@ public static function make(ConsumerConfigOptions $configOptions): Manager $middlewares = $configOptions->getMiddlewares(); foreach ($middlewares as &$middleware) { - $middleware = is_string($middleware) ? app($middleware, ['consumerConfigOptions' => $configOptions]) : $middleware; + $middleware = is_string($middleware) + ? app( + $middleware, + ['consumerConfigOptions' => $configOptions] + ) + : $middleware; } - $middlewares[] = app(ConsumerMiddleware::class, ['consumerTopicHandler' => $handler]); + $middlewares[] = app( + ConsumerMiddleware::class, + ['consumerTopicHandler' => $handler] + ); $dispatcher = self::getMiddlewareDispatcher($middlewares); - return new Manager($consumer, $handler, $dispatcher, $autoCommit, $commitAsync); - } - - protected static function requiresPartition(ConsumerConfigOptions $configOptions): bool - { - $partition = $configOptions->getPartition(); - - return !is_null($partition) && $partition >= 0; + return new Manager( + $consumer, + $handler, + $dispatcher, + $autoCommit, + $commitAsync + ); } public static function getConsumer(bool $autoCommit, ConsumerConfigOptions $configOptions): ConsumerInterface { if (self::requiresPartition($configOptions)) { - return app(LowLevel::class)->getConsumer($autoCommit, $configOptions); + return app(LowLevel::class)->getConsumer( + $autoCommit, + $configOptions + ); } return app(HighLevel::class)->getConsumer($autoCommit, $configOptions); } + protected static function requiresPartition(ConsumerConfigOptions $configOptions): bool + { + $partition = $configOptions->getPartition(); + + return !is_null($partition) && $partition >= 0; + } + private static function getMiddlewareDispatcher(array $middlewares): Dispatcher { return new Dispatcher($middlewares); diff --git a/src/Connectors/Consumer/LowLevel.php b/src/Connectors/Consumer/LowLevel.php index e897d467..33d0c1de 100644 --- a/src/Connectors/Consumer/LowLevel.php +++ b/src/Connectors/Consumer/LowLevel.php @@ -27,9 +27,15 @@ public function getConsumer(bool $autoCommit, ConfigOptions $configOptions): Con $consumer->addBrokers($broker->getConnections()); $topicConf = $this->getTopicConfigs($configOptions); - $topicConsumer = $consumer->newTopic($configOptions->getTopicId(), $topicConf); + $topicConsumer = $consumer->newTopic( + $configOptions->getTopicId(), + $topicConf + ); - $topicConsumer->consumeStart($configOptions->getPartition(), $configOptions->getOffset()); + $topicConsumer->consumeStart( + $configOptions->getPartition(), + $configOptions->getOffset() + ); return new LowLevelConsumer($topicConsumer, $configOptions); } @@ -41,7 +47,10 @@ protected function getTopicConfigs(ConfigOptions $configOptions) // Set where to start consuming messages when there is no initial offset in // offset store or the desired offset is out of range. // 'smallest': start from the beginning - $topicConfig->set('auto.offset.reset', $configOptions->getOffsetReset()); + $topicConfig->set( + 'auto.offset.reset', + $configOptions->getOffsetReset() + ); return $topicConfig; } diff --git a/src/Connectors/Producer/Connector.php b/src/Connectors/Producer/Connector.php index 1109a598..3d427afc 100644 --- a/src/Connectors/Producer/Connector.php +++ b/src/Connectors/Producer/Connector.php @@ -12,8 +12,10 @@ class Connector { - public function getProducerTopic(HandlerInterface $handler, ProducerConfigOptions $producerConfigOptions): KafkaProducer - { + public function getProducerTopic( + HandlerInterface $handler, + ProducerConfigOptions $producerConfigOptions + ): KafkaProducer { $conf = resolve(Conf::class); if ($this->canHandleResponse($handler)) { diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index c65bf454..d592a8b1 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -1,4 +1,5 @@ getTopicId().PHP_EOL; - $text .= ' on consumer group: '.$configOptions->getConsumerGroup().PHP_EOL; - $text .= 'Connecting in '.$configOptions->getBroker()->getConnections().PHP_EOL; + $text = 'Starting consumer for topic: ' . $configOptions->getTopicId() . PHP_EOL; + $text .= ' on consumer group: ' . $configOptions->getConsumerGroup() . PHP_EOL; + $text .= 'Connecting in ' . $configOptions->getBroker()->getConnections() . PHP_EOL; $text .= 'Running consumer..'; $this->output->writeln($text); diff --git a/src/Console/ConsumerCommand.php b/src/Console/ConsumerCommand.php index 93420418..ded31d7f 100644 --- a/src/Console/ConsumerCommand.php +++ b/src/Console/ConsumerCommand.php @@ -51,9 +51,9 @@ public function handle(Config $config) private function writeStartingConsumer(Consumer $consumer) { - $text = 'Starting consumer for topic: '.$consumer->getTopicId().PHP_EOL; - $text .= ' on consumer group: '.$consumer->getConsumerGroup().PHP_EOL; - $text .= 'Connecting in '.$consumer->getBroker()->getConnections().PHP_EOL; + $text = 'Starting consumer for topic: ' . $consumer->getTopicId() . PHP_EOL; + $text .= ' on consumer group: ' . $consumer->getConsumerGroup() . PHP_EOL; + $text .= 'Connecting in ' . $consumer->getBroker()->getConnections() . PHP_EOL; $text .= 'Running consumer..'; $this->output->writeln($text); diff --git a/src/MetamorphosisServiceProvider.php b/src/MetamorphosisServiceProvider.php index 36dffdf5..6a64d8c4 100644 --- a/src/MetamorphosisServiceProvider.php +++ b/src/MetamorphosisServiceProvider.php @@ -14,12 +14,12 @@ class MetamorphosisServiceProvider extends ServiceProvider public function boot() { $this->publishes([ - __DIR__.'/../config/kafka.php' => config_path('kafka.php'), - __DIR__.'/../config/service.php' => config_path('service.php'), + __DIR__ . '/../config/kafka.php' => config_path('kafka.php'), + __DIR__ . '/../config/service.php' => config_path('service.php'), ], 'config'); - $this->mergeConfigFrom(__DIR__.'/../config/kafka.php', 'kafka'); - $this->mergeConfigFrom(__DIR__.'/../config/service.php', 'service'); + $this->mergeConfigFrom(__DIR__ . '/../config/kafka.php', 'kafka'); + $this->mergeConfigFrom(__DIR__ . '/../config/service.php', 'service'); } public function register() diff --git a/src/Middlewares/AvroSchemaDecoder.php b/src/Middlewares/AvroSchemaDecoder.php index d7a39f90..07f2193d 100644 --- a/src/Middlewares/AvroSchemaDecoder.php +++ b/src/Middlewares/AvroSchemaDecoder.php @@ -16,10 +16,14 @@ class AvroSchemaDecoder implements MiddlewareInterface public function __construct(ClientFactory $factory, ConsumerConfigOptions $consumerConfigOptions) { if (!$consumerConfigOptions->getAvroSchema()->getUrl()) { - throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaDecoder Middleware"); + throw new ConfigurationException( + "Avro schema url not found, it's required to use AvroSchemaDecoder Middleware" + ); } - $this->decoder = new MessageDecoder($factory->make($consumerConfigOptions->getAvroSchema())); + $this->decoder = new MessageDecoder( + $factory->make($consumerConfigOptions->getAvroSchema()) + ); } public function process(RecordInterface $record, Closure $next) diff --git a/src/Middlewares/AvroSchemaMixedEncoder.php b/src/Middlewares/AvroSchemaMixedEncoder.php index 6969a80e..67c09076 100644 --- a/src/Middlewares/AvroSchemaMixedEncoder.php +++ b/src/Middlewares/AvroSchemaMixedEncoder.php @@ -21,18 +21,22 @@ class AvroSchemaMixedEncoder implements MiddlewareInterface private CachedSchemaRegistryClient $schemaRegistry; - /** - * @var ProducerConfigOptions - */ - private $producerConfigOptions; + private ProducerConfigOptions $producerConfigOptions; - public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, ProducerConfigOptions $producerConfigOptions) - { + public function __construct( + SchemaId $schemaIdEncoder, + ClientFactory $factory, + ProducerConfigOptions $producerConfigOptions + ) { if (!$producerConfigOptions->getAvroSchema()->getUrl()) { - throw new ConfigurationException("Avro schema url not found, it's required to use AvroSchemaEncoder Middleware"); + throw new ConfigurationException( + "Avro schema url not found, it's required to use AvroSchemaEncoder Middleware" + ); } - $schemaRegistry = $factory->make($producerConfigOptions->getAvroSchema()); + $schemaRegistry = $factory->make( + $producerConfigOptions->getAvroSchema() + ); $this->schemaIdEncoder = $schemaIdEncoder; $this->schemaRegistry = $schemaRegistry; $this->producerConfigOptions = $producerConfigOptions; @@ -41,7 +45,10 @@ public function __construct(SchemaId $schemaIdEncoder, ClientFactory $factory, P public function process(RecordInterface $record, Closure $next) { $topic = $this->producerConfigOptions->getTopicId(); - $schema = $this->schemaRegistry->getBySubjectAndVersion("{$topic}-value", 'latest'); + $schema = $this->schemaRegistry->getBySubjectAndVersion( + "{$topic}-value", + 'latest' + ); $arrayPayload = json_decode($record->getPayload(), true); $encodedPayload = $this->schemaIdEncoder->encode( $schema, diff --git a/src/Producer.php b/src/Producer.php index 4fa9ee5b..b3211f7c 100644 --- a/src/Producer.php +++ b/src/Producer.php @@ -36,20 +36,36 @@ public function build(HandlerInterface $producerHandler): Dispatcher $middlewares = $producerConfigOptions->getMiddlewares(); foreach ($middlewares as &$middleware) { - $middleware = is_string($middleware) ? app($middleware, ['producerConfigOptions' => $producerConfigOptions]) : $middleware; + $middleware = is_string($middleware) + ? app( + $middleware, + ['producerConfigOptions' => $producerConfigOptions] + ) + : $middleware; } - $middlewares[] = $this->getProducerMiddleware($producerHandler, $producerConfigOptions); + $middlewares[] = $this->getProducerMiddleware( + $producerHandler, + $producerConfigOptions + ); return new Dispatcher($middlewares); } - public function getProducerMiddleware(HandlerInterface $producerHandler, ProducerConfigOptions $producerConfigOptions): ProducerMiddleware - { - $producer = $this->connector->getProducerTopic($producerHandler, $producerConfigOptions); + public function getProducerMiddleware( + HandlerInterface $producerHandler, + ProducerConfigOptions $producerConfigOptions + ): ProducerMiddleware { + $producer = $this->connector->getProducerTopic( + $producerHandler, + $producerConfigOptions + ); $topic = $producer->newTopic($producerConfigOptions->getTopicId()); - $poll = app(Poll::class, ['producer' => $producer, 'producerConfigOptions' => $producerConfigOptions]); + $poll = app( + Poll::class, + ['producer' => $producer, 'producerConfigOptions' => $producerConfigOptions] + ); $partition = $producerConfigOptions->getPartition(); return app( diff --git a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php index 4dd62736..2d28a28c 100644 --- a/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php +++ b/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php @@ -25,7 +25,10 @@ private static function getConsumerGroupConfig(array $topicData): array $consumer = $topicData['consumer']; $topicData['consumerGroup'] = $consumer['consumer_group']; - return array_merge_recursive($topicData, self::convertConfigAttributes($consumer)); + return array_merge_recursive( + $topicData, + self::convertConfigAttributes($consumer) + ); } private static function convertConfigAttributes(array $consumerConfig): array diff --git a/src/TopicHandler/Producer/AbstractProducer.php b/src/TopicHandler/Producer/AbstractProducer.php index 694aedd0..ecab7c93 100644 --- a/src/TopicHandler/Producer/AbstractProducer.php +++ b/src/TopicHandler/Producer/AbstractProducer.php @@ -20,7 +20,7 @@ class AbstractProducer implements HandlerInterface */ protected $producer; - public function __construct($record, Producer $configOptions, string $key = null) + public function __construct($record, Producer $configOptions, ?string $key = null) { $this->record = $record; $this->key = $key; From d886eb4af17c2c30a1e21babcd128099b9ac715f Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 14:15:25 -0300 Subject: [PATCH 58/72] style: fix coding standard violations Signed-off-by: hcdias --- tests/Integration/ConsumerTest.php | 5 +- tests/Integration/ProducerTest.php | 38 +++++---- tests/Integration/ProducerWithAvroTest.php | 12 ++- tests/Unit/Authentication/FactoryTest.php | 6 +- .../Authentication/SASLAuthenticationTest.php | 6 +- .../Authentication/SSLAuthenticationTest.php | 6 +- tests/Unit/Connectors/Consumer/ConfigTest.php | 4 +- .../Unit/Connectors/Consumer/FactoryTest.php | 83 ++++++++++--------- .../Connectors/Producer/ConnectorTest.php | 14 +++- tests/Unit/Console/ConsumerCommandTest.php | 54 ++++++------ tests/Unit/Consumers/LowLevelTest.php | 5 +- .../Middlewares/AvroSchemaDecoderTest.php | 10 ++- .../AvroSchemaMixedEncoderTest.php | 32 +++++-- tests/Unit/ProducerTest.php | 10 +-- 14 files changed, 177 insertions(+), 108 deletions(-) diff --git a/tests/Integration/ConsumerTest.php b/tests/Integration/ConsumerTest.php index 1d4397e8..2fb7b1bf 100644 --- a/tests/Integration/ConsumerTest.php +++ b/tests/Integration/ConsumerTest.php @@ -59,7 +59,10 @@ public function testItShouldSetup(): void $saleOrderDispatcher = Metamorphosis::build($messageProducer); $saleOrderDispatcher->handle($messageProducer->createRecord()); - $consumer = $this->app->make(Consumer::class, ['configOptions' => $consumerConfigOptions]); + $consumer = $this->app->make( + Consumer::class, + ['configOptions' => $consumerConfigOptions] + ); $expected = '{"id":"MESSAGE_ID"}'; // Actions diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 94ab5626..8fbe5305 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -18,22 +18,9 @@ class ProducerTest extends LaravelTestCase protected string $firstLowLevelMessage; - /** - * @var string - */ - protected $secondLowLevelMessage; + protected string $secondLowLevelMessage; - /** - * @var string - */ - protected $topicId; - - protected function setUp(): void - { - parent::setUp(); - $this->withoutAuthentication(); - $this->topicId = 'kafka-test-'.Str::random(5); - } + protected string $topicId; /** * @group runProducer @@ -66,6 +53,14 @@ public function testShouldRunAProducerAndReceiveMessagesWithALowLevelConsumer(): $this->runTheLowLevelConsumerSkippingTheFirstTwoMessagesAndLimitingToTwoMessagesConsumed(); } + protected function setUp(): void + { + parent::setUp(); + + $this->withoutAuthentication(); + $this->topicId = 'kafka-test-' . Str::random(5); + } + protected function withoutAuthentication(): void { config(['service.broker.auth' => []]); @@ -73,7 +68,9 @@ protected function withoutAuthentication(): void protected function haveAConsumerHandlerConfigured(): void { - config(['kafka.topics.default.consumer.handler' => MessageConsumer::class]); + config( + ['kafka.topics.default.consumer.handler' => MessageConsumer::class] + ); } protected function haveAConsumerPartitionConfigured(): void @@ -145,7 +142,9 @@ private function haveSomeRandomMessagesProduced(): void { $this->highLevelMessage = Str::random(10); - $producerConfigOptions = $this->createProducerConfigOptions($this->topicId); + $producerConfigOptions = $this->createProducerConfigOptions( + $this->topicId + ); $producer = app(MessageProducer::class, [ 'record' => $this->highLevelMessage, 'configOptions' => $producerConfigOptions, @@ -158,7 +157,9 @@ private function haveSomeRandomMessagesProduced(): void private function produceRecordMessage(string $record): string { - $producerConfigOptions = $this->createProducerConfigOptions('low_level'); + $producerConfigOptions = $this->createProducerConfigOptions( + 'low_level' + ); $producer = app(MessageProducer::class, [ 'record' => $record, 'configOptions' => $producerConfigOptions, @@ -206,6 +207,7 @@ private function haveFourProducedMessages(): void private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { $broker = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( $topicId, $broker, diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index b0c801ad..54793d98 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -22,11 +22,12 @@ class ProducerWithAvroTest extends LaravelTestCase /** * @var string[] */ - protected $records; + protected array $records; public function setUp(): void { parent::setUp(); + $this->records = ['saleOrderId' => 'SALE_ORDER_ID', 'productId' => 'PRODUCT_ID']; } @@ -110,8 +111,12 @@ protected function haveAHandlerConfigured(): void private function haveSomeRandomMessagesProduced(): void { - $producerConfigOptionsSale = $this->createProducerConfigOptions('sale_order'); - $producerConfigOptionsProduct = $this->createProducerConfigOptions('product'); + $producerConfigOptionsSale = $this->createProducerConfigOptions( + 'sale_order' + ); + $producerConfigOptionsProduct = $this->createProducerConfigOptions( + 'product' + ); $saleOrderProducer = app(MessageProducer::class, [ 'record' => ['saleOrderId' => 'SALE_ORDER_ID'], @@ -166,6 +171,7 @@ private function setLogExpectationsFor(string $message): void private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { $broker = new Broker('kafka:9092', new None()); + return new ProducerConfigOptions( $topicId, $broker, diff --git a/tests/Unit/Authentication/FactoryTest.php b/tests/Unit/Authentication/FactoryTest.php index 717abe16..f13b2e46 100644 --- a/tests/Unit/Authentication/FactoryTest.php +++ b/tests/Unit/Authentication/FactoryTest.php @@ -16,7 +16,11 @@ class FactoryTest extends LaravelTestCase public function testItMakesSslAuthenticationClass(): void { // Set - $configOptionsSsl = new Ssl('path/to/ca', 'path/to/certificate', 'path/to/key'); + $configOptionsSsl = new Ssl( + 'path/to/ca', + 'path/to/certificate', + 'path/to/key' + ); $conf = new Conf(); $expected = [ 'security.protocol' => 'ssl', diff --git a/tests/Unit/Authentication/SASLAuthenticationTest.php b/tests/Unit/Authentication/SASLAuthenticationTest.php index f9e9a474..45a4dc3a 100644 --- a/tests/Unit/Authentication/SASLAuthenticationTest.php +++ b/tests/Unit/Authentication/SASLAuthenticationTest.php @@ -12,7 +12,11 @@ class SASLAuthenticationTest extends LaravelTestCase public function testItShouldValidateAuthenticationConfigurations(): void { // Set - $configSaslSsl = new SaslSsl('PLAIN', 'some-username', 'some-password'); + $configSaslSsl = new SaslSsl( + 'PLAIN', + 'some-username', + 'some-password' + ); $conf = new Conf(); $expected = [ diff --git a/tests/Unit/Authentication/SSLAuthenticationTest.php b/tests/Unit/Authentication/SSLAuthenticationTest.php index 35a9168e..9ea6c6b0 100644 --- a/tests/Unit/Authentication/SSLAuthenticationTest.php +++ b/tests/Unit/Authentication/SSLAuthenticationTest.php @@ -13,7 +13,11 @@ public function testItShouldValidateAuthenticationConfigurations(): void { // Set $conf = new Conf(); - $configSsl = new Ssl('path/to/ca', 'path/to/certificate', 'path/to/key'); + $configSsl = new Ssl( + 'path/to/ca', + 'path/to/certificate', + 'path/to/key' + ); $expected = [ 'security.protocol' => 'ssl', 'ssl.ca.location' => 'path/to/ca', diff --git a/tests/Unit/Connectors/Consumer/ConfigTest.php b/tests/Unit/Connectors/Consumer/ConfigTest.php index ffeaf5cd..224f5666 100644 --- a/tests/Unit/Connectors/Consumer/ConfigTest.php +++ b/tests/Unit/Connectors/Consumer/ConfigTest.php @@ -85,7 +85,9 @@ public function testShouldValidateConsumerConfig(): void ]); // Actions - $configManager = $config->makeWithConfigOptions(ConsumerHandlerDummy::class); + $configManager = $config->makeWithConfigOptions( + ConsumerHandlerDummy::class + ); // Assertions $this->assertArraySubset($expected, $configManager->toArray()); diff --git a/tests/Unit/Connectors/Consumer/FactoryTest.php b/tests/Unit/Connectors/Consumer/FactoryTest.php index 2ccb9cd9..fcd1a6d3 100644 --- a/tests/Unit/Connectors/Consumer/FactoryTest.php +++ b/tests/Unit/Connectors/Consumer/FactoryTest.php @@ -11,6 +11,52 @@ class FactoryTest extends LaravelTestCase { + public function testItMakesManagerWithLowLevelConsumer(): void + { + // Set + $this->haveAConsumerWithPartitionConfigured(); + + $config = new Config(); + $configConsumer = $config->make( + ['timeout' => 61], + ['topic' => 'topic_key', 'consumer_group' => 'with-partition'] + ); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); + } + + public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid(): void + { + // Set + $this->haveAConsumerWithoutPartitionConfigured(); + $config = new Config(); + $configConsumer = $config->make( + ['timeout' => 61, 'partition' => -1], + ['topic' => 'topic_key', 'consumer_group' => 'with-partition'] + ); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); + } + + public function testItMakesHighLevelClass(): void + { + // Set + $this->haveAConsumerWithoutPartitionConfigured(); + $config = new Config(); + $configConsumer = $config->make( + ['timeout' => 61], + ['topic' => 'topic_key', 'consumer_group' => 'without-partition'] + ); + $manager = Factory::make($configConsumer); + + // Assertions + $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); + } + protected function setUp(): void { parent::setUp(); @@ -47,43 +93,6 @@ protected function setUp(): void ]); } - public function testItMakesManagerWithLowLevelConsumer(): void - { - // Set - $this->haveAConsumerWithPartitionConfigured(); - - $config = new Config(); - $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); - $manager = Factory::make($configConsumer); - - // Assertions - $this->assertInstanceOf(LowLevel::class, $manager->getConsumer()); - } - - public function testItMakesManagerWithHighLevelConsumerWhenPartitionIsNotValid(): void - { - // Set - $this->haveAConsumerWithoutPartitionConfigured(); - $config = new Config(); - $configConsumer = $config->make(['timeout' => 61, 'partition' => -1], ['topic' => 'topic_key', 'consumer_group' => 'with-partition']); - $manager = Factory::make($configConsumer); - - // Assertions - $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); - } - - public function testItMakesHighLevelClass(): void - { - // Set - $this->haveAConsumerWithoutPartitionConfigured(); - $config = new Config(); - $configConsumer = $config->make(['timeout' => 61], ['topic' => 'topic_key', 'consumer_group' => 'without-partition']); - $manager = Factory::make($configConsumer); - - // Assertions - $this->assertInstanceOf(HighLevel::class, $manager->getConsumer()); - } - private function haveAConsumerWithPartitionConfigured() { config([ diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index f456dfb7..2869318b 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -33,7 +33,7 @@ public function testItShouldMakeSetup(): void $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); - $handler = new class('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface { + $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface { public function success(Message $message): void { } @@ -57,7 +57,10 @@ public function failed(Message $message): void ->andReturn($broker); // Actions - $result = $connector->getProducerTopic($handler, $producerConfigOptions); + $result = $connector->getProducerTopic( + $handler, + $producerConfigOptions + ); // Assertions $this->assertInstanceOf(KafkaProducer::class, $result); @@ -79,7 +82,7 @@ public function testItShouldMakeSetupWithoutHandleResponse(): void $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); - $handler = new class('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface { + $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface { public function success(Message $message): void { } @@ -102,7 +105,10 @@ public function failed(Message $message): void ->andReturn($broker); // Actions - $result = $connector->getProducerTopic($handler, $producerConfigOptions); + $result = $connector->getProducerTopic( + $handler, + $producerConfigOptions + ); // Assertions $this->assertInstanceOf(KafkaProducer::class, $result); diff --git a/tests/Unit/Console/ConsumerCommandTest.php b/tests/Unit/Console/ConsumerCommandTest.php index 5a5a8fcf..c4737533 100644 --- a/tests/Unit/Console/ConsumerCommandTest.php +++ b/tests/Unit/Console/ConsumerCommandTest.php @@ -10,33 +10,6 @@ class ConsumerCommandTest extends LaravelTestCase { - protected function setUp(): void - { - parent::setUp(); - - config([ - 'kafka' => [ - 'topics' => [ - 'topic_key' => [ - 'topic_id' => 'topic_name', - 'consumer' => [ - 'consumer_group' => 'default', - 'offset_reset' => 'earliest', - 'handler' => ConsumerHandlerDummy::class, - 'timeout' => 123, - ], - ], - ], - ], - 'service' => [ - 'broker' => [ - 'connections' => 'test_kafka:6680', - 'auth' => [], - ], - ], - ]); - } - public function testItCallsCommandWithInvalidTopic(): void { // Set @@ -158,4 +131,31 @@ public function testItOverridesBrokerConnectionWhenCallingCommand(): void $this->artisan($command, $parameters); } + + protected function setUp(): void + { + parent::setUp(); + + config([ + 'kafka' => [ + 'topics' => [ + 'topic_key' => [ + 'topic_id' => 'topic_name', + 'consumer' => [ + 'consumer_group' => 'default', + 'offset_reset' => 'earliest', + 'handler' => ConsumerHandlerDummy::class, + 'timeout' => 123, + ], + ], + ], + ], + 'service' => [ + 'broker' => [ + 'connections' => 'test_kafka:6680', + 'auth' => [], + ], + ], + ]); + } } diff --git a/tests/Unit/Consumers/LowLevelTest.php b/tests/Unit/Consumers/LowLevelTest.php index b39ac2aa..e2347b60 100644 --- a/tests/Unit/Consumers/LowLevelTest.php +++ b/tests/Unit/Consumers/LowLevelTest.php @@ -36,7 +36,10 @@ public function testItShouldConsume(): void $consumerTopic = m::mock(ConsumerTopic::class); $message = new Message(); - $lowLevelConsumer = new LowLevel($consumerTopic, $consumerConfigOptions); + $lowLevelConsumer = new LowLevel( + $consumerTopic, + $consumerConfigOptions + ); // Expectations $consumerTopic->expects() diff --git a/tests/Unit/Middlewares/AvroSchemaDecoderTest.php b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php index 6c8d31b0..ba9edd29 100644 --- a/tests/Unit/Middlewares/AvroSchemaDecoderTest.php +++ b/tests/Unit/Middlewares/AvroSchemaDecoderTest.php @@ -1,4 +1,5 @@ getAvroSchema() ->andReturn($avroSchema); - $avroSchemaDecoder = new AvroSchemaDecoder($clientFactory, $consumerConfigOptions); + $avroSchemaDecoder = new AvroSchemaDecoder( + $clientFactory, + $consumerConfigOptions + ); $result = $avroSchemaDecoder->process($consumerRecord, $closure); diff --git a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php index fcff2d6c..7fc33c0f 100644 --- a/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php +++ b/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php @@ -1,4 +1,5 @@ getAvroSchema(); $clientFactory = m::mock(ClientFactory::class); - $cachedSchemaRegistryClient = m::mock(CachedSchemaRegistryClient::class); - $schemaIdEncoder = m::mock(SchemaId::class, [$cachedSchemaRegistryClient]); + $cachedSchemaRegistryClient = m::mock( + CachedSchemaRegistryClient::class + ); + $schemaIdEncoder = m::mock( + SchemaId::class, + [$cachedSchemaRegistryClient] + ); $schema = new Schema(); - $parsedSchema = $schema->parse($avroSchema, '123', 'kafka-test-value', 'latest'); + $parsedSchema = $schema->parse( + $avroSchema, + '123', + 'kafka-test-value', + 'latest' + ); $record = $this->getRecord($parsedSchema->getAvroSchema()); $producerRecord = new ProducerRecord($record, 'kafka-test'); @@ -62,7 +76,11 @@ public function testShouldEncodeRecord() ->andReturn($encodedMessage); // Actions - $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder($schemaIdEncoder, $clientFactory, $producerConfigOptions); + $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder( + $schemaIdEncoder, + $clientFactory, + $producerConfigOptions + ); $result = $avroSchemaMixedEncoder->process($producerRecord, $closure); // Assertions @@ -93,6 +111,8 @@ private function getRecord(AvroSchema $avroSchema): string private function getSchemaFixture(): string { - return file_get_contents(__DIR__.'/../fixtures/schemas/sales_price.avsc'); + return file_get_contents( + __DIR__ . '/../fixtures/schemas/sales_price.avsc' + ); } } diff --git a/tests/Unit/ProducerTest.php b/tests/Unit/ProducerTest.php index 02808390..61f43155 100644 --- a/tests/Unit/ProducerTest.php +++ b/tests/Unit/ProducerTest.php @@ -41,7 +41,7 @@ public function testItShouldProduceRecordAsArrayThroughMiddlewareQueue(): void $topic, $broker ); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations @@ -85,7 +85,7 @@ public function testItShouldProduceRecordAsStringThroughMiddlewareQueue(): void $kafkaProducer = m::mock(KafkaProducer::class); $producerTopic = m::mock(ProducerTopic::class); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations @@ -134,7 +134,7 @@ public function testItShouldThrowJsonExceptionWhenPassingMalFormattedArray(): vo 500, 1 ); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations @@ -187,7 +187,7 @@ public function testShouldBuildDispatcher(): void 1 ); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations @@ -236,7 +236,7 @@ public function testShouldBuildDispatcherWithConfigOptions(): void 500, 1 ); - $producerHandler = new class($record, $producerConfigOptions, $topic) extends AbstractProducer { + $producerHandler = new class ($record, $producerConfigOptions, $topic) extends AbstractProducer { }; // Expectations From f6b6af4781d77c570a13387247bea960307b2632 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 14:21:32 -0300 Subject: [PATCH 59/72] style: supress phpcs typehing warning Signed-off-by: hcdias --- src/Console/ConfigOptionsCommand.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Console/ConfigOptionsCommand.php b/src/Console/ConfigOptionsCommand.php index d592a8b1..52d62dde 100644 --- a/src/Console/ConfigOptionsCommand.php +++ b/src/Console/ConfigOptionsCommand.php @@ -11,16 +11,19 @@ class ConfigOptionsCommand extends BaseCommand { /** * @var {inheritdoc} + * @phpcsSuppress SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingAnyTypeHint */ protected $name = 'kafka:consume-config-class'; /** * @var {inheritdoc} + * @phpcsSuppress SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingAnyTypeHint */ protected $description = 'Consumes something with a based class config'; /** * @var {inheritdoc} + * @phpcsSuppress SlevomatCodingStandard.TypeHints.PropertyTypeHint.MissingAnyTypeHint */ protected $signature = 'kafka:consume-config-class {handler : handler.} From 5afadef6f5459a4255dcba00d45f1edfd527e34a Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 14:39:10 -0300 Subject: [PATCH 60/72] style: supress phpcs typehing warning Signed-off-by: hcdias --- src/Connectors/AbstractConfig.php | 7 ++++++- src/Connectors/Consumer/Config.php | 2 +- tests/Unit/Connectors/Producer/ConnectorTest.php | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index b71c683c..5b084abc 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -7,6 +7,11 @@ abstract class AbstractConfig { + /** + * @var string[] + */ + protected array $rules; + protected function getBrokerConfig(string $servicesFile): array { if (!$brokerConfig = config($servicesFile . '.broker')) { @@ -15,7 +20,7 @@ protected function getBrokerConfig(string $servicesFile): array ); } - return $brokerConfig; + return $brokerConfig->all(); } protected function getSchemaConfig(string $servicesFile): array diff --git a/src/Connectors/Consumer/Config.php b/src/Connectors/Consumer/Config.php index d13976be..1e870981 100644 --- a/src/Connectors/Consumer/Config.php +++ b/src/Connectors/Consumer/Config.php @@ -19,7 +19,7 @@ class Config extends AbstractConfig { /** - * @var array + * @var string[] */ protected array $rules = [ 'topic' => 'required', diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index 2869318b..650dbd02 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -34,6 +34,7 @@ public function testItShouldMakeSetup(): void $connector = new Connector(); $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface { + /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */ public function success(Message $message): void { } @@ -83,6 +84,7 @@ public function testItShouldMakeSetupWithoutHandleResponse(): void $connector = new Connector(); $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface { + /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */ public function success(Message $message): void { } From f9a312a88e8d39ffb47991f95516c69a956d47cb Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 15:23:31 -0300 Subject: [PATCH 61/72] chore: remove call to inexistent method --- src/Connectors/AbstractConfig.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index 5b084abc..3ed44502 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -20,7 +20,7 @@ protected function getBrokerConfig(string $servicesFile): array ); } - return $brokerConfig->all(); + return $brokerConfig; } protected function getSchemaConfig(string $servicesFile): array From c7ffc005392370f5d9e55633d6fb36f20c8abac7 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 15:30:05 -0300 Subject: [PATCH 62/72] chore: supress psalm alert --- src/Connectors/AbstractConfig.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Connectors/AbstractConfig.php b/src/Connectors/AbstractConfig.php index 3ed44502..ef26cbf5 100644 --- a/src/Connectors/AbstractConfig.php +++ b/src/Connectors/AbstractConfig.php @@ -12,6 +12,9 @@ abstract class AbstractConfig */ protected array $rules; + /** + * @psalm-suppress InvalidReturnStatement + */ protected function getBrokerConfig(string $servicesFile): array { if (!$brokerConfig = config($servicesFile . '.broker')) { From 6d2fedb6fc3ada641cd6610c348cf5e9d81edc5f Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 15:55:36 -0300 Subject: [PATCH 63/72] tests: remove configManager related tests --- tests/Unit/Connectors/Consumer/ManagerTest.php | 18 ------------------ .../Unit/Middlewares/Handler/ProducerTest.php | 16 ---------------- tests/Unit/Producer/PollTest.php | 4 ++-- 3 files changed, 2 insertions(+), 36 deletions(-) diff --git a/tests/Unit/Connectors/Consumer/ManagerTest.php b/tests/Unit/Connectors/Consumer/ManagerTest.php index 230513b2..52e3bcc1 100644 --- a/tests/Unit/Connectors/Consumer/ManagerTest.php +++ b/tests/Unit/Connectors/Consumer/ManagerTest.php @@ -184,22 +184,4 @@ function () use ($messages, &$count, $exception) { $runner->handleMessage(); $runner->handleMessage(); } - - protected function setUp(): void - { - parent::setUp(); - - $configManager = new ConsumerConfigManager(); - $configManager->set([ - 'connections' => 'kafka:2019', - 'topic' => 'topic_key', - 'broker' => 'default', - 'offset_reset' => 'earliest', - 'offset' => 0, - 'timeout' => 30, - 'handler' => ConsumerHandlerDummy::class, - 'middlewares' => [], - 'consumer_group' => 'consumer-id', - ]); - } } diff --git a/tests/Unit/Middlewares/Handler/ProducerTest.php b/tests/Unit/Middlewares/Handler/ProducerTest.php index df2750e6..561a7c78 100644 --- a/tests/Unit/Middlewares/Handler/ProducerTest.php +++ b/tests/Unit/Middlewares/Handler/ProducerTest.php @@ -37,20 +37,4 @@ public function testItShouldSendMessageToKafkaBroker(): void $producerHandler = new Producer($producerTopic, $poll, 1); $producerHandler->process($record, $closure); } - - protected function setUp(): void - { - parent::setUp(); - - $configManager = new ProducerConfigManager(); - $configManager->set([ - 'topic_id' => 'topic_name', - 'timeout' => 4000, - 'is_async' => true, - 'max_poll_records' => 500, - 'flush_attempts' => 10, - 'required_acknowledgment' => true, - 'partition' => 0, - ]); - } } diff --git a/tests/Unit/Producer/PollTest.php b/tests/Unit/Producer/PollTest.php index 0576cd6c..ba216928 100644 --- a/tests/Unit/Producer/PollTest.php +++ b/tests/Unit/Producer/PollTest.php @@ -68,8 +68,8 @@ public function testShouldThrowExceptionWhenFlushFailed(): void ->poll(0); $kafkaProducer->expects() - ->flush(1000) - ->times(3) + ->flush(100) + ->times(10) ->andReturn(1); $this->expectException(RuntimeException::class); From 42ad7dceb7fd652191d9df3f62be4a874160daa3 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 16:14:30 -0300 Subject: [PATCH 64/72] gha: update kafka broker connection --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 59efe8d6..aa9333b8 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -18,7 +18,7 @@ jobs: - '8.0' - '8.1' env: - KAFKA_BROKER_CONNECTIONS: 'localhost:9092' + KAFKA_BROKER_CONNECTIONS: 'kafka:9092' services: zookeeper: image: bitnami/zookeeper From 96e072c534032d48855d5c57746347575ebdf687 Mon Sep 17 00:00:00 2001 From: hcdias Date: Mon, 9 Oct 2023 18:06:43 -0300 Subject: [PATCH 65/72] gha: undo update kafka broker connection --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index aa9333b8..59efe8d6 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -18,7 +18,7 @@ jobs: - '8.0' - '8.1' env: - KAFKA_BROKER_CONNECTIONS: 'kafka:9092' + KAFKA_BROKER_CONNECTIONS: 'localhost:9092' services: zookeeper: image: bitnami/zookeeper From 33556c5c127de5b5b1f8ab2be46449ef3e518c31 Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 10 Oct 2023 11:02:30 -0300 Subject: [PATCH 66/72] gha: remove kafka broker variable --- .github/workflows/continuous-integration.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 59efe8d6..7a8052b7 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -17,8 +17,6 @@ jobs: - '7.4' - '8.0' - '8.1' - env: - KAFKA_BROKER_CONNECTIONS: 'localhost:9092' services: zookeeper: image: bitnami/zookeeper From a827976de1814fd5d2397584127c1575093bffed Mon Sep 17 00:00:00 2001 From: hcdias Date: Tue, 10 Oct 2023 11:30:48 -0300 Subject: [PATCH 67/72] gha: get back kafka broker variable --- .github/workflows/continuous-integration.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 7a8052b7..59efe8d6 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -17,6 +17,8 @@ jobs: - '7.4' - '8.0' - '8.1' + env: + KAFKA_BROKER_CONNECTIONS: 'localhost:9092' services: zookeeper: image: bitnami/zookeeper From be49acff0d71ca32669d8cc2e88adf9a4183c6dc Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 11:49:20 -0300 Subject: [PATCH 68/72] fix(tests): get connection by env --- tests/Integration/ProducerTest.php | 3 ++- tests/Integration/ProducerWithAvroTest.php | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/Integration/ProducerTest.php b/tests/Integration/ProducerTest.php index 8fbe5305..39cedbfd 100644 --- a/tests/Integration/ProducerTest.php +++ b/tests/Integration/ProducerTest.php @@ -206,7 +206,8 @@ private function haveFourProducedMessages(): void private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { - $broker = new Broker('kafka:9092', new None()); + $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); + $broker = new Broker($connections, new None()); return new ProducerConfigOptions( $topicId, diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index 54793d98..2a83cca3 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -170,7 +170,8 @@ private function setLogExpectationsFor(string $message): void private function createProducerConfigOptions(string $topicId): ProducerConfigOptions { - $broker = new Broker('kafka:9092', new None()); + $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); + $broker = new Broker($connections, new None()); return new ProducerConfigOptions( $topicId, From 0e528157f11cf1d1b9c345d3ea079ab44a4d6f22 Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 11:56:13 -0300 Subject: [PATCH 69/72] test(producer): add expect not to perform assertion --- tests/Integration/ProducerWithAvroTest.php | 1 + tests/Integration/ProducerWithConfigOptionsTest.php | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/Integration/ProducerWithAvroTest.php b/tests/Integration/ProducerWithAvroTest.php index 2a83cca3..09237bfc 100644 --- a/tests/Integration/ProducerWithAvroTest.php +++ b/tests/Integration/ProducerWithAvroTest.php @@ -41,6 +41,7 @@ public function testShouldRunAProducerMessagesWithAAvroSchema(): void // I expect that $this->myMessagesHaveBeenLogged(); + $this->expectNotToPerformAssertions(); } protected function haveAHandlerConfigured(): void diff --git a/tests/Integration/ProducerWithConfigOptionsTest.php b/tests/Integration/ProducerWithConfigOptionsTest.php index 4e256a42..ec37a620 100644 --- a/tests/Integration/ProducerWithConfigOptionsTest.php +++ b/tests/Integration/ProducerWithConfigOptionsTest.php @@ -31,6 +31,7 @@ public function testShouldRunAProducerMessagesWithConfigOptions(): void // I Expect That $this->myMessagesHaveBeenLogged(); + $this->expectNotToPerformAssertions(); // When I $this->runTheConsumer(); From f44e2f93f50e0f19283de8b6a5aa3767061873c2 Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 12:02:25 -0300 Subject: [PATCH 70/72] fix(tests): enable to get broker connection from env --- config/service.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/service.php b/config/service.php index 64f81077..7eacd41f 100644 --- a/config/service.php +++ b/config/service.php @@ -19,7 +19,7 @@ 'password' => 'PASSWORD', ], 'broker' => [ - 'connections' => 'kafka:9092', + 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'), 'auth' => [ 'type' => 'ssl', // ssl and none 'ca' => storage_path('ca.pem'), From a3551b50b1ebdb5f7d06107ddf37cf3ad92d1a97 Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 12:11:43 -0300 Subject: [PATCH 71/72] fix(tests): enable to get broker connection from env --- tests/Unit/Connectors/Producer/ConnectorTest.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index 650dbd02..dc2dfe96 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -29,7 +29,8 @@ public function testItShouldMakeSetup(): void m::mock(KafkaProducer::class) ); - $broker = new Broker('kafka:9092', new None()); + $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); + $broker = new Broker($connections, new None()); $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); @@ -79,7 +80,8 @@ public function testItShouldMakeSetupWithoutHandleResponse(): void m::mock(KafkaProducer::class) ); - $broker = new Broker('kafka:9092', new None()); + $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'); + $broker = new Broker($connections, new None()); $producerConfigOptions = m::mock(ProducerConfigOptions::class); $connector = new Connector(); From 40173c0b288ff195ed483f4c06a83f418b836012 Mon Sep 17 00:00:00 2001 From: GetulioMR Date: Tue, 10 Oct 2023 12:17:05 -0300 Subject: [PATCH 72/72] fix(tests): adjust mock expectations --- tests/Unit/Connectors/Producer/ConnectorTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Unit/Connectors/Producer/ConnectorTest.php b/tests/Unit/Connectors/Producer/ConnectorTest.php index dc2dfe96..6652a0e7 100644 --- a/tests/Unit/Connectors/Producer/ConnectorTest.php +++ b/tests/Unit/Connectors/Producer/ConnectorTest.php @@ -52,7 +52,7 @@ public function failed(Message $message): void ->withAnyArgs(); $conf->expects() - ->set('metadata.broker.list', 0); + ->set('metadata.broker.list', $connections); $producerConfigOptions->expects() ->getBroker() @@ -102,7 +102,7 @@ public function failed(Message $message): void ->never(); $conf->expects() - ->set('metadata.broker.list', 0); + ->set('metadata.broker.list', $connections); $producerConfigOptions->expects() ->getBroker()