Skip to content

Commit

Permalink
Generalize the ReindexConfig
Browse files Browse the repository at this point in the history
  • Loading branch information
Toflar committed Dec 13, 2024
1 parent ecb4b51 commit 56afb12
Show file tree
Hide file tree
Showing 6 changed files with 114 additions and 74 deletions.
27 changes: 10 additions & 17 deletions integrations/symfony/src/Command/ReindexCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
namespace CmsIg\Seal\Integration\Symfony\Command;

use CmsIg\Seal\EngineRegistry;
use CmsIg\Seal\Reindex\PartialReindexConfig;
use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
Expand Down Expand Up @@ -44,7 +44,7 @@ protected function configure(): void
$this->addOption('engine', null, InputOption::VALUE_REQUIRED, 'The name of the engine to create the schema for.');
$this->addOption('index', null, InputOption::VALUE_REQUIRED, 'The name of the index to create the schema for.');
$this->addOption('drop', null, InputOption::VALUE_NONE, 'Drop the index before reindexing.');
$this->addOption('bulk-size', null, InputOption::VALUE_REQUIRED, 'The bulk size for reindexing, defaults to 100.');
$this->addOption('bulk-size', null, InputOption::VALUE_REQUIRED, 'The bulk size for reindexing, defaults to 100.', 100);
$this->addOption('datetime-boundary', null, InputOption::VALUE_REQUIRED, 'Do a partial update and limit to only documents that have been changed since a given datetime object.');
$this->addOption('identifiers', null, InputOption::VALUE_REQUIRED, 'Do a partial update and limit to only a comma-separated list of identifiers.');
}
Expand All @@ -54,17 +54,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int
$ui = new SymfonyStyle($input, $output);
/** @var string|null $engineName */
$engineName = $input->getOption('engine');
/** @var string|null $indexName */
$indexName = $input->getOption('index');
/** @var bool $drop */
$drop = $input->getOption('drop');
/** @var int $bulkSize */
$bulkSize = ((int) $input->getOption('bulk-size')) ?: 100; // @phpstan-ignore-line

$partialReindexConfig = PartialReindexConfig::createConditional(
$input->getOption('datetime-boundary') ? new \DateTimeImmutable($input->getOption('datetime-boundary')) : null,
$input->getOption('identifiers') ? \explode(',', (string) $input->getOption('identifiers')) : null,
);

$reindexConfig = ReindexConfig::create()
->withIndex($input->getOption('index'))
->withBulkSize((int) $input->getOption('bulk-size'))
->withDropIndex((bool) $input->getOption('drop'))
->withDateTimeBoundary($input->getOption('datetime-boundary') ? new \DateTimeImmutable($input->getOption('datetime-boundary')) : null)
->withIdentifiers(\explode(',', (string) $input->getOption('identifiers')));

foreach ($this->engineRegistry->getEngines() as $name => $engine) {
if ($engineName && $engineName !== $name) {
Expand All @@ -77,9 +73,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int

$engine->reindex(
$this->reindexProviders,
$indexName,
$drop,
$bulkSize,
$reindexConfig,
function (string $index, int $count, int|null $total) use ($progressBar) {
if (null !== $total) {
$progressBar->setMaxSteps($total);
Expand All @@ -88,7 +82,6 @@ function (string $index, int $count, int|null $total) use ($progressBar) {
$progressBar->setMessage($index);
$progressBar->setProgress($count);
},
$partialReindexConfig,
);

$progressBar->finish();
Expand Down
19 changes: 8 additions & 11 deletions packages/seal/src/Engine.php
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

use CmsIg\Seal\Adapter\AdapterInterface;
use CmsIg\Seal\Exception\DocumentNotFoundException;
use CmsIg\Seal\Reindex\PartialReindexConfig;
use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;
use CmsIg\Seal\Schema\Schema;
use CmsIg\Seal\Search\Condition\IdentifierCondition;
Expand Down Expand Up @@ -135,11 +135,8 @@ public function dropSchema(array $options = []): TaskInterface|null

public function reindex(
iterable $reindexProviders,
string|null $index = null,
bool $dropIndex = false,
int $bulkSize = 100,
ReindexConfig $reindexConfig,
callable|null $progressCallback = null,
PartialReindexConfig|null $partialReindexConfig = null,
): void {
/** @var array<string, ReindexProviderInterface[]> $reindexProvidersPerIndex */
$reindexProvidersPerIndex = [];
Expand All @@ -148,13 +145,13 @@ public function reindex(
continue;
}

if ($reindexProvider::getIndex() === $index || null === $index) {
if ($reindexProvider::getIndex() === $reindexConfig->getIndex() || null === $reindexConfig->getIndex()) {
$reindexProvidersPerIndex[$reindexProvider::getIndex()][] = $reindexProvider;
}
}

foreach ($reindexProvidersPerIndex as $index => $reindexProviders) {
if ($dropIndex && $this->existIndex($index)) {
if ($reindexConfig->shouldDropIndex() && $this->existIndex($index)) {
$task = $this->dropIndex($index, ['return_slow_promise_result' => true]);
$task->wait();
$task = $this->createIndex($index, ['return_slow_promise_result' => true]);
Expand All @@ -167,18 +164,18 @@ public function reindex(
foreach ($reindexProviders as $reindexProvider) {
$this->bulk(
$index,
(function () use ($index, $reindexProvider, $bulkSize, $progressCallback, $partialReindexConfig) {
(function () use ($index, $reindexProvider, $reindexConfig, $progressCallback) {
$count = 0;
$total = $reindexProvider->total();

$lastCount = -1;
foreach ($reindexProvider->provide($partialReindexConfig) as $document) {
foreach ($reindexProvider->provide($reindexConfig) as $document) {
++$count;

yield $document;

if (null !== $progressCallback
&& 0 === ($count % $bulkSize)
&& 0 === ($count % $reindexConfig->getBulkSize())
) {
$lastCount = $count;
$progressCallback($index, $count, $total);
Expand All @@ -192,7 +189,7 @@ public function reindex(
}
})(),
[],
$bulkSize,
$reindexConfig->getBulkSize(),
);
}
}
Expand Down
7 changes: 2 additions & 5 deletions packages/seal/src/EngineInterface.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
namespace CmsIg\Seal;

use CmsIg\Seal\Exception\DocumentNotFoundException;
use CmsIg\Seal\Reindex\PartialReindexConfig;
use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;
use CmsIg\Seal\Search\SearchBuilder;
use CmsIg\Seal\Task\TaskInterface;
Expand Down Expand Up @@ -93,10 +93,7 @@ public function dropSchema(array $options = []): TaskInterface|null;
*/
public function reindex(
iterable $reindexProviders,
string|null $index = null,
bool $dropIndex = false,
int $bulkSize = 100,
ReindexConfig $reindexConfig,
callable|null $progressCallback = null,
PartialReindexConfig|null $partialReindexConfig = null,
): void;
}
40 changes: 0 additions & 40 deletions packages/seal/src/Reindex/PartialReindexConfig.php

This file was deleted.

93 changes: 93 additions & 0 deletions packages/seal/src/Reindex/ReindexConfig.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
<?php

declare(strict_types=1);

/*
* This file is part of the CMS-IG SEAL project.
*
* (c) Alexander Schranz <[email protected]>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/

namespace CmsIg\Seal\Reindex;

final class ReindexConfig
{
private string|null $index = null;
private bool $dropIndex = false;
private int $bulkSize = 100;
private \DateTimeInterface|null $dateTimeBoundary = null;
private array $identifiers = [];

public function getIndex(): string|null
{
return $this->index;
}

public function shouldDropIndex(): bool
{
return $this->dropIndex;
}

public function getBulkSize(): int
{
return $this->bulkSize;
}

public function getDateTimeBoundary(): \DateTimeInterface|null
{
return $this->dateTimeBoundary;
}

public function getIdentifiers(): array
{
return $this->identifiers;
}

public function withDateTimeBoundary(\DateTimeInterface|null $dateTimeBoundary): self
{
$clone = clone $this;
$clone->dateTimeBoundary = $dateTimeBoundary;

return $clone;
}

public function withIdentifiers(array $identifiers): self
{
$clone = clone $this;
$clone->identifiers = $identifiers;

return $clone;
}

public function withBulkSize(int $bulkSize): self
{
$clone = clone $this;
$clone->bulkSize = $bulkSize;

return $clone;
}

public function withIndex(string|null $index): self
{
$clone = clone $this;
$clone->index = $index;

return $clone;
}

public function withDropIndex(bool $dropIndex): self
{
$clone = clone $this;
$clone->dropIndex = $dropIndex;

return $clone;
}

public static function create(): self
{
return new self();
}
}
2 changes: 1 addition & 1 deletion packages/seal/src/Reindex/ReindexProviderInterface.php
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public function total(): int|null;
*
* @return \Generator<array<string, mixed>>
*/
public function provide(PartialReindexConfig|null $partialReindexConfig = null): \Generator;
public function provide(ReindexConfig $reindexConfig): \Generator;

/**
* The name of the index for which the documents are for.
Expand Down

0 comments on commit 56afb12

Please sign in to comment.