diff --git a/CHANGELOG.md b/CHANGELOG.md index 977b0fba0..0d0056302 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- CBOR formatted update requests ### Changed diff --git a/composer.json b/composer.json index 26934ffdb..cffdc3fc0 100755 --- a/composer.json +++ b/composer.json @@ -24,6 +24,7 @@ "require-dev": { "ext-curl": "*", "ext-iconv": "*", + "2tvenom/cborencode": "^1.0", "escapestudios/symfony2-coding-standard": "^3.11", "nyholm/psr7": "^1.8", "php-http/guzzle7-adapter": "^1.0", @@ -34,8 +35,12 @@ "phpunit/phpunit": "^10.5", "rawr/phpunit-data-provider": "^3.3", "roave/security-advisories": "dev-master", + "spomky-labs/cbor-php": "^3.1", "symfony/event-dispatcher": "^5.0 || ^6.0 || ^7.0" }, + "suggest": { + "spomky-labs/cbor-php": "Needed to use CBOR formatted requests with Solr 9.3+" + }, "prefer-stable": true, "config": { "sort-packages": true, diff --git a/docs/plugins.md b/docs/plugins.md index dcba2a9bd..5b82121f0 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -14,6 +14,7 @@ This can be done very easily with this plugin, you can simply keep feeding docum ### Some notes - Solarium issues JSON formatted update requests by default. If you require XML specific functionality, you can set the request format to XML on the plugin instance. XML requests are slower than JSON. +- Solr 9.3 and higher also supports CBOR formatted update requests. You can set the request format to CBOR on the plugin instance if your documents adhere to [current limitations](queries/update-query/best-practices-for-updates.md#known-cbor-limitations). - You can set a custom buffer size. The default is 100 documents, a safe value. By increasing this you can get even better performance, but depending on your document size at some level you will run into memory or request limits. A value of 1000 has been successfully used for indexing 200k documents. - You can use the createDocument method with array input, but you can also manually create document instance and use the addDocument(s) method. - With buffer size X an update request will be sent to Solr for each X docs. You can just keep feeding docs. These buffer flushes don’t include a commit. This is done on purpose. You can add a commit when you’re done, or you can use the Solr auto commit feature. diff --git a/docs/queries/update-query/best-practices-for-updates.md b/docs/queries/update-query/best-practices-for-updates.md index 7e8202525..94de608cc 100644 --- a/docs/queries/update-query/best-practices-for-updates.md +++ b/docs/queries/update-query/best-practices-for-updates.md @@ -40,6 +40,31 @@ $update = $client->createUpdate(); $update->setRequestFormat($update::REQUEST_FORMAT_XML); ``` -### Raw XML update commands +#### Raw XML update commands Solarium makes it easy to build update commands without having to know the underlying XML structure. If you already have XML formatted update commands, you can add them directly to an update query. Make sure they are valid as Solarium will not check this, and set the [XML request format](#xml-vs-json-formatted-update-requests) on the update query. + +### CBOR formatted update requests + +Since Solr 9.3, Solr also supports the [CBOR format for indexing](https://solr.apache.org/guide/solr/latest/indexing-guide/indexing-with-cbor.html). While CBOR requests might be faster to handle by Solr, they are significantly slower and require more memory to build in Solarium than JSON or XML requests. Benchmark your own use cases to determine if this is the right choice for you. + +In order to use CBOR with Solarium, you need to install the `spomky-labs/cbor-php` library separately. + +```sh +composer require spomky-labs/cbor-php +``` + +```php +// get an update query instance +$update = $client->createUpdate(); + +// set CBOR request format +$update->setRequestFormat($update::REQUEST_FORMAT_CBOR); +``` + +#### Known CBOR limitations + +As outlined in [SOLR-17510](https://issues.apache.org/jira/browse/SOLR-17510?focusedCommentId=17892000#comment-17892000), CBOR formatted updates currently have some limitations. + +- You can only add documents, other commands such as delete and commit aren't supported yet. +- There is no support for atomic updates. diff --git a/docs/queries/update-query/building-an-update-query/building-an-update-query.md b/docs/queries/update-query/building-an-update-query/building-an-update-query.md index d9998afe9..e804e8036 100644 --- a/docs/queries/update-query/building-an-update-query/building-an-update-query.md +++ b/docs/queries/update-query/building-an-update-query/building-an-update-query.md @@ -1,5 +1,5 @@ An update query has options and commands. These commands and options are instructions for the client classes to build and execute a request and return the correct result. In the following sections both the options and commands will be discussed in detail. -You can also take a look at the [XML](https://solr.apache.org/guide/uploading-data-with-index-handlers.html#xml-formatted-index-updates) or [JSON](https://solr.apache.org/guide/uploading-data-with-index-handlers.html#json-formatted-index-updates) request formats for more information about the underlying Solr update handler. +You can also take a look at the [XML](https://solr.apache.org/guide/uploading-data-with-index-handlers.html#xml-formatted-index-updates), [JSON](https://solr.apache.org/guide/uploading-data-with-index-handlers.html#json-formatted-index-updates), or [CBOR](https://solr.apache.org/guide/solr/latest/indexing-guide/indexing-with-cbor.html) request formats for more information about the underlying Solr update handler. Options ------- @@ -10,7 +10,7 @@ However, if you do need to customize them for a special case, you can. ### RequestFormat -Solarium issues JSON formatted update requests by default. Set this to XML if you require XML specific functionality. +Solarium issues JSON formatted update requests by default. Set this to XML if you require XML specific functionality. You can also set this to CBOR if you use Solr 9.3 or higher and your use case falls within [current limitations](../best-practices-for-updates.md#known-cbor-limitations). ### ResultClass diff --git a/docs/queries/update-query/update-query.md b/docs/queries/update-query/update-query.md index f0f2f248d..bdbd159ed 100644 --- a/docs/queries/update-query/update-query.md +++ b/docs/queries/update-query/update-query.md @@ -5,3 +5,4 @@ Update queries allow you to add, delete, commit, optimize and rollback commands. - Always use a database or other persistent storage as the source for building documents to add. Don't be tempted to emulate an update command by selecting a document, altering it and adding it. Almost all schemas will have fields that are indexed and not stored. You will lose the data in those fields. - The best way to use update queries is also related to your Solr config. If you are for instance using the autocommit feature of Solr you probably don't want to use a commit command in your update queries. Make sure you know the configuration details of the Solr core you use. - Some functionality is only available with XML formatted or JSON formatted update queries, but not both. Set the appropriate request format if necessary. +- Solr 9.3 and higher also supports CBOR formatted update queries. Be aware that there are some [current limitations](best-practices-for-updates.md#known-cbor-limitations) with this request format. diff --git a/examples/7.5.3-plugin-bufferedupdate-benchmarks.php b/examples/7.5.3-plugin-bufferedupdate-benchmarks.php index 24d963cc9..102883d69 100644 --- a/examples/7.5.3-plugin-bufferedupdate-benchmarks.php +++ b/examples/7.5.3-plugin-bufferedupdate-benchmarks.php @@ -2,27 +2,31 @@ require_once(__DIR__.'/init.php'); +use Composer\InstalledVersions; use Solarium\Core\Client\Adapter\TimeoutAwareInterface; use Solarium\Core\Client\Request; +use Solarium\QueryType\Update\Query\Query; set_time_limit(0); -ini_set('memory_limit', ini_get('suhosin.memory_limit') ?: '-1'); +ini_set('memory_limit', -1); ob_implicit_flush(true); @ob_end_flush(); htmlHeader(); -if (!isset($weight) || !isset($requestFormat)) { +if (!isset($weight) || !isset($addRequestFormat) || !isset($delRequestFormat)) { echo <<<'EOT'

Usage

-

This file is intended to be included by a script that sets two variables:

+

This file is intended to be included by a script that sets three variables:

$weight
Either '' for the regular plugins or 'lite' for the lite versions.
-
$requestFormat
+
$addRequestFormat
Any of the Solarium\QueryType\Update\Query\Query::REQUEST_FORMAT_* constants.
+
$delRequestFormat
+
Solarium\QueryType\Update\Query\Query::REQUEST_FORMAT_JSON or REQUEST_FORMAT_XML.

Example

@@ -35,7 +39,8 @@ use Solarium\QueryType\Update\Query\Query; $weight = ''; - $requestFormat = Query::REQUEST_FORMAT_JSON; + $addRequestFormat = Query::REQUEST_FORMAT_JSON; + $delRequestFormat = Query::REQUEST_FORMAT_JSON; require(__DIR__.'/7.5.3-plugin-bufferedupdate-benchmarks.php'); @@ -46,6 +51,14 @@ exit; } +if (in_array(Query::REQUEST_FORMAT_CBOR, [$addRequestFormat, $delRequestFormat]) && !InstalledVersions::isInstalled('spomky-labs/cbor-php')) { + echo '

Note: The CBOR benchmark requires spomky-labs/cbor-php

'; + + htmlFooter(); + + exit; +} + echo '

Note: These benchmarks can take some time to run!

'; // create a client instance and don't let the adapter timeout @@ -76,10 +89,10 @@ $addBuffer = $client->getPlugin($addPlugin = 'bufferedadd'.$weight); $delBuffer = $client->getPlugin($delPlugin = 'buffereddelete'.$weight); -$addBuffer->setRequestFormat($requestFormat); -$delBuffer->setRequestFormat($requestFormat); +$addBuffer->setRequestFormat($addRequestFormat); +$delBuffer->setRequestFormat($delRequestFormat); -echo '

'.$addPlugin.' / '.$delPlugin.' ('.strtoupper($requestFormat).')

'; +echo '

'.$addPlugin.' ('.strtoupper($addRequestFormat).') / '.$delPlugin.' ('.strtoupper($delRequestFormat).')

'; echo ''; echo ''; echo ''; diff --git a/examples/7.5.3.0-plugin-bufferedupdate-benchmarks-build-only.php b/examples/7.5.3.0-plugin-bufferedupdate-benchmarks-build-only.php new file mode 100644 index 000000000..b72e654a1 --- /dev/null +++ b/examples/7.5.3.0-plugin-bufferedupdate-benchmarks-build-only.php @@ -0,0 +1,119 @@ +Note: These benchmarks build the requests but don\'t execute them'; + +$requestFormats = [ + Query::REQUEST_FORMAT_XML, + Query::REQUEST_FORMAT_JSON, +]; + +if (InstalledVersions::isInstalled('spomky-labs/cbor-php')) { + $requestFormats[] = Query::REQUEST_FORMAT_CBOR; +} else { + echo '

Note: The CBOR benchmark requires spomky-labs/cbor-php

'; +} + +// memory usage is only useful with PHP 8.2+, earlier version don't allow resetting between benchmarks +$withMemoryUsage = function_exists('memory_reset_peak_usage'); + +// create a client instance +$client = new Solarium\Client($adapter, $eventDispatcher, $config); + +// autoload the buffered add plugin +$addBuffer = $client->getPlugin('bufferedaddlite'); + +// return a dummy response instead of executing the query +$response = new Response('', ['HTTP/1.0 200 OK']); + +// keep measures for individual build times +$start = 0; +$buildTimes = []; + +$client->getEventDispatcher()->addListener( + Events::PRE_EXECUTE_REQUEST, + function (PreExecuteRequest $event) use ($response, &$start, &$buildTimes) { + $buildTimes[] = (hrtime(true) - $start) / 1000000; + $event->setResponse($response); + $start = hrtime(true); + } +); + +$docs = 1200000; + +foreach ($requestFormats as $requestFormat) { + $addBuffer->setRequestFormat($requestFormat); + + echo '

'.strtoupper($requestFormat).'

'; + echo '
add buffer sizeadd timedelete buffer sizedelete time
'; + echo ''; + if ($withMemoryUsage) { + echo ''; + } + echo ''; + echo ''; + echo ''; + + foreach ([2000, 200, 20, 2] as $flushes) { + $bufferSize = $docs / $flushes; + + $addBuffer->setBufferSize($bufferSize); + + echo ''; + + $buildTimes = []; + + if ($withMemoryUsage) { + memory_reset_peak_usage(); + } + + $start = hrtime(true); + + for ($i = 0; $i < $docs; ++$i) { + $data = [ + 'id' => sprintf('test-%08d', $i), + 'name' => 'test for buffered add', + 'cat' => ['solarium-test', 'solarium-test-bufferedadd'], + ]; + $addBuffer->createDocument($data); + } + + sort($buildTimes); + $halfway = $flushes / 2; + $total = array_sum($buildTimes); + $min = reset($buildTimes); + $max = end($buildTimes); + $mean = $total / $flushes; + $median = ($buildTimes[$halfway - 1] + $buildTimes[$halfway]) / 2; + echo ''; + echo ''; + echo ''; + echo ''; + echo ''; + + if ($withMemoryUsage) { + $memoryPeakUsage = memory_get_peak_usage() / 1024; + echo ''; + } + + echo ''; + } + + echo '
buffer sizebuild timemem peak usage
minmaxmeanmediantotal
'.$bufferSize.''.(int) $min.' ms'.(int) $max.' ms'.(int) $mean.' ms'.(int) $median.' ms'.(int) $total.' ms'.(int) $memoryPeakUsage.' KiB
'; +} + +htmlFooter(); diff --git a/examples/7.5.3.1-plugin-bufferedupdate-benchmarks-xml.php b/examples/7.5.3.1-plugin-bufferedupdate-benchmarks-xml.php index 7fe3ddc83..610501286 100644 --- a/examples/7.5.3.1-plugin-bufferedupdate-benchmarks-xml.php +++ b/examples/7.5.3.1-plugin-bufferedupdate-benchmarks-xml.php @@ -5,6 +5,7 @@ use Solarium\QueryType\Update\Query\Query; $weight = ''; -$requestFormat = Query::REQUEST_FORMAT_XML; +$addRequestFormat = Query::REQUEST_FORMAT_XML; +$delRequestFormat = Query::REQUEST_FORMAT_XML; require(__DIR__.'/7.5.3-plugin-bufferedupdate-benchmarks.php'); diff --git a/examples/7.5.3.2-plugin-bufferedupdate-lite-benchmarks-xml.php b/examples/7.5.3.2-plugin-bufferedupdate-lite-benchmarks-xml.php index c3437f248..45954334a 100644 --- a/examples/7.5.3.2-plugin-bufferedupdate-lite-benchmarks-xml.php +++ b/examples/7.5.3.2-plugin-bufferedupdate-lite-benchmarks-xml.php @@ -5,6 +5,7 @@ use Solarium\QueryType\Update\Query\Query; $weight = 'lite'; -$requestFormat = Query::REQUEST_FORMAT_XML; +$addRequestFormat = Query::REQUEST_FORMAT_XML; +$delRequestFormat = Query::REQUEST_FORMAT_XML; require(__DIR__.'/7.5.3-plugin-bufferedupdate-benchmarks.php'); diff --git a/examples/7.5.3.3-plugin-bufferedupdate-benchmarks-json.php b/examples/7.5.3.3-plugin-bufferedupdate-benchmarks-json.php index 6bdc3d834..2c8162079 100644 --- a/examples/7.5.3.3-plugin-bufferedupdate-benchmarks-json.php +++ b/examples/7.5.3.3-plugin-bufferedupdate-benchmarks-json.php @@ -5,6 +5,7 @@ use Solarium\QueryType\Update\Query\Query; $weight = ''; -$requestFormat = Query::REQUEST_FORMAT_JSON; +$addRequestFormat = Query::REQUEST_FORMAT_JSON; +$delRequestFormat = Query::REQUEST_FORMAT_JSON; require(__DIR__.'/7.5.3-plugin-bufferedupdate-benchmarks.php'); diff --git a/examples/7.5.3.4-plugin-bufferedupdate-lite-benchmarks-json.php b/examples/7.5.3.4-plugin-bufferedupdate-lite-benchmarks-json.php index e11d4a280..d34e4cf36 100644 --- a/examples/7.5.3.4-plugin-bufferedupdate-lite-benchmarks-json.php +++ b/examples/7.5.3.4-plugin-bufferedupdate-lite-benchmarks-json.php @@ -5,6 +5,7 @@ use Solarium\QueryType\Update\Query\Query; $weight = 'lite'; -$requestFormat = Query::REQUEST_FORMAT_JSON; +$addRequestFormat = Query::REQUEST_FORMAT_JSON; +$delRequestFormat = Query::REQUEST_FORMAT_JSON; require(__DIR__.'/7.5.3-plugin-bufferedupdate-benchmarks.php'); diff --git a/examples/7.5.3.5-plugin-bufferedupdate-benchmarks-cbor.php b/examples/7.5.3.5-plugin-bufferedupdate-benchmarks-cbor.php new file mode 100644 index 000000000..950c3b48d --- /dev/null +++ b/examples/7.5.3.5-plugin-bufferedupdate-benchmarks-cbor.php @@ -0,0 +1,12 @@ +Examples
  • 7.5.2 Buffered Delete by ID and query
  • 7.5.3 Benchmarks (can take some time to run!)
  • 7.6 Prefetch iterator for select queries
    • diff --git a/src/Core/Client/Request.php b/src/Core/Client/Request.php index be690a2ae..c471f18e7 100644 --- a/src/Core/Client/Request.php +++ b/src/Core/Client/Request.php @@ -46,6 +46,11 @@ class Request extends Configurable implements RequestParamsInterface */ const METHOD_PUT = 'PUT'; + /** + * Content-Type for CBOR payloads. + */ + const CONTENT_TYPE_APPLICATION_CBOR = 'application/cbor'; + /** * Content-Type for JSON payloads. */ diff --git a/src/Plugin/BufferedAdd/BufferedAdd.php b/src/Plugin/BufferedAdd/BufferedAdd.php index ed5e1fca3..a2d0871a9 100755 --- a/src/Plugin/BufferedAdd/BufferedAdd.php +++ b/src/Plugin/BufferedAdd/BufferedAdd.php @@ -118,7 +118,16 @@ public function commit(?bool $overwrite = null, ?bool $softCommit = null, ?bool } $this->updateQuery->add(null, $command); - $this->updateQuery->addCommit($event->getSoftCommit(), $event->getWaitSearcher(), $event->getExpungeDeletes()); + + if ($this->updateQuery::REQUEST_FORMAT_CBOR === $this->getRequestFormat()) { + $this->updateQuery->addParam('commit', true); + $this->updateQuery->addParam('softCommit', $event->getSoftCommit()); + $this->updateQuery->addParam('waitSearcher', $event->getWaitSearcher()); + $this->updateQuery->addParam('expungeDeletes', $event->getExpungeDeletes()); + } else { + $this->updateQuery->addCommit($event->getSoftCommit(), $event->getWaitSearcher(), $event->getExpungeDeletes()); + } + $result = $this->client->update($this->updateQuery, $this->getEndpoint()); $this->clear(); diff --git a/src/Plugin/BufferedAdd/BufferedAddLite.php b/src/Plugin/BufferedAdd/BufferedAddLite.php index e8298bc85..137044630 100644 --- a/src/Plugin/BufferedAdd/BufferedAddLite.php +++ b/src/Plugin/BufferedAdd/BufferedAddLite.php @@ -201,7 +201,16 @@ public function commit(?bool $overwrite = null, ?bool $softCommit = null, ?bool } $this->updateQuery->add(null, $command); - $this->updateQuery->addCommit($softCommit, $waitSearcher, $expungeDeletes); + + if ($this->updateQuery::REQUEST_FORMAT_CBOR === $this->getRequestFormat()) { + $this->updateQuery->addParam('commit', true); + $this->updateQuery->addParam('softCommit', $softCommit); + $this->updateQuery->addParam('waitSearcher', $waitSearcher); + $this->updateQuery->addParam('expungeDeletes', $expungeDeletes); + } else { + $this->updateQuery->addCommit($softCommit, $waitSearcher, $expungeDeletes); + } + $result = $this->client->update($this->updateQuery, $this->getEndpoint()); $this->clear(); diff --git a/src/Plugin/BufferedDelete/BufferedDeleteLite.php b/src/Plugin/BufferedDelete/BufferedDeleteLite.php index bbf62856a..bd04cac29 100644 --- a/src/Plugin/BufferedDelete/BufferedDeleteLite.php +++ b/src/Plugin/BufferedDelete/BufferedDeleteLite.php @@ -9,6 +9,7 @@ namespace Solarium\Plugin\BufferedDelete; +use Solarium\Exception\InvalidArgumentException; use Solarium\Exception\RuntimeException; use Solarium\Plugin\AbstractBufferedUpdate\AbstractBufferedUpdate; use Solarium\Plugin\BufferedDelete\Delete\Id as DeleteById; @@ -113,6 +114,26 @@ public function getDeletes(): array return $this->buffer; } + /** + * Set the request format for the updates. + * + * Use UpdateQuery::REQUEST_FORMAT_JSON or UpdateQuery::REQUEST_FORMAT_XML as value. + * + * @param string $requestFormat + * + * @throws InvalidArgumentException + * + * @return self Provides fluent interface + */ + public function setRequestFormat(string $requestFormat): self + { + if ($this->updateQuery::REQUEST_FORMAT_CBOR === strtolower($requestFormat)) { + throw new InvalidArgumentException('Unsupported request format: CBOR can only be used to add documents'); + } + + return parent::setRequestFormat($requestFormat); + } + /** * Flush any buffered deletes to Solr. * diff --git a/src/QueryType/Update/Query/Query.php b/src/QueryType/Update/Query/Query.php index ccbbfaa1c..da446f83c 100644 --- a/src/QueryType/Update/Query/Query.php +++ b/src/QueryType/Update/Query/Query.php @@ -23,6 +23,7 @@ use Solarium\QueryType\Update\Query\Command\Optimize as OptimizeCommand; use Solarium\QueryType\Update\Query\Command\RawXml as RawXmlCommand; use Solarium\QueryType\Update\Query\Command\Rollback as RollbackCommand; +use Solarium\QueryType\Update\RequestBuilder\Cbor as CborRequestBuilder; use Solarium\QueryType\Update\RequestBuilder\Json as JsonRequestBuilder; use Solarium\QueryType\Update\RequestBuilder\Xml as XmlRequestBuilder; use Solarium\QueryType\Update\ResponseParser; @@ -67,6 +68,11 @@ class Query extends BaseQuery */ const COMMAND_ROLLBACK = 'rollback'; + /** + * CBOR request format. + */ + const REQUEST_FORMAT_CBOR = 'cbor'; + /** * JSON request format. */ @@ -97,6 +103,7 @@ class Query extends BaseQuery * @var array */ protected $requestFormats = [ + self::REQUEST_FORMAT_CBOR => CborRequestBuilder::class, self::REQUEST_FORMAT_JSON => JsonRequestBuilder::class, self::REQUEST_FORMAT_XML => XmlRequestBuilder::class, ]; diff --git a/src/QueryType/Update/RequestBuilder/Cbor.php b/src/QueryType/Update/RequestBuilder/Cbor.php new file mode 100644 index 000000000..65afb5f21 --- /dev/null +++ b/src/QueryType/Update/RequestBuilder/Cbor.php @@ -0,0 +1,208 @@ +getInputEncoding(); + + if (null !== $inputEncoding && 0 !== strcasecmp('UTF-8', $inputEncoding)) { + // @see https://www.rfc-editor.org/rfc/rfc8949#section-3.1-2.8 + // @see https://www.rfc-editor.org/rfc/rfc8949#section-5.3.1-2.4 + throw new RuntimeException('CBOR requests can only contain UTF-8 strings'); + } + + $this->request = parent::build($query); + $this->request->setMethod(Request::METHOD_POST); + $this->request->setContentType(Request::CONTENT_TYPE_APPLICATION_CBOR); + $this->request->setRawData($this->getRawData($query)); + + return $this->request; + } + + /** + * Generates raw POST data. + * + * Each commandtype is delegated to a separate builder method. + * + * @param UpdateQuery $query + * + * @throws RuntimeException + * + * @return string + */ + public function getRawData(UpdateQuery $query): string + { + $cbor = IndefiniteLengthListObject::create(); + + foreach ($query->getCommands() as $command) { + if (UpdateQuery::COMMAND_ADD === $command->getType()) { + /* @var Add $command */ + $this->addDocuments($command->getDocuments(), $cbor); + + if (null !== $overwrite = $command->getOverwrite()) { + $this->request->addParam('overwrite', $overwrite, true); + } + + if (null !== $commitWithin = $command->getCommitWithin()) { + $this->request->addParam('commitWithin', $commitWithin, true); + } + } else { + throw new RuntimeException('Unsupported command type, CBOR queries can only be used to add documents'); + } + } + + return (string) $cbor; + } + + /** + * Add documents. + * + * @param DocumentInterface[] $documents + * @param IndefiniteLengthListObject $cbor + */ + protected function addDocuments(array $documents, IndefiniteLengthListObject $cbor): void + { + foreach ($documents as $doc) { + $fields = IndefiniteLengthMapObject::create(); + + foreach ($doc->getFields() as $name => $value) { + $modifier = $doc->getFieldModifier($name); + + $fields->add( + TextStringObject::create($name), + $this->buildFieldCborObject($value, $modifier) + ); + } + + if (null !== $version = $doc->getVersion()) { + $fields->add( + TextStringObject::create('_version_'), + 0 > $version ? NegativeIntegerObject::create($version) : UnsignedIntegerObject::create($version) + ); + } + + $cbor->add($fields); + } + } + + /** + * Build a CBOR object that represents a document field value. + * + * @param mixed $value + * @param string|null $modifier + * + * @return CBORObject + */ + protected function buildFieldCborObject($value, ?string $modifier = null): CBORObject + { + if (\is_array($value)) { + if (empty($value)) { + $cbor = ListObject::create(); + } elseif (is_numeric(array_key_first($value))) { + $cbor = ListObject::create(array_map( + fn ($v): CBORObject => $this->buildFieldCborObject($v), + $value + )); + } else { + $cbor = IndefiniteLengthMapObject::create(); + + foreach ($value as $k => $v) { + $cbor->add( + TextStringObject::create($k), + $this->buildFieldCborObject($v) + ); + } + } + } else { + $cbor = $this->buildScalarCborObject($value); + } + + if (null !== $modifier) { + $cbor = MapObject::create([ + MapItem::create(TextStringObject::create($modifier), $cbor), + ]); + } + + return $cbor; + } + + /** + * Build a CBOR object that represents a scalar value. + * + * @param scalar $value + * + * @return CBORObject + */ + protected function buildScalarCborObject($value): CBORObject + { + if (null === $value) { + return NullObject::create(); + } elseif (false === $value) { + return FalseObject::create(); + } elseif (true === $value) { + return TrueObject::create(); + } elseif (\is_int($value)) { + return 0 > $value ? NegativeIntegerObject::create($value) : UnsignedIntegerObject::create($value); + } elseif (\is_float($value)) { + return DoublePrecisionFloatObject::create(pack('E', $value)); + } elseif ($value instanceof \DateTimeInterface) { + return TextStringObject::create($this->getHelper()->formatDate($value)); + } else { + return TextStringObject::create($value); + } + } +} diff --git a/src/QueryType/Update/RequestBuilder/Json.php b/src/QueryType/Update/RequestBuilder/Json.php index 3cc852fef..e26bfe4ad 100644 --- a/src/QueryType/Update/RequestBuilder/Json.php +++ b/src/QueryType/Update/RequestBuilder/Json.php @@ -126,7 +126,7 @@ public function buildAddJson(Add $command, array &$json): void */ public function buildDeleteJson(Delete $command, array &$json): void { - if (0 !== count($ids = $command->getIds())) { + if (0 !== \count($ids = $command->getIds())) { $json[] = '"delete":'.json_encode($ids); } @@ -136,7 +136,7 @@ public function buildDeleteJson(Delete $command, array &$json): void } /** - * Build JSON for an optimize command. + * Add JSON for an optimize command. * * @param Optimize $command * @param array $json @@ -161,7 +161,7 @@ public function buildOptimizeJson(Optimize $command, array &$json): void } /** - * Build JSON for a commit command. + * Add JSON for a commit command. * * @param Commit $command * @param array $json @@ -186,7 +186,7 @@ public function buildCommitJson(Commit $command, array &$json): void } /** - * Build JSON for a rollback command. + * Add JSON for a rollback command. * * @param array $json */ diff --git a/tests/Integration/AbstractTechproductsTestCase.php b/tests/Integration/AbstractTechproductsTestCase.php index 4f5c7df3f..ce888c223 100644 --- a/tests/Integration/AbstractTechproductsTestCase.php +++ b/tests/Integration/AbstractTechproductsTestCase.php @@ -213,6 +213,8 @@ public static function responseWriterProvider(): array /** * This data provider should be used by all UpdateQuery tests that don't test request * format specific Commands to ensure functional equivalence between the formats. + * + * CBOR format isn't included here because it can only be used for Add Commands. */ public static function updateRequestFormatProvider(): array { @@ -3197,6 +3199,165 @@ public function testAnonymouslyNestedDocuments(string $requestFormat) $this->assertCount(0, $result); } + public function testUpdateCbor() + { + // support for CBOR format was added in Solr 9.3, pass tacitly for older versions + if (9 > self::$solrVersion) { + $this->expectNotToPerformAssertions(); + + return; + } + + // add + $update = self::$client->createUpdate(); + $update->setRequestFormat($update::REQUEST_FORMAT_CBOR); + $doc1 = $update->createDocument(); + $doc1->setField('id', 'solarium-cbor-test-1'); + $doc1->setField('name', 'Sølåríùm CBOR Tëst 1'); + $doc1->setField('cat', 'solarium-cbor-test'); + $doc1->setField('price', 3.14); + $doc1->setField('popularity', 5); + $doc1->setField('inStock', true); + $doc1->setField('content', ['foo', 'bar']); + $doc2 = $update->createDocument(); + $doc2->setField('id', 'solarium-cbor-test-2'); + $doc2->setField('name', 'Sølåríùm CBOR Tëst 2'); + $doc2->setField('cat', 'solarium-cbor-test'); + $doc2->setField('price', 42.0); + $doc2->setField('popularity', -1); + $doc2->setField('inStock', false); + $doc2->setField('content', []); + $update->addDocuments([$doc1, $doc2]); + $update->addParam('commit', true); + $update->addParam('softCommit', true); + self::$client->update($update); + + $select = self::$client->createSelect(); + $select->setQuery('cat:solarium-cbor-test'); + $select->addSort('id', $select::SORT_ASC); + $select->setFields('id,name,price,popularity,inStock,content'); + + $result = self::$client->select($select); + $this->assertCount(2, $result); + $iterator = $result->getIterator(); + $this->assertSame([ + 'id' => 'solarium-cbor-test-1', + 'name' => 'Sølåríùm CBOR Tëst 1', + 'price' => 3.14, + 'popularity' => 5, + 'inStock' => true, + 'content' => [ + 'foo', + 'bar', + ], + ], $iterator->current()->getFields()); + $iterator->next(); + $this->assertSame([ + 'id' => 'solarium-cbor-test-2', + 'name' => 'Sølåríùm CBOR Tëst 2', + 'price' => 42.0, + 'popularity' => -1, + 'inStock' => false, + ], $iterator->current()->getFields()); + + // atomic updates + try { + $update = self::$client->createUpdate(); + $update->setRequestFormat($update::REQUEST_FORMAT_CBOR); + $doc = $update->createDocument(); + $doc->setKey('id', 'solarium-cbor-test-1'); + $doc->setField('popularity', -1); + $doc->setFieldModifier('popularity', $doc::MODIFIER_INC); + $doc->setField('content', ['bar', 'baz']); + $doc->setFieldModifier('content', $doc::MODIFIER_ADD_DISTINCT); + $doc->setField('inStock', null); + $doc->setFieldModifier('inStock', $doc::MODIFIER_SET); + $update->addDocument($doc); + $update->addParam('commit', true); + $update->addParam('softCommit', true); + self::$client->update($update); + + $result = self::$client->select($select); + $this->assertCount(2, $result); + $iterator = $result->getIterator(); + $this->assertSame([ + 'id' => 'solarium-cbor-test-1', + 'name' => 'Sølåríùm CBOR Tëst 1', + 'price' => 3.14, + 'popularity' => 4, + 'content' => [ + 'foo', + 'bar', + 'baz', + ], + ], $iterator->current()->getFields()); + } catch (HttpException $e) { + // rethrow if it's not a Solr version not supporting atomic updates with CBOR + // @see https://issues.apache.org/jira/browse/SOLR-17510?focusedCommentId=17892000#comment-17892000 + if (400 !== $e->getCode() || !str_contains($e->getMessage(), '[doc=solarium-cbor-test-1/popularity#] unknown field \'inc\'')) { + throw $e; + } + } + + // nested documents + $data = [ + 'id' => 'solarium-cbor-test-3', + 'cat' => ['solarium-cbor-test'], + 'single_child' => [ + 'id' => 'solarium-single-child', + 'cat' => ['solarium-cbor-test'], + ], + 'children' => [ + [ + 'id' => 'solarium-child-1', + 'cat' => ['solarium-cbor-test'], + ], + [ + 'id' => 'solarium-child-2', + 'cat' => ['solarium-cbor-test'], + ], + ], + ]; + $update = self::$client->createUpdate(); + $update->setRequestFormat($update::REQUEST_FORMAT_CBOR); + $doc = $update->createDocument($data); + $update->addDocument($doc); + $update->addParam('commit', true); + $update->addParam('softCommit', true); + self::$client->update($update); + + $select = self::$client->createSelect(); + $select->setQuery('id:solarium-cbor-test-3'); + $select->addSort('id', $select::SORT_ASC); + $select->setFields('id,single_child,children,[child]'); + + $result = self::$client->select($select); + $this->assertCount(1, $result); + $iterator = $result->getIterator(); + $this->assertSame([ + 'id' => 'solarium-cbor-test-3', + 'single_child' => [ + 'id' => 'solarium-single-child', + ], + 'children' => [ + [ + 'id' => 'solarium-child-1', + ], + [ + 'id' => 'solarium-child-2', + ], + ], + ], $iterator->current()->getFields()); + + // cleanup with default request format (can't delete with CBOR) + $update = self::$client->createUpdate(); + $update->addDeleteQuery('cat:solarium-cbor-test'); + $update->addCommit(true, true); + self::$client->update($update); + $result = self::$client->select($select); + $this->assertCount(0, $result); + } + public function testUpdateWithoutControlCharacterFiltering() { $data = [ @@ -3880,6 +4041,90 @@ public function testBufferedAddAndDeleteLite(string $requestFormat) $this->assertSame(0, $result->getNumFound()); } + public function testBufferedAddCbor() + { + // support for CBOR format was added in Solr 9.3, pass tacitly for older versions + if (9 > self::$solrVersion) { + $this->expectNotToPerformAssertions(); + + return; + } + + $bufferSize = 10; + $totalDocs = 25; + + $buffer = self::$client->getPlugin('bufferedaddlite'); + $buffer->setRequestFormat(UpdateQuery::REQUEST_FORMAT_CBOR); + $buffer->setBufferSize($bufferSize); + + $update = self::$client->createUpdate(); + for ($i = 1; $i <= $totalDocs; ++$i) { + $data = [ + 'id' => 'solarium-bufferedadd-cbor-'.$i, + 'cat' => 'solarium-bufferedadd-cbor', + 'weight' => $i, + ]; + $document = $update->createDocument($data); + $buffer->addDocument($document); + } + + // flush first to test if commit works with an empty buffer + // (unlike other request formats, commit doesn't add a command to the update query) + $buffer->flush(); + $buffer->commit(null, true, true); + + $select = self::$client->createSelect(); + $select->setQuery('cat:solarium-bufferedadd-cbor'); + $select->addSort('weight', $select::SORT_ASC); + $select->setFields('id'); + $select->setRows($totalDocs); + $result = self::$client->select($select); + $this->assertSame($totalDocs, $result->getNumFound()); + + $ids = []; + /** @var \Solarium\QueryType\Select\Result\Document $document */ + foreach ($result as $document) { + $ids[] = $document->id; + } + + $this->assertEquals([ + 'solarium-bufferedadd-cbor-1', + 'solarium-bufferedadd-cbor-2', + 'solarium-bufferedadd-cbor-3', + 'solarium-bufferedadd-cbor-4', + 'solarium-bufferedadd-cbor-5', + 'solarium-bufferedadd-cbor-6', + 'solarium-bufferedadd-cbor-7', + 'solarium-bufferedadd-cbor-8', + 'solarium-bufferedadd-cbor-9', + 'solarium-bufferedadd-cbor-10', + 'solarium-bufferedadd-cbor-11', + 'solarium-bufferedadd-cbor-12', + 'solarium-bufferedadd-cbor-13', + 'solarium-bufferedadd-cbor-14', + 'solarium-bufferedadd-cbor-15', + 'solarium-bufferedadd-cbor-16', + 'solarium-bufferedadd-cbor-17', + 'solarium-bufferedadd-cbor-18', + 'solarium-bufferedadd-cbor-19', + 'solarium-bufferedadd-cbor-20', + 'solarium-bufferedadd-cbor-21', + 'solarium-bufferedadd-cbor-22', + 'solarium-bufferedadd-cbor-23', + 'solarium-bufferedadd-cbor-24', + 'solarium-bufferedadd-cbor-25', + ], $ids); + + // cleanup + self::$client->removePlugin('bufferedaddlite'); + $update = self::$client->createUpdate(); + $update->addDeleteQuery('cat:solarium-bufferedadd-cbor'); + $update->addCommit(true, true); + self::$client->update($update); + $result = self::$client->select($select); + $this->assertCount(0, $result); + } + public function testLoadbalancerFailover() { $invalidEndpointConfig = self::$config['endpoint']['localhost']; diff --git a/tests/Plugin/BufferedAdd/BufferedAddLiteTest.php b/tests/Plugin/BufferedAdd/BufferedAddLiteTest.php index d32848ba1..1c5c27e2f 100644 --- a/tests/Plugin/BufferedAdd/BufferedAddLiteTest.php +++ b/tests/Plugin/BufferedAdd/BufferedAddLiteTest.php @@ -35,6 +35,15 @@ public function setUp(): void $this->plugin->initPlugin(TestClientFactory::createWithCurlAdapter(), []); } + public static function updateRequestFormatProvider(): array + { + return [ + [Query::REQUEST_FORMAT_XML], + [Query::REQUEST_FORMAT_JSON], + [Query::REQUEST_FORMAT_CBOR], + ]; + } + public function testInitPlugin() { $client = TestClientFactory::createWithCurlAdapter(); @@ -317,13 +326,19 @@ public function testFlushEmptyBuffer() $this->assertFalse($this->plugin->flush()); } - public function testFlush() + /** + * @dataProvider updateRequestFormatProvider + */ + public function testFlush(string $requestFormat) { $doc1 = new Document(['id' => '123', 'name' => 'test 1']); $doc2 = new Document(['id' => '456', 'name' => 'test 2']); $doc3 = new Document(['id' => '789', 'name' => 'test 3']); - $mockUpdate = $this->createMock(Query::class); + /** @var Query|MockObject $mockUpdate */ + $mockUpdate = $this->getMockBuilder(Query::class) + ->onlyMethods(['add']) + ->getMock(); $mockUpdate->expects($this->once()) ->method('add') ->with( @@ -340,28 +355,41 @@ public function testFlush() $pluginClass = \get_class($this->plugin); $plugin = new $pluginClass(); $plugin->initPlugin($mockClient, []); + $plugin->setRequestFormat($requestFormat); $plugin->addDocuments([$doc1, $doc2]); $plugin->addDocument($doc3); $this->assertSame($mockResult, $plugin->flush(true, 12)); } - public function testCommit() + /** + * @dataProvider updateRequestFormatProvider + */ + public function testCommit(string $requestFormat) { $doc1 = new Document(['id' => '123', 'name' => 'test 1']); $doc2 = new Document(['id' => '456', 'name' => 'test 2']); $doc3 = new Document(['id' => '789', 'name' => 'test 3']); - $mockUpdate = $this->createMock(Query::class); + /** @var Query|MockObject $mockUpdate */ + $mockUpdate = $this->getMockBuilder(Query::class) + ->onlyMethods(['add', 'addCommit']) + ->getMock(); $mockUpdate->expects($this->once()) ->method('add') ->with( $this->equalTo(null), $this->equalTo((new AddCommand())->setOverwrite(true)->addDocuments([$doc1, $doc2, $doc3])), ); - $mockUpdate->expects($this->once()) - ->method('addCommit') - ->with($this->equalTo(false), $this->equalTo(true), $this->equalTo(false)); + + if (Query::REQUEST_FORMAT_CBOR === $requestFormat) { + $mockUpdate->expects($this->never()) + ->method('addCommit'); + } else { + $mockUpdate->expects($this->once()) + ->method('addCommit') + ->with($this->equalTo(false), $this->equalTo(true), $this->equalTo(false)); + } $mockResult = $this->createMock(Result::class); @@ -372,27 +400,49 @@ public function testCommit() $pluginClass = \get_class($this->plugin); $plugin = new $pluginClass(); $plugin->initPlugin($mockClient, []); + $plugin->setRequestFormat($requestFormat); $plugin->addDocument($doc1); $plugin->addDocuments([$doc2, $doc3]); $this->assertSame($mockResult, $plugin->commit(true, false, true, false)); + + if (Query::REQUEST_FORMAT_CBOR === $requestFormat) { + $params = $mockUpdate->getParams(); + + $this->assertTrue($params['commit']); + $this->assertFalse($params['softCommit']); + $this->assertTrue($params['waitSearcher']); + $this->assertFalse($params['expungeDeletes']); + } } - public function testCommitWithOptionalValues() + /** + * @dataProvider updateRequestFormatProvider + */ + public function testCommitWithOptionalValues(string $requestFormat) { $doc1 = new Document(['id' => '123', 'name' => 'test 1']); $doc2 = new Document(['id' => '456', 'name' => 'test 2']); - $mockUpdate = $this->createMock(Query::class); + /** @var Query|MockObject $mockUpdate */ + $mockUpdate = $this->getMockBuilder(Query::class) + ->onlyMethods(['add', 'addCommit']) + ->getMock(); $mockUpdate->expects($this->once()) ->method('add') ->with( $this->equalTo(null), $this->equalTo((new AddCommand())->setOverwrite(true)->addDocuments([$doc1, $doc2])), ); - $mockUpdate->expects($this->once()) - ->method('addCommit') - ->with($this->equalTo(null), $this->equalTo(null), $this->equalTo(null)); + + if (Query::REQUEST_FORMAT_CBOR === $requestFormat) { + $mockUpdate->expects($this->never()) + ->method('addCommit'); + } else { + $mockUpdate->expects($this->once()) + ->method('addCommit') + ->with($this->equalTo(null), $this->equalTo(null), $this->equalTo(null)); + } $mockResult = $this->createMock(Result::class); @@ -403,11 +453,22 @@ public function testCommitWithOptionalValues() $pluginClass = \get_class($this->plugin); $plugin = new $pluginClass(); $plugin->initPlugin($mockClient, []); + $plugin->setRequestFormat($requestFormat); $plugin->addDocument($doc1); $plugin->addDocument($doc2); $plugin->setOverwrite(true); $this->assertSame($mockResult, $plugin->commit(null, null, null, null)); + + if (Query::REQUEST_FORMAT_CBOR === $requestFormat) { + $params = $mockUpdate->getParams(); + + $this->assertTrue($params['commit']); + // explicitly null or omitted is an implementation detail with the same end result + $this->assertNull($params['softCommit'] ?? null); + $this->assertNull($params['waitSearcher'] ?? null); + $this->assertNull($params['expungeDeletes'] ?? null); + } } public function testSetAndGetEndpoint() diff --git a/tests/Plugin/BufferedDelete/BufferedDeleteLiteTest.php b/tests/Plugin/BufferedDelete/BufferedDeleteLiteTest.php index a43ff989a..884b054de 100644 --- a/tests/Plugin/BufferedDelete/BufferedDeleteLiteTest.php +++ b/tests/Plugin/BufferedDelete/BufferedDeleteLiteTest.php @@ -38,6 +38,14 @@ public function setUp(): void $this->plugin->initPlugin(TestClientFactory::createWithCurlAdapter(), []); } + public static function updateRequestFormatProvider(): array + { + return [ + [Query::REQUEST_FORMAT_XML], + [Query::REQUEST_FORMAT_JSON], + ]; + } + public function testInitPlugin() { $client = TestClientFactory::createWithCurlAdapter(); @@ -334,9 +342,15 @@ public function testFlushEmptyBuffer() $this->assertFalse($this->plugin->flush()); } - public function testFlush() + /** + * @dataProvider updateRequestFormatProvider + */ + public function testFlush(string $requestFormat) { - $mockUpdate = $this->createMock(Query::class); + /** @var Query|MockObject $mockUpdate */ + $mockUpdate = $this->getMockBuilder(Query::class) + ->onlyMethods(['add']) + ->getMock(); $mockUpdate->expects($this->once()) ->method('add') ->with( @@ -353,6 +367,7 @@ public function testFlush() $pluginClass = \get_class($this->plugin); $plugin = new $pluginClass(); $plugin->initPlugin($mockClient, []); + $plugin->setRequestFormat($requestFormat); $plugin->addDeleteById('abc'); $plugin->addDeleteQuery('cat:def'); @@ -370,9 +385,15 @@ public function testFlushUnknownType() $plugin->flush(); } - public function testCommit() + /** + * @dataProvider updateRequestFormatProvider + */ + public function testCommit(string $requestFormat) { - $mockUpdate = $this->createMock(Query::class); + /** @var Query|MockObject $mockUpdate */ + $mockUpdate = $this->getMockBuilder(Query::class) + ->onlyMethods(['add', 'addCommit']) + ->getMock(); $mockUpdate->expects($this->once()) ->method('add') ->with( @@ -392,12 +413,48 @@ public function testCommit() $pluginClass = \get_class($this->plugin); $plugin = new $pluginClass(); $plugin->initPlugin($mockClient, []); + $plugin->setRequestFormat($requestFormat); $plugin->addDeleteById('abc'); $plugin->addDeleteQuery('cat:def'); $this->assertSame($mockResult, $plugin->commit(false, true, false)); } + /** + * @dataProvider updateRequestFormatProvider + */ + public function testCommitWithOptionalValues(string $requestFormat) + { + /** @var Query|MockObject $mockUpdate */ + $mockUpdate = $this->getMockBuilder(Query::class) + ->onlyMethods(['add', 'addCommit']) + ->getMock(); + $mockUpdate->expects($this->once()) + ->method('add') + ->with( + $this->equalTo(null), + $this->equalTo((new DeleteCommand())->addId('abc')->addQuery('cat:def')), + ); + $mockUpdate->expects($this->once()) + ->method('addCommit') + ->with($this->equalTo(null), $this->equalTo(null), $this->equalTo(null)); + + $mockResult = $this->createMock(Result::class); + + $mockClient = $this->getClient(); + $mockClient->expects($this->exactly(2))->method('createUpdate')->willReturn($mockUpdate); + $mockClient->expects($this->once())->method('update')->willReturn($mockResult); + + $pluginClass = \get_class($this->plugin); + $plugin = new $pluginClass(); + $plugin->initPlugin($mockClient, []); + $plugin->setRequestFormat($requestFormat); + $plugin->addDeleteById('abc'); + $plugin->addDeleteQuery('cat:def'); + + $this->assertSame($mockResult, $plugin->commit(null, null, null)); + } + public function testSetAndGetEndpoint() { $endpoint = new Endpoint(); @@ -417,6 +474,24 @@ public function testSetAndGetRequestFormat() $this->assertSame(Query::REQUEST_FORMAT_XML, $this->plugin->getRequestFormat()); } + /** + * @dataProvider cborRequestFormatProvider + */ + public function testSetCborRequestFormat(string $requestFormat) + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Unsupported request format: CBOR can only be used to add documents'); + $this->plugin->setRequestFormat($requestFormat); + } + + public static function cborRequestFormatProvider(): array + { + return [ + [strtolower(Query::REQUEST_FORMAT_CBOR)], + [strtoupper(Query::REQUEST_FORMAT_CBOR)], + ]; + } + public function testSetUnsupportedRequestFormat() { $this->expectException(InvalidArgumentException::class); diff --git a/tests/QueryType/Update/RequestBuilder/CborTest.php b/tests/QueryType/Update/RequestBuilder/CborTest.php new file mode 100644 index 000000000..222857b7e --- /dev/null +++ b/tests/QueryType/Update/RequestBuilder/CborTest.php @@ -0,0 +1,824 @@ +query = new Query(); + $this->query->setRequestFormat(Query::REQUEST_FORMAT_CBOR); + + $this->builder = new CborRequestBuilder(); + } + + public function testGetMethod() + { + $request = $this->builder->build($this->query); + $this->assertSame( + Request::METHOD_POST, + $request->getMethod() + ); + } + + public function testGetContentType() + { + $request = $this->builder->build($this->query); + $this->assertSame( + Request::CONTENT_TYPE_APPLICATION_CBOR, + $request->getContentType() + ); + } + + public function testGetUri() + { + $request = $this->builder->build($this->query); + $this->assertSame( + 'update?omitHeader=false&wt=json&json.nl=flat', + $request->getUri() + ); + } + + /** + * Update queries with a different input encoding than the default UTF-8 + * aren't supported by the CBOR request format. + * + * @see https://www.rfc-editor.org/rfc/rfc8949#section-3.1-2.8 + * @see https://www.rfc-editor.org/rfc/rfc8949#section-5.3.1-2.4 + */ + public function testBuildWithInputEncoding() + { + // not setting an input encoding is fine + $this->builder->build($this->query); + + $this->query->setInputEncoding('utf-8'); + + // setting UTF-8 input encoding explicitly is fine (but superfluous) + $this->builder->build($this->query); + + $this->query->setInputEncoding('us-ascii'); + + // setting a different input encoding is prohibited + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('CBOR requests can only contain UTF-8 strings'); + $this->builder->build($this->query); + } + + /** + * @dataProvider unsupportedCommandProvider + */ + public function testBuildWithUnsupportedCommandType(AbstractCommand $command) + { + $this->query->add(null, $command); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Unsupported command type, CBOR queries can only be used to add documents'); + $this->builder->build($this->query); + } + + public static function unsupportedCommandProvider(): array + { + return [ + [new CommitCommand()], + [new DeleteCommand()], + [new OptimizeCommand()], + [new RawXmlCommand()], + [new RollbackCommand()], + ]; + } + + public function testBuildAddCborNoParamsSingleDocument() + { + $command = new AddCommand(); + $command->addDocument(new Document(['id' => 1])); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1 + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithScalarValues() + { + $command = new AddCommand(); + $command->addDocument(new Document([ + 'id' => 1, + 'noid' => -5, + 'name' => 'test', + 'price' => 3.14, + 'discount' => -2.72, + 'visible' => true, + 'forsale' => false, + 'UTF8' => 'ΑΒΓαβγ АБВабв أبجد אבג カタカナ 漢字', + ])); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1, + "noid": -5, + "name": "test", + "price": 3.14, + "discount": -2.72, + "visible": true, + "forsale": false, + "UTF8": "\u0391\u0392\u0393\u03b1\u03b2\u03b3 \u0410\u0411\u0412\u0430\u0431\u0432 \u0623\u0628\u062c\u062f \u05d0\u05d1\u05d2 \u30ab\u30bf\u30ab\u30ca \u6f22\u5b57" + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithEmptyValues() + { + $command = new AddCommand(); + $command->addDocument(new Document(['id' => 0, 'empty_string' => '', 'empty_array' => [], 'array_of_empty_string' => [''], 'null' => null])); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + // Empty strings must be added to the document as empty fields. + // Empty arrays and NULL values can be (but don't have to be) skipped because Solr ignores them anyway. + $this->assertEquals( + json_decode('[ + { + "id": 0, + "empty_string": "", + "empty_array": [], + "array_of_empty_string": [""] + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithParams() + { + $command = new AddCommand(['overwrite' => true, 'commitwithin' => 100]); + $command->addDocument(new Document(['id' => 1])); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertSame('true', $request->getParam('overwrite')); + $this->assertSame(100, $request->getParam('commitWithin')); + $this->assertEquals( + json_decode('[ + { + "id": 1 + } + ]', true), + $object + ); + } + + public function testBuildAddCborMultipleCommandsWithDifferentParams() + { + $command = new AddCommand(['overwrite' => true, 'commitwithin' => 100]); + $command->addDocument(new Document(['id' => 1])); + $this->query->add(null, $command); + $command = new AddCommand(['overwrite' => false, 'commitwithin' => 500]); + $command->addDocument(new Document(['id' => 2])); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + // last occurrence determines which params will be sent + $this->assertSame('false', $request->getParam('overwrite')); + $this->assertSame(500, $request->getParam('commitWithin')); + $this->assertEquals( + json_decode('[ + { + "id": 1 + }, + { + "id": 2 + } + ]', true), + $object + ); + } + + public function testBuildAddCborMultipleCommandsWithAndWithoutParams() + { + $command = new AddCommand(['overwrite' => true, 'commitwithin' => 100]); + $command->addDocument(new Document(['id' => 1])); + $this->query->add(null, $command); + $command = new AddCommand(); + $command->addDocument(new Document(['id' => 2])); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + // first occurrence is kept if no further params are set + $this->assertSame('true', $request->getParam('overwrite')); + $this->assertSame(100, $request->getParam('commitWithin')); + $this->assertEquals( + json_decode('[ + { + "id": 1 + }, + { + "id": 2 + } + ]', true), + $object + ); + } + + public function testBuildAddCborMultivalueField() + { + $command = new AddCommand(); + $command->addDocument(new Document(['id' => [1, 2, 3], 'text' => ['test < 123 '.chr(8).' test', 'test '.chr(15).' 123 > test']])); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": [1, 2, 3], + "text": ["test < 123 \b test", "test \u000f 123 > test"] + } + ]', true), + $object + ); + } + + public function testBuildAddCborMultivalueFieldWithEmptyArray() + { + $command = new AddCommand(); + $command->addDocument(new Document(['id' => [1, 2, 3], 'text' => []])); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": [1, 2, 3], + "text": [] + } + ]', true), + $object + ); + } + + public function testBuildAddCborMultivalueFieldWithNonConsecutiveArrayIndices() + { + $command = new AddCommand(); + $command->addDocument(new Document(['id' => [0 => 1, 4 => 2, 6 => 3], 'text' => [1 => 'a', 2 => 'b', 3 => 'c']])); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": [1, 2, 3], + "text": ["a", "b", "c"] + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithEmptyStrings() + { + $command = new AddCommand(); + $command->addDocument(new Document(['id' => '', 'text' => ['']])); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": "", + "text": [""] + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithSingleNestedDocument() + { + $command = new AddCommand(); + $command->addDocument( + new Document( + [ + 'id' => [ + 'nested_id' => 42, + 'customer_ids' => [ + 15, + 16, + ], + ], + 'text' => 'test < 123 > test', + ] + ) + ); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": { + "nested_id": 42, + "customer_ids": [15, 16] + }, + "text": "test < 123 > test" + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithNestedDocuments() + { + $command = new AddCommand(); + $command->addDocument( + new Document( + [ + 'id' => [ + [ + 'nested_id' => 42, + 'customer_ids' => [ + 15, + 16, + ], + ], + [ + 'nested_id' => 'XLII', + 'customer_ids' => [ + 17, + 18, + ], + ], + 2, + 'foo', + ], + 'text' => 'test < 123 > test', + ] + ) + ); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": [ + { + "nested_id": 42, + "customer_ids": [15, 16] + }, + { + "nested_id": "XLII", + "customer_ids": [17, 18] + }, + 2, + "foo" + ], + "text": "test < 123 > test" + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithSingleAnonymouslyNestedDocument() + { + $command = new AddCommand(); + $command->addDocument( + new Document( + [ + 'id' => 1701, + 'cat' => ['A', 'D'], + 'text' => ':=._,<^>', + '_childDocuments_' => [ + 'id' => '1701-D', + 'cat' => ['D'], + ], + ] + ) + ); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1701, + "cat": ["A", "D"], + "text": ":=._,<^>", + "_childDocuments_": { + "id": "1701-D", + "cat": ["D"] + } + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithAnonymouslyNestedDocuments() + { + $command = new AddCommand(); + $command->addDocument( + new Document( + [ + 'id' => 1701, + 'cat' => ['A', 'D'], + 'text' => ':=._,<^>', + '_childDocuments_' => [ + [ + 'id' => '1701-A', + 'cat' => ['A'], + ], + [ + 'id' => '1701-D', + 'cat' => ['D'], + ], + ], + ] + ) + ); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1701, + "cat": ["A", "D"], + "text": ":=._,<^>", + "_childDocuments_": [ + { + "id": "1701-A", + "cat": ["A"] + }, + { + "id": "1701-D", + "cat": ["D"] + } + ] + } + ]', true), + $object + ); + } + + /** + * Document boosts aren't supported in CBOR update requests. + * + * @deprecated No longer supported since Solr 7 + */ + public function testBuildAddCborSingleDocumentWithBoost() + { + $doc = new Document(['id' => 1]); + $doc->setBoost(2.5); + $command = new AddCommand(); + $command->addDocument($doc); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1 + } + ]', true), + $object + ); + } + + /** + * Field boosts aren't supported in CBOR update requests. + */ + public function testBuildAddCborSingleDocumentWithFieldBoost() + { + $doc = new Document(['id' => 1]); + $doc->setFieldBoost('id', 2.1); + $command = new AddCommand(); + $command->addDocument($doc); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1 + } + ]', true), + $object + ); + } + + public function testBuildAddCborMultipleDocuments() + { + $command = new AddCommand(); + $command->addDocument(new Document(['id' => 1])); + $command->addDocument(new Document(['id' => 2])); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1 + }, + { + "id": 2 + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithFieldModifiers() + { + $doc = new Document(); + $doc->setKey('id', 1); + $doc->addField('category', 123, null, Document::MODIFIER_ADD); + $doc->addField('name', 'test', 2.5, Document::MODIFIER_SET); + $doc->setField('skills', null, null, Document::MODIFIER_SET); + $doc->setField('parts', [], null, Document::MODIFIER_SET); + $doc->setField('stock', 2, null, Document::MODIFIER_INC); + + $command = new AddCommand(); + $command->addDocument($doc); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1, + "category": { "add": 123 }, + "name": { "set": "test" }, + "skills": { "set": null }, + "parts": { "set": [] }, + "stock": { "inc": 2 } + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithFieldModifiersAndMultivalueFields() + { + $doc = new Document(); + $doc->setKey('id', 1); + $doc->addField('category', 123, null, Document::MODIFIER_ADD); + $doc->addField('category', 234, null, Document::MODIFIER_ADD); + $doc->addField('name', 'test', 2.3, Document::MODIFIER_SET); + $doc->setField('stock', 2, null, Document::MODIFIER_INC); + + $command = new AddCommand(); + $command->addDocument($doc); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1, + "category": { "add": [123, 234] }, + "name": { "set": "test" }, + "stock": { "inc": 2 } + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithVersionedDocument() + { + $doc = new Document(['id' => 1]); + $doc->setVersion(42); + + $command = new AddCommand(); + $command->addDocument($doc); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1, + "_version_": 42 + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithVersionMustNotExist() + { + $doc = new Document(['id' => 1]); + $doc->setVersion(Document::VERSION_MUST_NOT_EXIST); + + $command = new AddCommand(); + $command->addDocument($doc); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1, + "_version_": -1 + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithDateTime() + { + $command = new AddCommand(); + $command->addDocument( + new Document(['id' => 1, 'datetime' => new \DateTime('2013-01-15 14:41:58', new \DateTimeZone('+02:00'))]) + ); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1, + "datetime": "2013-01-15T12:41:58Z" + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithDateTimeImmutable() + { + $command = new AddCommand(); + $command->addDocument( + new Document(['id' => 1, 'datetime' => new \DateTimeImmutable('2013-01-15 14:41:58', new \DateTimeZone('-06:00'))]) + ); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1, + "datetime": "2013-01-15T20:41:58Z" + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithMultivalueDateTimes() + { + $command = new AddCommand(); + $command->addDocument( + new Document(['id' => 1, 'datetime' => [new \DateTime('2013-01-15 14:41:58', new \DateTimeZone('-02:00')), new \DateTimeImmutable('2014-02-16 15:42:59', new \DateTimeZone('+06:00'))]]) + ); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "id": 1, + "datetime": [ + "2013-01-15T16:41:58Z", + "2014-02-16T09:42:59Z" + ] + } + ]', true), + $object + ); + } + + public function testBuildAddCborWithFieldModifierAndNullValue() + { + $doc = new Document(); + $doc->setKey('employeeId', '05991'); + $doc->addField('skills', null, null, Document::MODIFIER_SET); + + $command = new AddCommand(); + $command->addDocument($doc); + $this->query->add(null, $command); + + $request = $this->builder->build($this->query); + $rawData = $request->getRawData(); + $object = CBOREncoder::decode($rawData); + + $this->assertEquals( + json_decode('[ + { + "employeeId": "05991", + "skills": { "set": null } + } + ]', true), + $object + ); + } +} diff --git a/tests/QueryType/Update/RequestBuilder/JsonTest.php b/tests/QueryType/Update/RequestBuilder/JsonTest.php index 1b5ed98cd..9704590fa 100644 --- a/tests/QueryType/Update/RequestBuilder/JsonTest.php +++ b/tests/QueryType/Update/RequestBuilder/JsonTest.php @@ -115,10 +115,19 @@ public function testBuildAddJsonNoParamsSingleDocument() ); } - public function testBuildAddJsonWithBooleanValues() + public function testBuildAddJsonWithScalarValues() { $command = new AddCommand(); - $command->addDocument(new Document(['id' => 1, 'visible' => true, 'forsale' => false])); + $command->addDocument(new Document([ + 'id' => 1, + 'noid' => -5, + 'name' => 'test', + 'price' => 3.14, + 'discount' => -2.72, + 'visible' => true, + 'forsale' => false, + 'UTF8' => 'ΑΒΓαβγ АБВабв أبجد אבג カタカナ 漢字', + ])); $json = []; $this->builder->buildAddJson($command, $json); @@ -129,8 +138,13 @@ public function testBuildAddJsonWithBooleanValues() "add": { "doc": { "id": 1, + "noid": -5, + "name": "test", + "price": 3.14, + "discount": -2.72, "visible": true, - "forsale": false + "forsale": false, + "UTF8": "\u0391\u0392\u0393\u03b1\u03b2\u03b3 \u0410\u0411\u0412\u0430\u0431\u0432 \u0623\u0628\u062c\u062f \u05d0\u05d1\u05d2 \u30ab\u30bf\u30ab\u30ca \u6f22\u5b57" } } }', @@ -308,7 +322,7 @@ public function testBuildAddJsonWithSingleNestedDocument() "text": "test < 123 > test" } } - }', + }', '{'.$json[0].'}' ); } @@ -612,6 +626,31 @@ public function testBuildAddJsonWithFieldModifiersAndMultivalueFields() } public function testBuildAddJsonWithVersionedDocument() + { + $doc = new Document(['id' => 1]); + $doc->setVersion(42); + + $command = new AddCommand(); + $command->addDocument($doc); + $json = []; + + $this->builder->buildAddJson($command, $json); + + $this->assertCount(1, $json); + $this->assertJsonStringEqualsJsonString( + '{ + "add": { + "doc": { + "id": 1, + "_version_": 42 + } + } + }', + '{'.$json[0].'}' + ); + } + + public function testBuildAddJsonWithVersionMustNotExist() { $doc = new Document(['id' => 1]); $doc->setVersion(Document::VERSION_MUST_NOT_EXIST);