Skip to content

Y
yii2
Commit 676b3669 by Carsten Brandt
Options

adjusted elasticsearch extension to version 1.0 of ES

breaking changes.
parent 76b3d89d
Showing with 210 additions and 171 deletions
extensions/elasticsearch/ActiveQuery.php
......@@ -7,6 +7,7 @@
namespace yii\elasticsearch;
use yii\base\NotSupportedException;
use yii\db\ActiveQueryInterface;
use yii\db\ActiveQueryTrait;
use yii\db\ActiveRelationTrait;
......@@ -143,47 +144,21 @@ class ActiveQuery extends Query implements ActiveQueryInterface
*/
public function all($db = null)
{
if ($this->asArray) {
// TODO implement with
return parent::all($db);
}
$result = $this->createCommand($db)->search();
if (empty($result['hits']['hits'])) {
return [];
}
if ($this->fields !== null) {
foreach ($result['hits']['hits'] as &$row) {
$row['_source'] = isset($row['fields']) ? $row['fields'] : [];
unset($row['fields']);
}
unset($row);
}
/** @var ActiveRecord $modelClass */
$modelClass = $this->modelClass;
$pk = $modelClass::primaryKey()[0];
if ($this->asArray && $this->indexBy) {
foreach ($result['hits']['hits'] as &$row) {
if ($pk === '_id') {
$row['_source']['_id'] = $row['_id'];
}
$row['_source']['_score'] = $row['_score'];
$row = $row['_source'];
}
unset($row);
}
$models = $this->createModels($result['hits']['hits']);
if ($this->asArray && !$this->indexBy) {
foreach ($models as $key => $model) {
if ($pk === '_id') {
$model['_source']['_id'] = $model['_id'];
}
$model['_source']['_score'] = $model['_score'];
$models[$key] = $model['_source'];
}
}
if (!empty($this->with)) {
$this->findWith($this->with, $models);
}
if (!$this->asArray) {
foreach ($models as $model) {
$model->afterFind();
}
foreach ($models as $model) {
$model->afterFind();
}
return $models;
......@@ -203,30 +178,34 @@ class ActiveQuery extends Query implements ActiveQueryInterface
return null;
}
if ($this->asArray) {
/** @var ActiveRecord $modelClass */
$modelClass = $this->modelClass;
$model = $result['_source'];
$pk = $modelClass::primaryKey()[0];
if ($pk === '_id') {
$model['_id'] = $result['_id'];
}
$model['_score'] = $result['_score'];
// TODO implement with
// /** @var ActiveRecord $modelClass */
// $modelClass = $this->modelClass;
// $model = $result['_source'];
// $pk = $modelClass::primaryKey()[0];
// if ($pk === '_id') {
// $model['_id'] = $result['_id'];
// }
// $model['_score'] = $result['_score'];
// if (!empty($this->with)) {
// $models = [$model];
// $this->findWith($this->with, $models);
// $model = $models[0];
// }
return $result;
} else {
/** @var ActiveRecord $class */
$class = $this->modelClass;
$model = $class::instantiate($result);
$class::populateRecord($model, $result);
}
if (!empty($this->with)) {
$models = [$model];
$this->findWith($this->with, $models);
$model = $models[0];
}
if (!$this->asArray) {
if (!empty($this->with)) {
$models = [$model];
$this->findWith($this->with, $models);
$model = $models[0];
}
$model->afterFind();
return $model;
}
return $model;
}
/**
......@@ -235,27 +214,14 @@ class ActiveQuery extends Query implements ActiveQueryInterface
public function search($db = null, $options = [])
{
$result = $this->createCommand($db)->search($options);
if (!empty($result['hits']['hits'])) {
// TODO implement with for asArray
if (!empty($result['hits']['hits']) && !$this->asArray) {
$models = $this->createModels($result['hits']['hits']);
if ($this->asArray) {
/** @var ActiveRecord $modelClass */
$modelClass = $this->modelClass;
$pk = $modelClass::primaryKey()[0];
foreach ($models as $key => $model) {
if ($pk === '_id') {
$model['_source']['_id'] = $model['_id'];
}
$model['_source']['_score'] = $model['_score'];
$models[$key] = $model['_source'];
}
}
if (!empty($this->with)) {
$this->findWith($this->with, $models);
}
if (!$this->asArray) {
foreach ($models as $model) {
$model->afterFind();
}
foreach ($models as $model) {
$model->afterFind();
}
$result['hits']['hits'] = $models;
}
......@@ -266,28 +232,12 @@ class ActiveQuery extends Query implements ActiveQueryInterface
/**
* @inheritdoc
*/
public function scalar($field, $db = null)
{
$record = parent::one($db);
if ($record !== false) {
if ($field == '_id') {
return $record['_id'];
} elseif (isset($record['_source'][$field])) {
return $record['_source'][$field];
}
}
return null;
}
/**
* @inheritdoc
*/
public function column($field, $db = null)
{
if ($field == '_id') {
$command = $this->createCommand($db);
$command->queryParts['fields'] = [];
$command->queryParts['_source'] = false;
$result = $command->search();
if (empty($result['hits']['hits'])) {
return [];
......
extensions/elasticsearch/ActiveRecord.php
......@@ -114,7 +114,7 @@ class ActiveRecord extends BaseActiveRecord
}
$command = static::getDb()->createCommand();
$result = $command->get(static::index(), static::type(), $primaryKey, $options);
if ($result['exists']) {
if ($result['found']) {
$model = static::instantiate($result);
static::populateRecord($model, $result);
$model->afterFind();
......@@ -150,7 +150,7 @@ class ActiveRecord extends BaseActiveRecord
$result = $command->mget(static::index(), static::type(), $primaryKeys, $options);
$models = [];
foreach ($result['docs'] as $doc) {
if ($doc['exists']) {
if ($doc['found']) {
$model = static::instantiate($doc);
static::populateRecord($model, $doc);
$model->afterFind();
......@@ -282,8 +282,23 @@ class ActiveRecord extends BaseActiveRecord
*/
public static function populateRecord($record, $row)
{
parent::populateRecord($record, $row['_source']);
$pk = static::primaryKey()[0];
$attributes = [];
if (isset($row['_source'])) {
$attributes = $row['_source'];
}
if (isset($row['fields'])) {
// reset fields in case it is scalar value TODO use field metadata for this
foreach($row['fields'] as $key => $value) {
if (count($value) == 1) {
$row['fields'][$key] = reset($value);
}
}
$attributes = array_merge($attributes, $row['fields']);
}
parent::populateRecord($record, $attributes);
$pk = static::primaryKey()[0];//TODO should always set ID in case of fields are not returned
if ($pk === '_id') {
$record->_id = $row['_id'];
}
......@@ -379,9 +394,9 @@ class ActiveRecord extends BaseActiveRecord
$options
);
if (!isset($response['ok'])) {
return false;
}
// if (!isset($response['ok'])) {
// return false;
// }
$pk = static::primaryKey()[0];
$this->$pk = $response['_id'];
if ($pk != '_id') {
......@@ -444,13 +459,13 @@ class ActiveRecord extends BaseActiveRecord
$n = 0;
$errors = [];
foreach ($response['items'] as $item) {
if (isset($item['update']['error'])) {
$errors[] = $item['update'];
} elseif ($item['update']['ok']) {
if (isset($item['update']['status']) && $item['update']['status'] == 200) {
$n++;
} else {
$errors[] = $item['update'];
}
}
if (!empty($errors)) {
if (!empty($errors) || isset($response['errors']) && $response['errors']) {
throw new Exception(__METHOD__ . ' failed updating records.', $errors);
}
......@@ -508,13 +523,13 @@ class ActiveRecord extends BaseActiveRecord
$n = 0;
$errors = [];
foreach ($response['items'] as $item) {
if (isset($item['update']['error'])) {
$errors[] = $item['update'];
} elseif ($item['update']['ok']) {
if (isset($item['update']['status']) && $item['update']['status'] == 200) {
$n++;
} else {
$errors[] = $item['update'];
}
}
if (!empty($errors)) {
if (!empty($errors) || isset($response['errors']) && $response['errors']) {
throw new Exception(__METHOD__ . ' failed updating records counters.', $errors);
}
......@@ -563,13 +578,15 @@ class ActiveRecord extends BaseActiveRecord
$n = 0;
$errors = [];
foreach ($response['items'] as $item) {
if (isset($item['delete']['error'])) {
if (isset($item['delete']['status']) && $item['delete']['status'] == 200) {
if (isset($item['delete']['found']) && $item['delete']['found']) {
$n++;
}
} else {
$errors[] = $item['delete'];
} elseif ($item['delete']['found'] && $item['delete']['ok']) {
$n++;
}
}
if (!empty($errors)) {
if (!empty($errors) || isset($response['errors']) && $response['errors']) {
throw new Exception(__METHOD__ . ' failed deleting records.', $errors);
}
......
extensions/elasticsearch/CHANGELOG.md
......@@ -4,7 +4,9 @@ Yii Framework 2 elasticsearch extension Change Log
2.0.0-rc under development
--------------------------
- no changes in this release.
- Chg: asArray in ActiveQuery is now equal to using the normal Query. This means, that the output structure has changed and `with` is supported anymore. (cebe)
- Chg: Deletion of a record is now also considered successfull if the record did not exist. (cebe)
- Chg: Requirement changes: Yii now requires elasticsearch version 1.0 or higher (cebe)
2.0.0-beta April 13, 2014
......
extensions/elasticsearch/Command.php
......@@ -315,7 +315,7 @@ class Command extends Component
{
$body = $mapping !== null ? (is_string($mapping) ? $mapping : Json::encode($mapping)) : null;
return $this->db->put([$index, $type, '_mapping'], $options, $body);
return $this->db->put([$index, '_mapping', $type], $options, $body);
}
/**
......@@ -326,7 +326,7 @@ class Command extends Component
*/
public function getMapping($index = '_all', $type = '_all')
{
return $this->db->get([$index, $type, '_mapping']);
return $this->db->get([$index, '_mapping', $type]);
}
/**
......@@ -337,7 +337,7 @@ class Command extends Component
*/
public function deleteMapping($index, $type)
{
return $this->db->delete([$index, $type]);
return $this->db->delete([$index, '_mapping', $type]);
}
/**
......@@ -346,10 +346,11 @@ class Command extends Component
* @return mixed
* @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-get-field-mapping.html
*/
public function getFieldMapping($index, $type = '_all')
{
return $this->db->put([$index, $type, '_mapping']);
}
// public function getFieldMapping($index, $type = '_all')
// {
// // TODO implement
// return $this->db->put([$index, $type, '_mapping']);
// }
/**
* @param $options
......
extensions/elasticsearch/Connection.php
......@@ -109,7 +109,7 @@ class Connection extends Component
if (strncmp($host, 'inet[/', 6) == 0) {
$host = substr($host, 6, -1);
}
$response = $this->httpRequest('GET', 'http://' . $host . '/_cluster/nodes');
$response = $this->httpRequest('GET', 'http://' . $host . '/_nodes');
$this->nodes = $response['nodes'];
if (empty($this->nodes)) {
throw new Exception('cluster autodetection did not find any active node.');
......
extensions/elasticsearch/Query.php
......@@ -58,13 +58,52 @@ class Query extends Component implements QueryInterface
/**
* @var array the fields being retrieved from the documents. For example, `['id', 'name']`.
* If not set, it means retrieving all fields. An empty array will result in no fields being
* retrieved. This means that only the primaryKey of a record will be available in the result.
* If not set, this option will not be applied to the query and no fields will be returned.
* In this case the `_source` field will be returned by default which can be configured using [[source]].
* Setting this to an empty array will result in no fields being retrieved, which means that only the primaryKey
* of a record will be available in the result.
*
* For each field you may also add an array representing a [script field]. Example:
*
* ```php
* $query->fields = [
* 'id',
* 'name',
* 'value_times_two' => [
* 'script' => "doc['my_field_name'].value * 2",
* ],
* 'value_times_factor' => [
* 'script' => "doc['my_field_name'].value * factor",
* 'params' => [
* 'factor' => 2.0
* ],
* ],
* ]
* ```
*
* > Note: Field values are [always returned as arrays] even if they only have one value.
*
* [always returned as arrays]: http://www.elasticsearch.org/guide/en/elasticsearch/reference/1.x/_return_values.html#_return_values
* [script field]: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-script-fields.html
*
* @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-fields.html#search-request-fields
* @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-script-fields.html
* @see fields()
* @see source
*/
public $fields;
/**
* @var array this option controls how the `_source` field is returned from the documents. For example, `['id', 'name']`
* means that only the `id` and `name` field should be returned from `_source`.
* If not set, it means retrieving the full `_source` field unless [[fields]] are specified.
* Setting this option to `false` will disable return of the `_source` field, this means that only the primaryKey
* of a record will be available in the result.
* @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-source-filtering.html
* @see source()
* @see fields
*/
public $source;
/**
* @var string|array The index to retrieve data from. This can be a string representing a single index
* or a an array of multiple indexes. If this is not set, indexes are being queried.
* @see from()
......@@ -137,25 +176,20 @@ class Query extends Component implements QueryInterface
return [];
}
$rows = $result['hits']['hits'];
if ($this->indexBy === null && $this->fields === null) {
if ($this->indexBy === null) {
return $rows;
}
$models = [];
foreach ($rows as $key => $row) {
if ($this->fields !== null) {
$row['_source'] = isset($row['fields']) ? $row['fields'] : [];
unset($row['fields']);
}
if ($this->indexBy !== null) {
if (is_string($this->indexBy)) {
$key = $row['_source'][$this->indexBy];
$key = isset($row['fields'][$this->indexBy]) ? reset($row['fields'][$this->indexBy]) : $row['_source'][$this->indexBy];
} else {
$key = call_user_func($this->indexBy, $row);
}
}
$models[$key] = $row;
}
return $models;
}
......@@ -173,10 +207,6 @@ class Query extends Component implements QueryInterface
return false;
}
$record = reset($result['hits']['hits']);
if ($this->fields !== null) {
$record['_source'] = isset($record['fields']) ? $record['fields'] : [];
unset($record['fields']);
}
return $record;
}
......@@ -195,25 +225,18 @@ class Query extends Component implements QueryInterface
public function search($db = null, $options = [])
{
$result = $this->createCommand($db)->search($options);
if (!empty($result['hits']['hits']) && ($this->indexBy === null || $this->fields === null)) {
if (!empty($result['hits']['hits']) && $this->indexBy !== null) {
$rows = [];
foreach ($result['hits']['hits'] as $key => $row) {
if ($this->fields !== null) {
$row['_source'] = isset($row['fields']) ? $row['fields'] : [];
unset($row['fields']);
}
if ($this->indexBy !== null) {
if (is_string($this->indexBy)) {
$key = $row['_source'][$this->indexBy];
} else {
$key = call_user_func($this->indexBy, $row);
}
if (is_string($this->indexBy)) {
$key = isset($row['fields'][$this->indexBy]) ? $row['fields'][$this->indexBy] : $row['_source'][$this->indexBy];
} else {
$key = call_user_func($this->indexBy, $row);
}
$rows[$key] = $row;
}
$result['hits']['hits'] = $rows;
}
return $result;
}
......@@ -247,12 +270,17 @@ class Query extends Component implements QueryInterface
*/
public function scalar($field, $db = null)
{
$record = self::one($db); // TODO limit fields to the one required
if ($record !== false && isset($record['_source'][$field])) {
return $record['_source'][$field];
} else {
return null;
$record = self::one($db);
if ($record !== false) {
if ($field === '_id') {
return $record['_id'];
} elseif (isset($record['_source'][$field])) {
return $record['_source'][$field];
} elseif (isset($record['fields'][$field])) {
return count($record['fields'][$field]) == 1 ? reset($record['fields'][$field]) : $record['fields'][$field];
}
}
return null;
}
/**
......@@ -265,14 +293,14 @@ class Query extends Component implements QueryInterface
public function column($field, $db = null)
{
$command = $this->createCommand($db);
$command->queryParts['fields'] = [$field];
$command->queryParts['_source'] = [$field];
$result = $command->search();
if (empty($result['hits']['hits'])) {
return [];
}
$column = [];
foreach ($result['hits']['hits'] as $row) {
$column[] = isset($row['fields'][$field]) ? $row['fields'][$field] : null;
$column[] = isset($row['_source'][$field]) ? $row['_source'][$field] : null;
}
return $column;
}
......@@ -498,6 +526,22 @@ class Query extends Component implements QueryInterface
}
/**
* Sets the source filtering, specifying how the `_source` field of the document should be returned.
* @param array $source the source patterns to be selected.
* @return static the query object itself
* @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-source-filtering.html
*/
public function source($source)
{
if (is_array($source) || $source === null) {
$this->source = $source;
} else {
$this->source = func_get_args();
}
return $this;
}
/**
* Sets the search timeout.
* @param integer $timeout A search timeout, bounding the search request to be executed within the specified time value
* and bail with the hits accumulated up to that point when expired. Defaults to no timeout.
......
extensions/elasticsearch/QueryBuilder.php
......@@ -45,8 +45,27 @@ class QueryBuilder extends \yii\base\Object
{
$parts = [];
if ($query->fields !== null) {
$parts['fields'] = (array) $query->fields;
if ($query->fields === []) {
$parts['fields'] = [];
} elseif ($query->fields !== null) {
$fields = [];
$scriptFields = [];
foreach($query->fields as $key => $field) {
if (is_int($key)) {
$fields[] = $field;
} else {
$scriptFields[$key] = $field;
}
}
if (!empty($fields)) {
$parts['fields'] = $fields;
}
if (!empty($scriptFields)) {
$parts['script_fields'] = $scriptFields;
}
}
if ($query->source !== null) {
$parts['_source'] = $query->source;
}
if ($query->limit !== null && $query->limit >= 0) {
$parts['size'] = $query->limit;
......
extensions/elasticsearch/README.md
......@@ -22,6 +22,10 @@ return [
];
```
Requirements
------------
elasticsearch version 1.0 or higher is required.
Installation
------------
......
tests/unit/data/ar/elasticsearch/OrderItem.php
......@@ -14,6 +14,8 @@ use yii\elasticsearch\Command;
*/
class OrderItem extends ActiveRecord
{
public $total;
public function attributes()
{
return ['order_id', 'item_id', 'quantity', 'subtotal'];
......
tests/unit/extensions/elasticsearch/ElasticSearchConnectionTest.php
......@@ -20,7 +20,7 @@ class ElasticSearchConnectionTest extends ElasticSearchTestCase
$connection->open();
$this->assertNotNull($connection->activeNode);
$this->assertArrayHasKey('name', reset($connection->nodes));
$this->assertArrayHasKey('hostname', reset($connection->nodes));
// $this->assertArrayHasKey('hostname', reset($connection->nodes));
$this->assertArrayHasKey('version', reset($connection->nodes));
$this->assertArrayHasKey('http_address', reset($connection->nodes));
}
......
tests/unit/extensions/elasticsearch/QueryBuilderTest.php
......@@ -49,7 +49,7 @@ class QueryBuilderTest extends ElasticSearchTestCase
public function testYiiCanBeFoundByQuery()
{
$this->prepareDbData();
$queryParts = ['field' => ['title' => 'yii']];
$queryParts = ['term' => ['title' => 'yii']];
$query = new Query();
$query->from('yiitest', 'article');
$query->query = $queryParts;
......
tests/unit/extensions/elasticsearch/QueryTest.php
......@@ -40,16 +40,16 @@ class QueryTest extends ElasticSearchTestCase
$this->assertEquals(['name', 'status'], $query->fields);
$result = $query->one($this->getConnection());
$this->assertEquals(2, count($result['_source']));
$this->assertArrayHasKey('status', $result['_source']);
$this->assertArrayHasKey('name', $result['_source']);
$this->assertEquals(2, count($result['fields']));
$this->assertArrayHasKey('status', $result['fields']);
$this->assertArrayHasKey('name', $result['fields']);
$this->assertArrayHasKey('_id', $result);
$query->fields([]);
$this->assertEquals([], $query->fields);
$result = $query->one($this->getConnection());
$this->assertEquals([], $result['_source']);
$this->assertArrayNotHasKey('fields', $result);
$this->assertArrayHasKey('_id', $result);
$query->fields(null);
......
tests/unit/framework/ar/ActiveRecordTestTrait.php
......@@ -70,25 +70,6 @@ trait ActiveRecordTestTrait
$this->assertTrue($customers[1] instanceof $customerClass);
$this->assertTrue($customers[2] instanceof $customerClass);
// find all asArray
$customers = $customerClass::find()->asArray()->all();
$this->assertEquals(3, count($customers));
$this->assertArrayHasKey('id', $customers[0]);
$this->assertArrayHasKey('name', $customers[0]);
$this->assertArrayHasKey('email', $customers[0]);
$this->assertArrayHasKey('address', $customers[0]);
$this->assertArrayHasKey('status', $customers[0]);
$this->assertArrayHasKey('id', $customers[1]);
$this->assertArrayHasKey('name', $customers[1]);
$this->assertArrayHasKey('email', $customers[1]);
$this->assertArrayHasKey('address', $customers[1]);
$this->assertArrayHasKey('status', $customers[1]);
$this->assertArrayHasKey('id', $customers[2]);
$this->assertArrayHasKey('name', $customers[2]);
$this->assertArrayHasKey('email', $customers[2]);
$this->assertArrayHasKey('address', $customers[2]);
$this->assertArrayHasKey('status', $customers[2]);
// find by a single primary key
$customer = $customerClass::findOne(2);
$this->assertTrue($customer instanceof $customerClass);
......@@ -136,6 +117,25 @@ trait ActiveRecordTestTrait
'status' => 1,
'profile_id' => null,
], $customer);
// find all asArray
$customers = $customerClass::find()->asArray()->all();
$this->assertEquals(3, count($customers));
$this->assertArrayHasKey('id', $customers[0]);
$this->assertArrayHasKey('name', $customers[0]);
$this->assertArrayHasKey('email', $customers[0]);
$this->assertArrayHasKey('address', $customers[0]);
$this->assertArrayHasKey('status', $customers[0]);
$this->assertArrayHasKey('id', $customers[1]);
$this->assertArrayHasKey('name', $customers[1]);
$this->assertArrayHasKey('email', $customers[1]);
$this->assertArrayHasKey('address', $customers[1]);
$this->assertArrayHasKey('status', $customers[1]);
$this->assertArrayHasKey('id', $customers[2]);
$this->assertArrayHasKey('name', $customers[2]);
$this->assertArrayHasKey('email', $customers[2]);
$this->assertArrayHasKey('address', $customers[2]);
$this->assertArrayHasKey('status', $customers[2]);
}
public function testFindScalar()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment