Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[5.3] Avoid extraneous database query when last chunk is partial #16180

Merged
merged 5 commits into from
Oct 31, 2016
Merged

[5.3] Avoid extraneous database query when last chunk is partial #16180

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
325c61f
Avoid extraneous database query when last chunk is partial
vlakoff Oct 30, 2016
02f2dcb
Alternative code for chunk() methods
vlakoff Oct 30, 2016
4d77f80
Compare directly against the Collections in chunk() tests
vlakoff Oct 30, 2016
148564c
Add test when last chunk is complete
vlakoff Oct 30, 2016
08e514c
Avoid extraneous query in chunkById() methods as well
vlakoff Oct 30, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 19 additions & 11 deletions src/Illuminate/Database/Eloquent/Builder.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,9 +371,16 @@ public function cursor()
*/
public function chunk($count, callable $callback)
{
$results = $this->forPage($page = 1, $count)->get();
$page = 1;

do {
$results = $this->forPage($page, $count)->get();
$countResults = $results->count();

if ($countResults == 0) {
break;
}

while (! $results->isEmpty()) {
// On each chunk result set, we will pass them to the callback and then let the
// developer take care of everything within the callback, which allows us to
// keep the memory low for spinning through large result sets for working.
Expand All @@ -382,9 +389,7 @@ public function chunk($count, callable $callback)
}

$page++;

$results = $this->forPage($page, $count)->get();
}
} while ($countResults == $count);

return true;
}
Expand All @@ -399,19 +404,22 @@ public function chunk($count, callable $callback)
*/
public function chunkById($count, callable $callback, $column = 'id')
{
$lastId = null;
$lastId = 0;

$results = $this->forPageAfterId($count, 0, $column)->get();
do {
$results = $this->forPageAfterId($count, $lastId, $column)->get();
$countResults = $results->count();

if ($countResults == 0) {
break;
}

while (! $results->isEmpty()) {
if (call_user_func($callback, $results) === false) {
return false;
}

$lastId = $results->last()->{$column};

$results = $this->forPageAfterId($count, $lastId, $column)->get();
}
} while ($countResults == $count);

return true;
}
Expand Down
32 changes: 20 additions & 12 deletions src/Illuminate/Database/Query/Builder.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1807,9 +1807,16 @@ public function cursor()
*/
public function chunk($count, callable $callback)
{
$results = $this->forPage($page = 1, $count)->get();
$page = 1;

do {
$results = $this->forPage($page, $count)->get();
$countResults = $results->count();

if ($countResults == 0) {
break;
}

while (! $results->isEmpty()) {
// On each chunk result set, we will pass them to the callback and then let the
// developer take care of everything within the callback, which allows us to
// keep the memory low for spinning through large result sets for working.
Expand All @@ -1818,9 +1825,7 @@ public function chunk($count, callable $callback)
}

$page++;

$results = $this->forPage($page, $count)->get();
}
} while ($countResults == $count);

return true;
}
Expand All @@ -1836,21 +1841,24 @@ public function chunk($count, callable $callback)
*/
public function chunkById($count, callable $callback, $column = 'id', $alias = null)
{
$lastId = null;

$alias = $alias ?: $column;

$results = $this->forPageAfterId($count, 0, $column)->get();
$lastId = 0;

do {
$results = $this->forPageAfterId($count, $lastId, $column)->get();
$countResults = $results->count();

if ($countResults == 0) {
break;
}

while (! $results->isEmpty()) {
if (call_user_func($callback, $results) === false) {
return false;
}

$lastId = $results->last()->{$alias};

$results = $this->forPageAfterId($count, $lastId, $column)->get();
}
} while ($countResults == $count);

return true;
}
Expand Down
69 changes: 51 additions & 18 deletions tests/Database/DatabaseEloquentBuilderTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,64 +154,97 @@ public function testValueMethodWithModelNotFound()
$this->assertNull($builder->value('name'));
}

public function testChunkExecuteCallbackOverPaginatedRequest()
public function testChunkWithLastChunkComplete()
{
$builder = m::mock('Illuminate\Database\Eloquent\Builder[forPage,get]', [$this->getMockQueryBuilder()]);
$chunk1 = new Collection(['foo1', 'foo2']);
$chunk2 = new Collection(['foo3', 'foo4']);
$chunk3 = new Collection([]);
$builder->shouldReceive('forPage')->once()->with(1, 2)->andReturn($builder);
$builder->shouldReceive('forPage')->once()->with(2, 2)->andReturn($builder);
$builder->shouldReceive('forPage')->once()->with(3, 2)->andReturn($builder);
$builder->shouldReceive('get')->times(3)->andReturn(new Collection(['foo1', 'foo2']), new Collection(['foo3']), new Collection([]));
$builder->shouldReceive('get')->times(3)->andReturn($chunk1, $chunk2, $chunk3);

$callbackExecutionAssertor = m::mock('StdClass');
$callbackExecutionAssertor->shouldReceive('doSomething')->with('foo1')->once();
$callbackExecutionAssertor->shouldReceive('doSomething')->with('foo2')->once();
$callbackExecutionAssertor->shouldReceive('doSomething')->with('foo3')->once();
$callbackExecutionAssertor->shouldReceive('doSomething')->with($chunk1)->once();
$callbackExecutionAssertor->shouldReceive('doSomething')->with($chunk2)->once();
$callbackExecutionAssertor->shouldReceive('doSomething')->with($chunk3)->never();

$builder->chunk(2, function ($results) use ($callbackExecutionAssertor) {
foreach ($results as $result) {
$callbackExecutionAssertor->doSomething($result);
}
$callbackExecutionAssertor->doSomething($results);
});
}

public function testChunkWithLastChunkPartial()
{
$builder = m::mock('Illuminate\Database\Eloquent\Builder[forPage,get]', [$this->getMockQueryBuilder()]);
$chunk1 = new Collection(['foo1', 'foo2']);
$chunk2 = new Collection(['foo3']);
$builder->shouldReceive('forPage')->once()->with(1, 2)->andReturn($builder);
$builder->shouldReceive('forPage')->once()->with(2, 2)->andReturn($builder);
$builder->shouldReceive('get')->times(2)->andReturn($chunk1, $chunk2);

$callbackExecutionAssertor = m::mock('StdClass');
$callbackExecutionAssertor->shouldReceive('doSomething')->with($chunk1)->once();
$callbackExecutionAssertor->shouldReceive('doSomething')->with($chunk2)->once();

$builder->chunk(2, function ($results) use ($callbackExecutionAssertor) {
$callbackExecutionAssertor->doSomething($results);
});
}

public function testChunkCanBeStoppedByReturningFalse()
{
$builder = m::mock('Illuminate\Database\Eloquent\Builder[forPage,get]', [$this->getMockQueryBuilder()]);
$chunk1 = new Collection(['foo1', 'foo2']);
$chunk2 = new Collection(['foo3']);
$builder->shouldReceive('forPage')->once()->with(1, 2)->andReturn($builder);
$builder->shouldReceive('forPage')->never()->with(2, 2);
$builder->shouldReceive('get')->times(1)->andReturn(new Collection(['foo1', 'foo2']));
$builder->shouldReceive('get')->times(1)->andReturn($chunk1);

$callbackExecutionAssertor = m::mock('StdClass');
$callbackExecutionAssertor->shouldReceive('doSomething')->with('foo1')->once();
$callbackExecutionAssertor->shouldReceive('doSomething')->with('foo2')->once();
$callbackExecutionAssertor->shouldReceive('doSomething')->with('foo3')->never();
$callbackExecutionAssertor->shouldReceive('doSomething')->with($chunk1)->once();
$callbackExecutionAssertor->shouldReceive('doSomething')->with($chunk2)->never();

$builder->chunk(2, function ($results) use ($callbackExecutionAssertor) {
foreach ($results as $result) {
$callbackExecutionAssertor->doSomething($result);
}
$callbackExecutionAssertor->doSomething($results);

return false;
});
}

public function testChunkPaginatesUsingId()
public function testChunkPaginatesUsingIdWithLastChunkComplete()
{
$builder = m::mock('Illuminate\Database\Eloquent\Builder[forPageAfterId,get]', [$this->getMockQueryBuilder()]);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturn($builder);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 2, 'someIdField')->andReturn($builder);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 10, 'someIdField')->andReturn($builder);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 11, 'someIdField')->andReturn($builder);

$builder->shouldReceive('get')->times(3)->andReturn(
new Collection([(object) ['someIdField' => 1], (object) ['someIdField' => 2]]),
new Collection([(object) ['someIdField' => 10]]),
new Collection([(object) ['someIdField' => 10], (object) ['someIdField' => 11]]),
new Collection([])
);

$builder->chunkById(2, function ($results) {
}, 'someIdField');
}

public function testChunkPaginatesUsingIdWithLastChunkPartial()
{
$builder = m::mock('Illuminate\Database\Eloquent\Builder[forPageAfterId,get]', [$this->getMockQueryBuilder()]);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturn($builder);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 2, 'someIdField')->andReturn($builder);

$builder->shouldReceive('get')->times(2)->andReturn(
new Collection([(object) ['someIdField' => 1], (object) ['someIdField' => 2]]),
new Collection([(object) ['someIdField' => 10]])
);

$builder->chunkById(2, function ($results) {
}, 'someIdField');
}

public function testPluckReturnsTheMutatedAttributesOfAModel()
{
$builder = $this->getBuilder();
Expand Down
22 changes: 19 additions & 3 deletions tests/Database/DatabaseQueryBuilderTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1734,24 +1734,40 @@ public function testTableValuedFunctionAsTableInSqlServer()
$this->assertEquals('select * from [users](1,2)', $builder->toSql());
}

public function testChunkPaginatesUsingId()
public function testChunkPaginatesUsingIdWithLastChunkComplete()
{
$builder = $this->getMockQueryBuilder();

$builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturn($builder);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 2, 'someIdField')->andReturn($builder);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 10, 'someIdField')->andReturn($builder);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 11, 'someIdField')->andReturn($builder);

$builder->shouldReceive('get')->times(3)->andReturn(
collect([(object) ['someIdField' => 1], (object) ['someIdField' => 2]]),
collect([(object) ['someIdField' => 10]]),
collect([(object) ['someIdField' => 10], (object) ['someIdField' => 11]]),
collect([])
);

$builder->chunkById(2, function ($results) {
}, 'someIdField');
}

public function testChunkPaginatesUsingIdWithLastChunkPartial()
{
$builder = $this->getMockQueryBuilder();

$builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturn($builder);
$builder->shouldReceive('forPageAfterId')->once()->with(2, 2, 'someIdField')->andReturn($builder);

$builder->shouldReceive('get')->times(2)->andReturn(
collect([(object) ['someIdField' => 1], (object) ['someIdField' => 2]]),
collect([(object) ['someIdField' => 10]])
);

$builder->chunkById(2, function ($results) {
}, 'someIdField');
}

public function testChunkPaginatesUsingIdWithAlias()
{
$builder = $this->getMockQueryBuilder();
Expand Down