From d104c31a93bf3c04454f16402c93c9f094eb25f8 Mon Sep 17 00:00:00 2001 From: davidbalbino Date: Thu, 19 Nov 2026 09:50:57 -0300 Subject: [PATCH 1/2] Page reads in one statement, cached insert plans for datetimes, inline datetime hydration - Select::fetchPage(limit, offset): the page and the COUNT of the whole query in one statement (the count is a scalar subquery column of the page query), through the cached shape plan; past the last row or for customized selects it falls back to count() plus fetchAll(). Shape plans map a value to every parameter it lands in. - SelectQuery::countQuery() is the query count() runs. - CompilerCache hashed a SelectQuery column through __toString(): its values were interpolated into the cache key and its parameters were never bound. - StatementPlans accept DateTimeInterface values (the driver formats them when binding), so inserts and updates with timestamps reuse the cached statement. - Driver::getTimezone() keeps one DateTimeZone per configured name. - DirectInsert keeps role descriptions per ORM instead of per unit of work. - Compiled row makers construct datetime fields inline in the driver timezone. --- src/Database/Driver/CompilerCache.php | 5 ++ src/Database/Driver/Driver.php | 10 ++- src/Database/Query/SelectQuery.php | 22 +++-- src/ORM/Command/Database/StatementPlans.php | 9 ++- src/ORM/Parser/Typecast.php | 35 ++++++-- src/ORM/Select.php | 81 ++++++++++++++++--- .../Service/Implementation/EntityFactory.php | 17 +++- src/ORM/Transaction/DirectInsert.php | 25 +++++- .../Driver/Common/Query/SelectQueryTest.php | 22 +++++ .../Driver/Common/Select/PaginateTest.php | 47 +++++++++++ 10 files changed, 239 insertions(+), 34 deletions(-) diff --git a/src/Database/Driver/CompilerCache.php b/src/Database/Driver/CompilerCache.php index 8bda3ad..07328be 100644 --- a/src/Database/Driver/CompilerCache.php +++ b/src/Database/Driver/CompilerCache.php @@ -378,6 +378,11 @@ protected function hashColumns(QueryParameters $params, array $columns): string { $hash = ''; foreach ($columns as $column) { + if ($column instanceof SelectQuery) { + // A scalar subquery column: its parameters, not its interpolated values. + $hash .= 'sq_' . ($column->getPrefix() ?? '') . '(' . $this->hashSelectQuery($params, $column->getTokens()) . '),'; + continue; + } if ($column instanceof Expression || $column instanceof Fragment || $column instanceof SubQuery) { foreach ($column->getTokens()['parameters'] as $param) { $params->push($param); diff --git a/src/Database/Driver/Driver.php b/src/Database/Driver/Driver.php index ceb0f9a..1b3e7d0 100644 --- a/src/Database/Driver/Driver.php +++ b/src/Database/Driver/Driver.php @@ -52,6 +52,8 @@ abstract class Driver implements DriverInterface, NamedInterface, LoggerAwareInt protected array $queryCache = []; private ?string $name = null; + private ?\DateTimeZone $timezone = null; + private ?string $timezoneName = null; private bool $useCache = true; protected function __construct( @@ -132,7 +134,13 @@ public function getSource(): string public function getTimezone(): \DateTimeZone { - return new \DateTimeZone($this->config->timezone); + // DateTimeZone has no mutators: one instance per configured name. + if ($this->timezone === null || $this->timezoneName !== $this->config->timezone) { + $this->timezone = new \DateTimeZone($this->config->timezone); + $this->timezoneName = $this->config->timezone; + } + + return $this->timezone; } public function getSchemaHandler(): HandlerInterface diff --git a/src/Database/Query/SelectQuery.php b/src/Database/Query/SelectQuery.php index 3d80317..a0af9a6 100644 --- a/src/Database/Query/SelectQuery.php +++ b/src/Database/Query/SelectQuery.php @@ -330,13 +330,11 @@ public function runChunks(int $limit, callable $callback): void } /** - * Count number of rows in query. Limit, offset, order by, group by values will be ignored. - * - * @param non-empty-string $column Column to count by (every column by default). + * The query {@see count()} runs: a copy selecting only the count, without order or grouping. * - * @return int<0, max> + * @param non-empty-string $column */ - public function count(string $column = '*', bool $distinct = false): int + public function countQuery(string $column = '*', bool $distinct = false): self { $select = clone $this; @@ -347,7 +345,19 @@ public function count(string $column = '*', bool $distinct = false): int $select->orderBy = []; $select->groupBy = []; - $st = $select->run(); + return $select; + } + + /** + * Count number of rows in query. Limit, offset, order by, group by values will be ignored. + * + * @param non-empty-string $column Column to count by (every column by default). + * + * @return int<0, max> + */ + public function count(string $column = '*', bool $distinct = false): int + { + $st = $this->countQuery($column, $distinct)->run(); try { return (int) $st->fetchColumn(); } finally { diff --git a/src/ORM/Command/Database/StatementPlans.php b/src/ORM/Command/Database/StatementPlans.php index f702295..284910d 100644 --- a/src/ORM/Command/Database/StatementPlans.php +++ b/src/ORM/Command/Database/StatementPlans.php @@ -14,7 +14,8 @@ /** * Statements compiled once per table and column set, so repeated writes skip - * the query builder and the compiler. Only scalar and null values qualify. + * the query builder and the compiler. Only scalar, datetime and null values qualify + * (a datetime is bound as the driver formats it, like the query builder does). * * @internal */ @@ -41,7 +42,7 @@ public static function update(DatabaseInterface $db, string $table, array $value $key .= "{$column}\0n\1"; continue; } - if (!\is_scalar($value)) { + if (!\is_scalar($value) && !$value instanceof \DateTimeInterface) { return null; } $key .= "{$column}\1"; @@ -49,7 +50,7 @@ public static function update(DatabaseInterface $db, string $table, array $value } $key .= "\2"; foreach ($where as $column => $value) { - if (!\is_scalar($value)) { + if (!\is_scalar($value) && !$value instanceof \DateTimeInterface) { return null; } $key .= "{$column}\1"; @@ -108,7 +109,7 @@ public static function insert(DatabaseInterface $db, string $table, array $value $key .= "{$column}\0n\1"; continue; } - if (!\is_scalar($value)) { + if (!\is_scalar($value) && !$value instanceof \DateTimeInterface) { return [false, null]; } $key .= "{$column}\1"; diff --git a/src/ORM/Parser/Typecast.php b/src/ORM/Parser/Typecast.php index 2e3ef63..951978c 100644 --- a/src/ORM/Parser/Typecast.php +++ b/src/ORM/Parser/Typecast.php @@ -56,6 +56,11 @@ final class Typecast implements CastableInterface, UncastableInterface */ private array $plain = []; + /** @var array Fields cast by the datetime rule. */ + private array $datetimes = []; + + private ?\DateTimeZone $timezone = null; + /** * @param non-empty-string $role The role of the entity being typecasted * @param DatabaseInterface $database The database instance used for typecasting @@ -67,8 +72,8 @@ public function __construct( public function setRules(array $rules): array { - $timezone = null; foreach ($rules as $key => $rule) { + unset($this->datetimes[$key]); // Static rules if (\in_array($rule, self::RULES, true)) { $this->casters[$key] = match ($rule) { @@ -76,10 +81,7 @@ public function setRules(array $rules): array 'bool' => static fn(mixed $value): bool => (bool) $value, 'float' => static fn(mixed $value): float => (float) $value, // The driver timezone is fixed per connection: resolved once, not per value. - 'datetime' => function (mixed $value) use (&$timezone): \DateTimeImmutable { - $timezone ??= $this->database->getDriver()->getTimezone(); - return new \DateTimeImmutable($value, $timezone); - }, + 'datetime' => fn (mixed $value): \DateTimeImmutable => new \DateTimeImmutable($value, $this->timezone()), // Mostly an array, but can also be a scalar type. 'json' => static fn(mixed $value): mixed => \json_decode( $value, @@ -92,8 +94,10 @@ public function setRules(array $rules): array if ($rule === 'int' || $rule === 'bool' || $rule === 'float') { $this->plain[$key] = $rule; } elseif ($rule === 'datetime') { - // Database timestamps always parse: the maker calls the caster directly. + // Database timestamps always parse: compiled makers construct them inline + // in the driver timezone (see plainRules()). $this->plain[$key] = $this->casters[$key]; + $this->datetimes[$key] = true; } else { unset($this->plain[$key]); } @@ -175,7 +179,24 @@ public function setRules(array $rules): array */ public function plainRules(): ?array { - return \count($this->plain) === \count($this->casters) ? $this->plain : null; + if (\count($this->plain) !== \count($this->casters)) { + return null; + } + $rules = $this->plain; + foreach ($this->datetimes as $key => $_) { + // A datetime field: the zone its values are constructed in. + $rules[$key] = $this->timezone(); + } + + return $rules; + } + + /** + * The driver timezone is fixed per connection: resolved once, not per value. + */ + private function timezone(): \DateTimeZone + { + return $this->timezone ??= $this->database->getDriver()->getTimezone(); } public function cast(array $data): array diff --git a/src/ORM/Select.php b/src/ORM/Select.php index 165d444..fc68a33 100644 --- a/src/ORM/Select.php +++ b/src/ORM/Select.php @@ -622,14 +622,61 @@ public function orWhereJsonLength(string $path, int $length, string $operator = public function count(?string $column = null): int { $query = $this->buildQuery(); - if ($column === null) { - $pk = (array) $this->loader->getPK(); - $column = \count($pk) > 1 || self::rowsAreUnique($query) - ? '*' - : \sprintf('DISTINCT(%s)', \reset($pk)); + + return (int) $this->builder->withQuery($query)->count($column ?? $this->countColumn($query)); + } + + /** + * One page of entities and the number of rows of the whole query, as {@see count()} gives. + * For a select of a cached shape one statement reads both: the count is a scalar subquery + * column of the page query. Otherwise, or when the page is past the last row, the count + * is read by its own statement. + * + * @param int<0, max> $limit + * @param int<0, max> $offset + * + * @return array{list, int<0, max>} + */ + public function fetchPage(int $limit, int $offset): array + { + if (!$this->customized && $this->limit === 0 && $this->offset === 0) { + $page = (clone $this)->limit($limit)->offset($offset); + $key = $page->shape . "\3count"; + $plan = $this->shapes[$key] ?? null; + if ($plan === null && \count($this->shapes) < 1024) { + $plan = $this->shapes[$key] = $page->shapePlan($this); + } + if ($plan) { + [$driver, $sql, $values, $slots, $maker] = $plan; + foreach ($slots as $index => $i) { + $values[$index] = new Parameter($this->shapeValues[$i]); + } + $statement = $driver->query($sql, $values); + $rows = $statement->fetchAll(StatementInterface::FETCH_NUM); + $statement->close(); + if ($rows !== []) { + return [$maker($rows), (int) $rows[0][\array_key_last($rows[0])]]; + } + + return [[], (clone $this)->count()]; + } } - return (int) $this->builder->withQuery($query)->count($column); + $items = (clone $this)->limit($limit)->offset($offset)->fetchAll(); + + return [\is_array($items) ? \array_values($items) : \iterator_to_array($items, false), (clone $this)->count()]; + } + + /** + * COUNT(*) when every row is a distinct entity, COUNT(DISTINCT pk) otherwise. + */ + private function countColumn(SelectQuery $query): string + { + $pk = (array) $this->loader->getPK(); + + return \count($pk) > 1 || self::rowsAreUnique($query) + ? '*' + : \sprintf('DISTINCT(%s)', \reset($pk)); } /** @@ -1180,7 +1227,7 @@ private function fetchShape(): ?array return null; } [$driver, $sql, $values, $slots, $maker] = $plan; - foreach ($slots as $i => $index) { + foreach ($slots as $index => $i) { $values[$index] = new Parameter($this->shapeValues[$i]); } $statement = $driver->query($sql, $values); @@ -1194,7 +1241,7 @@ private function fetchShape(): ?array * Compiles the recorded shape once with sentinel values and finds where each * value lands in the statement parameters. False when the shape can't be cached. */ - private function shapePlan(): array|false + private function shapePlan(?self $countOf = null): array|false { // A scope adds its own values (a tenant, a soft-delete flag) when the query is built; // a cached statement would keep the first ones. @@ -1215,9 +1262,21 @@ private function shapePlan(): array|false return false; } $query = $copy->addGroupByPK()->loader->buildQuery(); + if ($countOf !== null) { + // The count of the select without limit and offset, with the same values. + $counted = clone $countOf; + if (\count($counted->shapeValues) !== \count($sentinels)) { + return false; + } + $counted->shapeValues = $copy->shapeValues; + $counted->applyRecorded(); + $countQuery = $counted->buildQuery(); + $query->columns([...$query->getColumns(), $countQuery->countQuery($counted->countColumn($countQuery))]); + } $parameters = new QueryParameters(); $sql = $query->sqlStatement($parameters); $values = $parameters->getParameters(); + // Statement parameter index => shape value; a value may land in more than one place. $slots = []; foreach ($values as $index => $value) { $raw = $value instanceof ParameterInterface ? $value->getValue() : $value; @@ -1225,13 +1284,13 @@ private function shapePlan(): array|false continue; } $slot = $sentinels[$raw] ?? null; - if ($slot === null || isset($slots[$slot]) || !$value instanceof Parameter) { + if ($slot === null || !$value instanceof Parameter) { return false; } - $slots[$slot] = $index; + $slots[$index] = $slot; } - return \count($slots) === \count($sentinels) ? [$query->getDriver(), $sql, $values, $slots, $maker] : false; + return \count(\array_unique($slots)) === \count($sentinels) ? [$query->getDriver(), $sql, $values, $slots, $maker] : false; } /** diff --git a/src/ORM/Service/Implementation/EntityFactory.php b/src/ORM/Service/Implementation/EntityFactory.php index 35f0605..b467707 100644 --- a/src/ORM/Service/Implementation/EntityFactory.php +++ b/src/ORM/Service/Implementation/EntityFactory.php @@ -218,8 +218,14 @@ private function compileDataMaker(string $role): ?\Closure $items = []; $casters = []; + $zones = []; foreach (\array_intersect_key($rules, \array_flip($fields)) as $field => $rule) { $key = \var_export($field, true); + if ($rule instanceof \DateTimeZone) { + $zones[$field] = $rule; + $items[] = "\$d[{$key}] === null or \$d[{$key}] = new \\DateTimeImmutable(\$d[{$key}], \$zones[{$key}]);"; + continue; + } if ($rule instanceof \Closure) { $casters[$field] = $rule; $items[] = "\$d[{$key}] === null or \$d[{$key}] = \$casters[{$key}](\$d[{$key}]);"; @@ -227,7 +233,7 @@ private function compileDataMaker(string $role): ?\Closure } $items[] = "\$d[{$key}] === null or \$d[{$key}] = ({$rule}) \$d[{$key}];"; } - $cast = eval("declare(strict_types=1);\nreturn static function (array \$d) use (\$casters): array {\n" . \implode("\n", $items) . "\nreturn \$d;\n};"); + $cast = eval("declare(strict_types=1);\nreturn static function (array \$d) use (\$casters, \$zones): array {\n" . \implode("\n", $items) . "\nreturn \$d;\n};"); $flip = \array_fill_keys($fields, true); $count = \count($fields); $heap = $this->heap; @@ -276,9 +282,16 @@ private function compileRowMaker(string $role, array $columns, ?string $groupBy, // Typecast compiled into the array literal: null stays null, as in Typecast::cast(). $items = []; $casters = []; + $zones = []; foreach ($columns as $i => $field) { $key = \var_export($field, true); $rule = $rules[$field] ?? null; + if ($rule instanceof \DateTimeZone) { + // The datetime rule ({@see \Arcano\ORM\Parser\Typecast}) without a call per value. + $zones[$field] = $rule; + $items[] = "{$key} => \$row[{$i}] === null ? null : new \\DateTimeImmutable(\$row[{$i}], \$zones[{$key}])"; + continue; + } if ($rule instanceof \Closure) { $casters[$field] = $rule; $items[] = "{$key} => \$row[{$i}] === null ? null : \$casters[{$key}](\$row[{$i}])"; @@ -351,7 +364,7 @@ private function compileRowMaker(string $role, array $columns, ?string $groupBy, // $nodes the heap node of each result. $maker = eval(<<loadedStorage(); \$index = &\$heap->loadedIndex({$roleKey}); diff --git a/src/ORM/Transaction/DirectInsert.php b/src/ORM/Transaction/DirectInsert.php index b5d22f7..0f8032f 100644 --- a/src/ORM/Transaction/DirectInsert.php +++ b/src/ORM/Transaction/DirectInsert.php @@ -32,10 +32,24 @@ final class DirectInsert /** @var array */ private array $roles = []; + /** + * Role descriptions by ORM: they depend on the schema, mappers and sources only, so the + * unit of work of every request reuses them. + * + * @var \WeakMap, array}>|null + */ + private static ?\WeakMap $described = null; + public function __construct( private ORMInterface $orm, private Heap $heap, - ) {} + ) { + self::$described ??= new \WeakMap(); + if (isset(self::$described[$orm])) { + [$this->classRoles, $this->roles] = self::$described[$orm]; + } + } + /** * The insert of a new entity graph, or null when it needs the pool. @@ -239,11 +253,16 @@ private function describeEntity(object $entity, array &$claimed): ?array } $claimed[$id] = true; - $role = $this->classRoles[$entity::class] ??= $this->resolveRole($entity); + $role = $this->classRoles[$entity::class] ?? null; + if ($role === null || !isset($this->roles[$role !== false ? $role : ''])) { + $role = $this->classRoles[$entity::class] ??= $this->resolveRole($entity); + $role === false or $this->roles[$role] ??= $this->describe($role) ?? false; + self::$described[$this->orm] = [$this->classRoles, $this->roles]; + } if ($role === false) { return null; } - $describe = $this->roles[$role] ??= $this->describe($role) ?? false; + $describe = $this->roles[$role]; return $describe === false ? null : $describe; } diff --git a/tests/Database/Functional/Driver/Common/Query/SelectQueryTest.php b/tests/Database/Functional/Driver/Common/Query/SelectQueryTest.php index f233f36..fcf2abd 100644 --- a/tests/Database/Functional/Driver/Common/Query/SelectQueryTest.php +++ b/tests/Database/Functional/Driver/Common/Query/SelectQueryTest.php @@ -56,6 +56,28 @@ public function testCompileQueryExtraParam(): void $this->assertSameParameters(['string', 'string'], $select); } + public function testScalarSubqueryColumnKeepsItsParameters(): void + { + $count = $this->db()->select('COUNT(*)')->from('table')->where('name', 'Antony'); + $select = $this->db() + ->select('id', $count) + ->from('table') + ->where('value', '>', 10); + + $this->assertSameQuery( + 'SELECT {id}, (SELECT COUNT(*) FROM {table} WHERE {name} = ?) FROM {table} WHERE {value} > ?', + $select, + ); + $this->assertSameParameters(['Antony', 10], $select); + + // The compiled statement is cached by shape: other values are not baked into it. + $other = $this->db() + ->select('id', $this->db()->select('COUNT(*)')->from('table')->where('name', 'John')) + ->from('table') + ->where('value', '>', 20); + $this->assertSameParameters(['John', 20], $other); + } + public function testSimpleWhere(): void { $select = $this->database->select()->from('table')->where('id', 1); diff --git a/tests/ORM/Functional/Driver/Common/Select/PaginateTest.php b/tests/ORM/Functional/Driver/Common/Select/PaginateTest.php index 582f58a..bda464f 100644 --- a/tests/ORM/Functional/Driver/Common/Select/PaginateTest.php +++ b/tests/ORM/Functional/Driver/Common/Select/PaginateTest.php @@ -32,6 +32,53 @@ public function testPaginate(): void $this->assertCount(10, $selector->fetchData()); } + public function testFetchPageReadsRowsAndCountWithOneStatement(): void + { + $select = (new Select($this->orm, User::class))->where('balance', '>=', 1000)->orderBy('id', 'DESC'); + + $this->captureReadQueries(); + [$items, $total] = $select->fetchPage(7, 14); + $this->assertNumReads(1); + + $this->assertSame(90, $total); + $this->assertSame( + \array_map(static fn (User $user): int => (int) $user->id, (clone $select)->limit(7)->offset(14)->fetchAll()), + \array_map(static fn (User $user): int => (int) $user->id, $items), + ); + $this->assertSame(86, (int) $items[0]->id); + $this->assertSame(90, $select->count()); + } + + public function testFetchPageCachedStatementTakesTheValuesOfEachSelect(): void + { + foreach ([[0, 100], [5000, 50], [9900, 1], [9901, 0]] as [$balance, $expected]) { + [$items, $total] = (new Select($this->orm, User::class)) + ->where('balance', '>=', $balance) + ->orderBy('id') + ->fetchPage(10, 0); + $this->assertSame($expected, $total); + $this->assertCount(\min(10, $expected), $items); + } + } + + public function testFetchPagePastTheLastRowStillCounts(): void + { + [$items, $total] = (new Select($this->orm, User::class))->orderBy('id')->fetchPage(10, 100); + + $this->assertSame([], $items); + $this->assertSame(100, $total); + } + + public function testFetchPageOfACustomizedSelect(): void + { + $select = (new Select($this->orm, User::class))->where(['balance' => ['<' => 500]])->orderBy('id'); + + [$items, $total] = $select->fetchPage(2, 1); + + $this->assertSame(5, $total); + $this->assertSame(['1@world.com', '2@world.com'], \array_map(static fn (User $user): string => $user->email, $items)); + } + public function setUp(): void { parent::setUp(); From 22fca313b89a4c4759e81a43c60b5206db9458f2 Mon Sep 17 00:00:00 2001 From: davidbalbino Date: Thu, 19 Nov 2026 10:47:08 -0300 Subject: [PATCH 2/2] Row makers build UTC timestamps with pam_native_utc_datetime() when the runtime has it On the PAM runtime a UTC datetime column is a filled-in clone of a prototype instead of a parse: the same DateTimeImmutable the constructor returns, about 4x cheaper. Values the function does not take (other shapes, zones or non-strings) still go to the constructor. --- src/ORM/Service/Implementation/EntityFactory.php | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ORM/Service/Implementation/EntityFactory.php b/src/ORM/Service/Implementation/EntityFactory.php index b467707..bb7ebb1 100644 --- a/src/ORM/Service/Implementation/EntityFactory.php +++ b/src/ORM/Service/Implementation/EntityFactory.php @@ -283,13 +283,21 @@ private function compileRowMaker(string $role, array $columns, ?string $groupBy, $items = []; $casters = []; $zones = []; + $prototypes = []; foreach ($columns as $i => $field) { $key = \var_export($field, true); $rule = $rules[$field] ?? null; if ($rule instanceof \DateTimeZone) { // The datetime rule ({@see \Arcano\ORM\Parser\Typecast}) without a call per value. $zones[$field] = $rule; - $items[] = "{$key} => \$row[{$i}] === null ? null : new \\DateTimeImmutable(\$row[{$i}], \$zones[{$key}])"; + $construct = "new \\DateTimeImmutable(\$row[{$i}], \$zones[{$key}])"; + if ($rule->getName() === 'UTC' && \function_exists('pam_native_utc_datetime')) { + // On the PAM runtime a UTC database timestamp is a filled-in clone of a + // prototype: the same object the constructor gives, without its parser. + $prototypes[$field] = new \DateTimeImmutable('2000-01-01 00:00:00', $rule); + $construct = "(\\is_string(\$row[{$i}]) && (\$t = \\pam_native_utc_datetime(\$row[{$i}], \$prototypes[{$key}])) !== null ? \$t : {$construct})"; + } + $items[] = "{$key} => \$row[{$i}] === null ? null : {$construct}"; continue; } if ($rule instanceof \Closure) { @@ -364,7 +372,7 @@ private function compileRowMaker(string $role, array $columns, ?string $groupBy, // $nodes the heap node of each result. $maker = eval(<<loadedStorage(); \$index = &\$heap->loadedIndex({$roleKey});