Skip to content

Support column value replacement in SQL exports #821

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
adamziel wants to merge 1 commit into
base: trunk
Choose a base branch
from
Open

Support column value replacement in SQL exports #821

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 39 additions & 2 deletions packages/reprint-server/src/class-database-rows-reader.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,9 @@ class DatabaseRowsReader {
/** @var array<string,string> Internal SRID-prefix aliases keyed by spatial column name. */
private $spatial_prefix_aliases = [];

/** @var array<string,array<string,string>> Trusted SELECT expressions, keyed by table and column. */
private $column_read_expressions = [];

/** @var array<string,list<array{column:string,value:string}>> Row exclusions keyed by table. */
private $exclude_rows_by_table = [];

Expand Down Expand Up @@ -179,6 +182,7 @@ class DatabaseRowsReader {
* @type int $batch_size Maximum records per query.
* @type int|null $query_time_limit_ms Maximum query duration in milliseconds.
* @type int|null $maximum_inline_spatial_bytes Largest spatial value returned inline.
* @type array<string,array<string,string>> $column_read_expressions Trusted server-built SELECT expressions keyed by table and column; never request SQL.
* @type array $exclude_rows Table, column, and value exclusion rules.
* @type string[] $exclude_tables Table names to omit from automatic discovery.
* @type MultisiteDatabaseSelection $multisite_selection Site table and row rules built from source WordPress state.
Expand All @@ -187,6 +191,20 @@ class DatabaseRowsReader {
public function __construct($db, $options = [])
{
$this->db = $db;
$this->column_read_expressions = array_key_exists('column_read_expressions', $options) ? $options['column_read_expressions'] : [];
if (!is_array($this->column_read_expressions)) {
throw new \InvalidArgumentException('column_read_expressions must be an array; received ' . gettype($this->column_read_expressions) . '.');
}
foreach ($this->column_read_expressions as $table => $expressions) {
if (!is_string($table) || $table === '' || !is_array($expressions)) {
throw new \InvalidArgumentException('column_read_expressions must map non-empty table names to arrays; received ' . gettype($expressions) . ' for table ' . json_encode($table) . '.');
}
foreach ($expressions as $column => $expression) {
if (!is_string($column) || $column === '' || !is_string($expression) || $expression === '') {
throw new \InvalidArgumentException('column_read_expressions must map non-empty column names to non-empty SQL strings; received ' . json_encode($expression) . ' for column ' . json_encode($column) . '.');
}
}
}
$this->multisite_selection = $options["multisite_selection"] ?? null;
if ($this->multisite_selection !== null && !$this->multisite_selection instanceof MultisiteDatabaseSelection) {
throw new \InvalidArgumentException("multisite_selection must be a trusted MultisiteDatabaseSelection object.");
Expand Down Expand Up @@ -884,6 +902,7 @@ private function build_byte_preserving_select_from_current_table()
$select_parts = [];
foreach ($this->current_column_types as $column => $column_info) {
$quoted_column = $this->quote_identifier($column);
$read_expression = $this->get_column_read_expression($column);
if (
$this->maximum_inline_spatial_bytes !== null &&
$this->is_spatial_type($column_info["data_type"])
Expand Down Expand Up @@ -915,9 +934,9 @@ private function build_byte_preserving_select_from_current_table()
$this->is_numeric_type($column_info["data_type"]) ||
$this->is_binary_type($column_info["data_type"])
) {
$select_parts[] = $quoted_column;
$select_parts[] = $read_expression . " AS " . $quoted_column;
} else {
$select_parts[] = "CAST({$quoted_column} AS BINARY) AS {$quoted_column}";
$select_parts[] = "CAST({$read_expression} AS BINARY) AS {$quoted_column}";
}
}
if ($this->multisite_selection !== null) {
Expand All @@ -938,6 +957,24 @@ private function build_byte_preserving_select_from_current_table()
return $query;
}

/**
* Use the same exported value for ordered reads, reloads, and oversized chunks.
* Keep its source character set so SUBSTRING counts characters, not bytes.
*/
public function get_column_read_expression(string $column): string
{
if (!isset($this->column_read_expressions[$this->current_table][$column])) {
return $this->quote_identifier($column);
}
$expression = $this->column_read_expressions[$this->current_table][$column];
$collation = $this->get_column_metadata($column)['collation'];
if ($collation !== null) {
$charset = explode('_', $collation, 2)[0];
return "CONVERT(({$expression}) USING {$charset})";
}
return "({$expression})";
}

/** Returns an internal SELECT alias which cannot collide with a real column. */
private function get_spatial_length_alias($column)
{
Expand Down
15 changes: 12 additions & 3 deletions packages/reprint-server/src/class-mysql-dump-producer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,15 @@ class MySQLDumpProducer
* @param object $db Database connection — either a real PDO (MySQL) or a
* PDO-compatible adapter (SQLite sites). No type hint because the
* adapter isn't a PDO subclass and PHP 7.4 lacks union types.
* @param array $options {
* Producer settings, plus the options documented by DatabaseRowsReader.
*
* @type bool $create_table_query Include table definitions. Default true.
* @type int $max_statement_size SQL statement byte ceiling.
* @type int $target_max_allowed_packet Target packet byte ceiling.
* @type string $cursor Saved producer cursor for resume.
* }
* @see DatabaseRowsReader::__construct()
*/
public function __construct($db, $options = [])
{
Expand Down Expand Up @@ -2004,7 +2013,7 @@ private function fetch_value_substring_from_the_current_oversized_row(
bool $character_string
): array {
$quoted_table = $this->row_reader->quote_identifier($this->row_reader->get_current_table());
$quoted_column = $this->row_reader->quote_identifier($column);
$column_expression = $this->row_reader->get_column_read_expression($column);

$where_parts = $this->row_reader->get_current_row_selection_conditions(true);
foreach ($this->oversized_pk_values as $pk_col => $pk_value) {
Expand All @@ -2013,8 +2022,8 @@ private function fetch_value_substring_from_the_current_oversized_row(
$where_clause = implode(" AND ", $where_parts);

$value_expression = $character_string
? "SUBSTRING({$quoted_column}, {$start}, {$length})"
: "SUBSTRING(CAST({$quoted_column} AS BINARY), {$start}, {$length})";
? "SUBSTRING({$column_expression}, {$start}, {$length})"
: "SUBSTRING(CAST({$column_expression} AS BINARY), {$start}, {$length})";
$sql = $this->row_reader->get_select_prefix() . " CAST({$value_expression} AS BINARY) AS value_chunk,"
. " CHAR_LENGTH({$value_expression}) AS value_length"
. " FROM {$quoted_table} WHERE {$where_clause}";
Expand Down
58 changes: 58 additions & 0 deletions tests/MySQLDumpProducer/ColumnReadExpressionsTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
<?php

require_once __DIR__ . '/MySQLDumpProducerTestBase.php';

class ColumnReadExpressionsTest extends MySQLDumpProducerTestBase
{
public static function column_types(): array
{
return [
['LONGTEXT CHARACTER SET utf8mb4', "é漢🙂"],
['LONGTEXT CHARACTER SET latin1', "\xe9"],
['LONGBLOB', "\xff\xe9"],
];
}

/** @dataProvider column_types */
public function testReplacementSurvivesOversizedReadsAndResume(string $type, string $bytes): void
{
$this->pdo->exec("CREATE TABLE custom_options (option_id bigint PRIMARY KEY, option_name varchar(191), option_value {$type}, autoload varchar(10))");
$plugins = [];
for ($index = 0; $index < 300; ++$index) {
$plugins[] = 'plugin-' . $index . '-' . str_repeat($bytes, 5) . '/index.php';
}
$original = serialize(array_merge(['reprint/index.php'], $plugins));
$replacement = serialize($plugins);
$insert = $this->pdo->prepare('INSERT INTO custom_options VALUES (?, ?, FROM_BASE64(?), ?)');
$insert->execute([1, 'active_plugins', base64_encode($original), 'yes']);
$insert->execute([2, 'another_setting', base64_encode($original), 'no']);
$encoded = base64_encode($replacement);
$options = [
'max_statement_size' => 2048,
'batch_size' => 1,
'column_read_expressions' => ['custom_options' => ['option_value' =>
"CASE WHEN option_name = 'active_plugins' THEN FROM_BASE64('{$encoded}') ELSE option_value END"]],
];
$producer = $this->createProducer($options);
$sql = '';
$steps = 0;
while ($producer->next_sql_fragment()) {
$this->assertLessThan(500, ++$steps, 'The resumed export must reach the end of the changed value.');
$sql .= $producer->get_sql_fragment();
if ($producer->is_finished()) {
break;
}
$options['cursor'] = $producer->get_reentrancy_cursor();
$producer->close();
$producer = $this->createProducer($options);
}
$this->assertStringContainsString('UPDATE `custom_options`', $sql, 'Exercise the oversized substring path, not just a normal INSERT.');
$target = $this->executeDumpInNewDatabase($sql);
$rows = $target->query('SELECT CAST(option_value AS BINARY) AS option_value, autoload FROM custom_options ORDER BY option_id')->fetchAll(PDO::FETCH_ASSOC);
$this->assertSame([
['option_value' => $replacement, 'autoload' => 'yes'],
['option_value' => $original, 'autoload' => 'no'],
], $rows);
$this->assertSame($original, $this->pdo->query('SELECT CAST(option_value AS BINARY) FROM custom_options WHERE option_id = 1')->fetchColumn());
}
}
Loading