diff --git a/components/DataLiberation/BlockMarkup/class-blockmarkupurlprocessor.php b/components/DataLiberation/BlockMarkup/class-blockmarkupurlprocessor.php index 5c0452afe..e8aaf6b60 100644 --- a/components/DataLiberation/BlockMarkup/class-blockmarkupurlprocessor.php +++ b/components/DataLiberation/BlockMarkup/class-blockmarkupurlprocessor.php @@ -3,8 +3,9 @@ namespace WordPress\DataLiberation\BlockMarkup; use Rowbot\URL\URL; -use WordPress\DataLiberation\URL\URLInTextProcessor; +use WordPress\DataLiberation\Shortcode\ShortcodeProcessor; use WordPress\DataLiberation\URL\CSSURLProcessor; +use WordPress\DataLiberation\URL\URLInTextProcessor; use WordPress\DataLiberation\URL\WPURL; /** @@ -23,6 +24,11 @@ class BlockMarkupUrlProcessor extends BlockMarkupProcessor { private $url_in_text_node_updated; private $css_url_processor; private $css_url_processor_updated; + private $text_node_uses_shortcode_processor; + private $shortcode_processor; + private $shortcode_css_url_processor; + private $shortcode_url_context; + private $shortcode_processor_updated; /** * The list of names of URL-related HTML attributes that may be available on @@ -47,6 +53,11 @@ public function __construct( $html, ?string $base_url_string = null ) { } public function get_updated_html(): string { + if ( $this->shortcode_processor_updated ) { + $this->set_modifiable_text_raw( $this->shortcode_processor->get_updated_text() ); + $this->shortcode_processor_updated = false; + } + if ( $this->url_in_text_node_updated ) { $this->set_modifiable_text( $this->url_in_text_processor->get_updated_text() ); $this->url_in_text_node_updated = false; @@ -74,14 +85,18 @@ public function get_parsed_url() { public function next_token(): bool { $this->get_updated_html(); - $this->raw_url = null; - $this->parsed_url = null; - $this->inspecting_html_attributes = null; - $this->url_in_text_processor = null; - $this->css_url_processor = null; + $this->raw_url = null; + $this->parsed_url = null; + $this->inspecting_html_attributes = null; + $this->url_in_text_processor = null; + $this->css_url_processor = null; + $this->text_node_uses_shortcode_processor = null; + $this->shortcode_processor = null; + $this->shortcode_css_url_processor = null; + $this->shortcode_url_context = null; /* - * Do not reset url_in_text_node_updated or css_url_processor_updated – they're reset - * in get_updated_html() which is called in parent::next_token(). + * Do not reset the update flags. They are reset in get_updated_html(), + * which is called above before advancing the parent processor. */ return parent::next_token(); @@ -116,6 +131,32 @@ private function next_url_in_text_node() { return false; } + if ( null === $this->text_node_uses_shortcode_processor ) { + $raw_text = $this->get_modifiable_text_raw(); + $shortcode_probe = new ShortcodeProcessor( $raw_text ); + + $this->text_node_uses_shortcode_processor = $shortcode_probe->next_shortcode( + array( 'escaped' => false ) + ); + + if ( $this->text_node_uses_shortcode_processor ) { + $this->shortcode_processor = new ShortcodeProcessor( $raw_text ); + } + } + + if ( $this->text_node_uses_shortcode_processor ) { + return $this->next_url_in_shortcode_text_node(); + } + + return $this->next_url_in_plain_text_node(); + } + + /** + * Finds the next URL in an HTML text node without shortcode markup. + * + * @return bool Whether a URL was found. + */ + private function next_url_in_plain_text_node() { if ( null === $this->url_in_text_processor ) { /* * Use the base URL for URLs matched in text nodes. This is the only @@ -141,6 +182,112 @@ private function next_url_in_text_node() { return false; } + /** + * Finds the next URL while respecting shortcode token boundaries. + * + * Shortcode attributes may own another grammar, such as CSS. Process that + * grammar before deciding whether the complete attribute is itself a URL. + * Text between shortcodes retains the existing free-text URL matching. + * + * @return bool Whether a URL was found. + */ + private function next_url_in_shortcode_text_node() { + while ( true ) { + if ( null !== $this->shortcode_css_url_processor ) { + if ( $this->next_url_in_shortcode_css_attribute() ) { + return true; + } + + $this->shortcode_css_url_processor = null; + $this->shortcode_url_context = null; + } + + if ( null !== $this->url_in_text_processor ) { + while ( $this->url_in_text_processor->next_url() ) { + $this->raw_url = $this->url_in_text_processor->get_raw_url(); + $this->parsed_url = $this->url_in_text_processor->get_parsed_url(); + + return true; + } + + $this->url_in_text_processor = null; + $this->shortcode_url_context = null; + + if ( ! $this->shortcode_processor->next_token() ) { + return false; + } + } + + if ( ShortcodeProcessor::TOKEN_TEXT === $this->shortcode_processor->get_token_type() ) { + $text = $this->shortcode_processor->get_modifiable_text(); + if ( null !== $text ) { + $this->url_in_text_processor = new URLInTextProcessor( $text, $this->base_url_string ); + $this->shortcode_url_context = 'text'; + continue; + } + } elseif ( + ShortcodeProcessor::TOKEN_SHORTCODE === $this->shortcode_processor->get_token_type() && + ! $this->shortcode_processor->is_escaped() && + ! $this->shortcode_processor->is_tag_closer() + ) { + while ( $this->shortcode_processor->next_attribute() ) { + $value = $this->shortcode_processor->get_attribute_value(); + if ( null === $value ) { + continue; + } + + $this->shortcode_css_url_processor = new CSSURLProcessor( $value ); + if ( $this->next_url_in_shortcode_css_attribute() ) { + return true; + } + $this->shortcode_css_url_processor = null; + + $parsed_url = WPURL::parse( $value ); + if ( false === $parsed_url ) { + continue; + } + + $this->raw_url = $value; + $this->parsed_url = $parsed_url; + $this->shortcode_url_context = 'attribute'; + + return true; + } + } + + if ( ! $this->shortcode_processor->next_token() ) { + return false; + } + } + } + + /** + * Advances through CSS URLs in the current shortcode attribute. + * + * @return bool Whether a URL was found. + */ + private function next_url_in_shortcode_css_attribute() { + while ( $this->shortcode_css_url_processor->next_url() ) { + if ( $this->shortcode_css_url_processor->is_data_uri() ) { + continue; + } + + $raw_url = $this->shortcode_css_url_processor->get_raw_url(); + $parsed_url = WPURL::parse( $raw_url, $this->base_url_string ); + if ( false === $parsed_url ) { + continue; + } + + $this->raw_url = $raw_url; + $this->parsed_url = $parsed_url; + $this->shortcode_url_context = 'css-attribute'; + + return true; + } + + return false; + } + /** * Advances to the next CSS URL in the `style` attribute of the current tag token. * @@ -373,6 +520,10 @@ public function set_url( $raw_url, $parsed_url ) { return $this->set_block_attribute_value( $raw_url ); case '#text': + if ( $this->text_node_uses_shortcode_processor ) { + return $this->set_url_in_shortcode_text_node( $raw_url ); + } + if ( null === $this->url_in_text_processor ) { return false; } @@ -382,6 +533,40 @@ public function set_url( $raw_url, $parsed_url ) { } } + /** + * Replaces the current URL through the nested shortcode grammar. + * + * @param string $raw_url Replacement URL. + * @return bool Whether the URL was set. + */ + private function set_url_in_shortcode_text_node( $raw_url ) { + if ( 'attribute' === $this->shortcode_url_context ) { + $updated = $this->shortcode_processor->set_attribute_value( $raw_url ); + } elseif ( 'css-attribute' === $this->shortcode_url_context ) { + $updated = $this->shortcode_css_url_processor->set_raw_url( $raw_url ); + if ( $updated ) { + $updated = $this->shortcode_processor->set_attribute_value( + $this->shortcode_css_url_processor->get_updated_css() + ); + } + } elseif ( 'text' === $this->shortcode_url_context ) { + $updated = $this->url_in_text_processor->set_raw_url( $raw_url ); + if ( $updated ) { + $updated = $this->shortcode_processor->set_modifiable_text( + $this->url_in_text_processor->get_updated_text() + ); + } + } else { + return false; + } + + if ( $updated ) { + $this->shortcode_processor_updated = true; + } + + return $updated; + } + /** * Rewrites the components of the currently matched URL from ones * provided in $from_url to ones specified in $to_url. @@ -398,6 +583,10 @@ public function replace_base_url( $to_url, $base_url = null ) { if ( ! $base_url ) { return false; } + $is_shortcode_attribute_url = ( + '#text' === $this->get_token_type() && + in_array( $this->shortcode_url_context, array( 'attribute', 'css-attribute' ), true ) + ); $result = WPURL::replace_base_url( $this->get_parsed_url(), @@ -407,12 +596,12 @@ public function replace_base_url( $to_url, $base_url = null ) { 'raw_url' => $this->get_raw_url(), 'is_relative' => ( /** - * In text nodes, the only detected URLs are absolute. The tricky part - * is they may start without a protocol, e.g. `wordpress.org`. Therefore, - * we need to tell WPURL::replace_base_url what's our intention regarding - * the URL's relativity. It cannot just infer it from the URL itself. + * Free-text URLs are treated as absolute even when they omit a protocol, + * as in `wordpress.org`. Shortcode attributes are structured regions, + * so relative CSS URLs inside them retain their relative form just like + * URLs in HTML and block attributes. */ - '#text' !== $this->get_token_type() && + ( '#text' !== $this->get_token_type() || $is_shortcode_attribute_url ) && ! WPURL::can_parse( $this->get_raw_url() ) ), ) @@ -422,9 +611,7 @@ public function replace_base_url( $to_url, $base_url = null ) { return false; } - $this->set_url( $result . '', $result->new_url ); - - return true; + return $this->set_url( $result . '', $result->new_url ); } /** diff --git a/components/DataLiberation/Shortcode/class-shortcodeprocessor.php b/components/DataLiberation/Shortcode/class-shortcodeprocessor.php new file mode 100644 index 000000000..229050555 --- /dev/null +++ b/components/DataLiberation/Shortcode/class-shortcodeprocessor.php @@ -0,0 +1,1198 @@ + + */ + private $attributes = array(); + + /** + * Index of the current attribute. + * + * @var int + */ + private $attribute_index = -1; + + /** + * Lexical replacements keyed by the token and field that owns the update. + * + * @var array[] + * @phpstan-var array + */ + private $lexical_updates = array(); + + /** + * Byte offset at which scanning may resume after a failed candidate. + * + * @var int|null + */ + private $scan_at_after_failure = null; + + /** + * Constructor. + * + * @param string $text Text containing possible shortcode markup. + */ + public function __construct( string $text ) { + $this->text = $text; + $this->length = strlen( $text ); + $this->last_closing_bracket_at = strrpos( $text, ']' ); + } + + /** + * Finds the next shortcode matching an optional query. + * + * A string query matches an exact, case-sensitive shortcode tag name. + * An array query may contain: + * + * - `tag_name`: exact, case-sensitive tag name. + * - `tag_prefix`: case-sensitive tag-name prefix. + * - `tag_closers`: `visit` (default) or `skip`. + * - `match_offset`: one-based index of the matching token to return. + * - `escaped`: whether the token must use the complete `[[tag]]` form. + * + * @param array|string|null $query Optional shortcode query. + * @return bool Whether a matching shortcode token was found. + */ + public function next_shortcode( $query = null ): bool { + $tag_name = null; + $tag_prefix = null; + $tag_closers = 'visit'; + $match_offset = 1; + $escaped = null; + + if ( is_string( $query ) ) { + $tag_name = $query; + } elseif ( is_array( $query ) ) { + $tag_name = isset( $query['tag_name'] ) ? (string) $query['tag_name'] : null; + $tag_prefix = isset( $query['tag_prefix'] ) ? (string) $query['tag_prefix'] : null; + $tag_closers = isset( $query['tag_closers'] ) ? (string) $query['tag_closers'] : 'visit'; + $match_offset = isset( $query['match_offset'] ) ? max( 1, (int) $query['match_offset'] ) : 1; + $escaped = isset( $query['escaped'] ) ? (bool) $query['escaped'] : null; + } + + $matches = 0; + while ( $this->next_token() ) { + if ( self::TOKEN_SHORTCODE !== $this->token_type ) { + continue; + } + + if ( 'skip' === $tag_closers && $this->is_closing_tag ) { + continue; + } + + $current_tag = $this->get_tag(); + if ( null !== $tag_name && $current_tag !== $tag_name ) { + continue; + } + + if ( + null !== $tag_prefix && + 0 !== strncmp( $current_tag, $tag_prefix, strlen( $tag_prefix ) ) + ) { + continue; + } + + if ( null !== $escaped && $this->is_escaped() !== $escaped ) { + continue; + } + + ++$matches; + if ( $matches >= $match_offset ) { + return true; + } + } + + return false; + } + + /** + * Advances to the next shortcode or text token. + * + * The processor searches forward for a complete shortcode candidate. + * Invalid or truncated candidates remain part of a text token. + * + * @return bool Whether another token was found. + */ + public function next_token(): bool { + $this->after_token(); + + if ( $this->at >= $this->length ) { + return false; + } + + $scan_at = $this->at; + while ( $scan_at < $this->length ) { + $candidate_at = strpos( $this->text, '[', $scan_at ); + if ( false === $candidate_at ) { + break; + } + + if ( + false === $this->last_closing_bracket_at || + $candidate_at > $this->last_closing_bracket_at + ) { + // The remaining `[` bytes cannot begin complete shortcode tokens. + break; + } + + $shortcode = $this->scan_shortcode_at( $candidate_at ); + if ( false === $shortcode ) { + $scan_at = null === $this->scan_at_after_failure + ? $candidate_at + 1 + : $this->scan_at_after_failure; + continue; + } + + if ( $candidate_at > $this->at ) { + $this->token_type = self::TOKEN_TEXT; + $this->token_starts_at = $this->at; + $this->token_length = $candidate_at - $this->at; + $this->at = $candidate_at; + + return true; + } + + $this->set_shortcode_token( $shortcode ); + $this->at = $candidate_at + $shortcode['length']; + + return true; + } + + $this->token_type = self::TOKEN_TEXT; + $this->token_starts_at = $this->at; + $this->token_length = $this->length - $this->at; + $this->at = $this->length; + + return true; + } + + /** + * Returns the current token type. + * + * @return string|null `#shortcode`, `#text`, or null when not on a token. + */ + public function get_token_type(): ?string { + return $this->token_type; + } + + /** + * Returns the exact source bytes of the current token. + * + * @return string|null Current token text, or null when not on a token. + */ + public function get_token_text(): ?string { + if ( null === $this->token_starts_at || null === $this->token_length ) { + return null; + } + + return substr( $this->text, $this->token_starts_at, $this->token_length ); + } + + /** + * Returns the current shortcode tag name exactly as written. + * + * Shortcode tag names are case-sensitive. + * + * @return string|null Current shortcode tag name, or null on a text token. + */ + public function get_tag(): ?string { + if ( + self::TOKEN_SHORTCODE !== $this->token_type || + null === $this->tag_name_starts_at || + null === $this->tag_name_length + ) { + return null; + } + + return substr( $this->text, $this->tag_name_starts_at, $this->tag_name_length ); + } + + /** + * Indicates whether the current shortcode token is a closing token. + * + * @return bool Whether the current token is a shortcode closer. + */ + public function is_tag_closer(): bool { + return self::TOKEN_SHORTCODE === $this->token_type && $this->is_closing_tag; + } + + /** + * Indicates whether the current shortcode token has a self-closing flag. + * + * @return bool Whether the current token is self-closing. + */ + public function has_self_closing_flag(): bool { + return ( + self::TOKEN_SHORTCODE === $this->token_type && + $this->has_self_closing_flag + ); + } + + /** + * Indicates whether the current shortcode uses complete `[[tag]]` escaping. + * + * @return bool Whether both the opening and closing brackets are doubled. + */ + public function is_escaped(): bool { + return ( + self::TOKEN_SHORTCODE === $this->token_type && + $this->has_escaped_opening_bracket && + $this->has_escaped_closing_bracket + ); + } + + /** + * Returns the current token's byte offset. + * + * @return int|null Current token start, or null when not on a token. + */ + public function get_token_start(): ?int { + return $this->token_starts_at; + } + + /** + * Returns the current token's byte length. + * + * @return int|null Current token length, or null when not on a token. + */ + public function get_token_length(): ?int { + return $this->token_length; + } + + /** + * Advances to the next attribute on the current shortcode opener. + * + * Named and positional attributes are both reported. Positional attributes + * return null from get_attribute_name(). + * + * @return bool Whether another attribute was found. + */ + public function next_attribute(): bool { + if ( + self::TOKEN_SHORTCODE !== $this->token_type || + $this->is_closing_tag + ) { + return false; + } + + if ( $this->attribute_index + 1 >= count( $this->attributes ) ) { + $this->attribute_index = count( $this->attributes ); + return false; + } + + ++$this->attribute_index; + return true; + } + + /** + * Returns the current attribute name exactly as written. + * + * @return string|null Attribute name, or null for positional attributes and + * when not positioned on an attribute. + */ + public function get_attribute_name(): ?string { + $attribute = $this->get_current_attribute(); + return null === $attribute ? null : $attribute['name']; + } + + /** + * Returns the current attribute value without surrounding quotes. + * + * The value is returned as exact source bytes. Unlike shortcode_parse_atts(), + * this method does not call stripcslashes() or normalize whitespace. + * + * @return string|null Current attribute value. + */ + public function get_attribute_value(): ?string { + $attribute = $this->get_current_attribute(); + if ( null === $attribute ) { + return null; + } + + $update_key = $this->get_attribute_update_key( $this->attribute_index ); + if ( isset( $this->lexical_updates[ $update_key ]['value'] ) ) { + return $this->lexical_updates[ $update_key ]['value']; + } + + return substr( + $this->text, + $attribute['value_start'], + $attribute['value_length'] + ); + } + + /** + * Returns the quote delimiting the current attribute value. + * + * @return string|null Single quote, double quote, or null for an unquoted + * value and when not positioned on an attribute. + */ + public function get_attribute_quote(): ?string { + $attribute = $this->get_current_attribute(); + return null === $attribute ? null : $attribute['quote']; + } + + /** + * Returns the current attribute's byte offset. + * + * @return int|null Current attribute start. + */ + public function get_attribute_start(): ?int { + $attribute = $this->get_current_attribute(); + return null === $attribute ? null : $attribute['start']; + } + + /** + * Returns the current attribute's byte length. + * + * @return int|null Current attribute length. + */ + public function get_attribute_length(): ?int { + $attribute = $this->get_current_attribute(); + return null === $attribute ? null : $attribute['length']; + } + + /** + * Returns the last value of a named shortcode attribute. + * + * Attribute names are compared ASCII-case-insensitively, matching + * shortcode_parse_atts(). When an attribute is repeated, the last value + * wins. + * + * @param string $name Attribute name. + * @return string|null Attribute value, or null if not present. + */ + public function get_attribute( string $name ): ?string { + if ( + self::TOKEN_SHORTCODE !== $this->token_type || + $this->is_closing_tag + ) { + return null; + } + + $comparable_name = strtolower( $name ); + for ( $i = count( $this->attributes ) - 1; $i >= 0; --$i ) { + if ( $this->attributes[ $i ]['comparable_name'] !== $comparable_name ) { + continue; + } + + $update_key = $this->get_attribute_update_key( $i ); + if ( isset( $this->lexical_updates[ $update_key ]['value'] ) ) { + return $this->lexical_updates[ $update_key ]['value']; + } + + return substr( + $this->text, + $this->attributes[ $i ]['value_start'], + $this->attributes[ $i ]['value_length'] + ); + } + + return null; + } + + /** + * Returns unique lowercase attribute names matching a prefix. + * + * @param string $prefix Attribute-name prefix. + * @return array|null Matching names, or null when not on a shortcode opener. + */ + public function get_attribute_names_with_prefix( string $prefix ): ?array { + if ( + self::TOKEN_SHORTCODE !== $this->token_type || + $this->is_closing_tag + ) { + return null; + } + + $prefix = strtolower( $prefix ); + $matches = array(); + foreach ( $this->attributes as $attribute ) { + $name = $attribute['comparable_name']; + if ( + null !== $name && + 0 === strncmp( $name, $prefix, strlen( $prefix ) ) + ) { + $matches[ $name ] = true; + } + } + + return array_keys( $matches ); + } + + /** + * Replaces the current attribute value with minimal lexical changes. + * + * Existing quotes are retained when possible. An unquoted value gains + * quotes only when the replacement cannot be represented unquoted. The + * method refuses values containing both quote delimiters when quoting is + * required, because WordPress shortcode syntax has no reliable quote escape. + * + * @param string $value Replacement attribute value. + * @return bool Whether the update was enqueued. + */ + public function set_attribute_value( string $value ): bool { + $attribute = $this->get_current_attribute(); + if ( null === $attribute ) { + return false; + } + + $start = $attribute['value_start']; + $length = $attribute['value_length']; + $text = $value; + $quote = $attribute['quote']; + + if ( null !== $quote ) { + if ( false !== strpos( $value, $quote ) ) { + $alternate_quote = '"' === $quote ? "'" : '"'; + if ( false !== strpos( $value, $alternate_quote ) ) { + return false; + } + + $start = $attribute['value_start'] - 1; + $length = $attribute['value_length'] + 2; + $text = $alternate_quote . $value . $alternate_quote; + } + } else { + $requires_quotes = ! $this->can_use_unquoted_attribute_value( $value ); + if ( + ! $requires_quotes && + ! $this->has_self_closing_flag && + count( $this->attributes ) - 1 === $this->attribute_index && + '' !== $value && + '/' === $value[ strlen( $value ) - 1 ] + ) { + $requires_quotes = true; + } + + if ( $requires_quotes ) { + if ( false === strpos( $value, '"' ) ) { + $text = '"' . $value . '"'; + } elseif ( false === strpos( $value, "'" ) ) { + $text = "'" . $value . "'"; + } else { + return false; + } + } + } + + $this->lexical_updates[ $this->get_attribute_update_key( $this->attribute_index ) ] = array( + 'start' => $start, + 'length' => $length, + 'text' => $text, + 'value' => $value, + ); + + return true; + } + + /** + * Returns the exact text represented by the current text token. + * + * @return string|null Current text, or null on a shortcode token. + */ + public function get_modifiable_text(): ?string { + if ( self::TOKEN_TEXT !== $this->token_type ) { + return null; + } + + $update_key = $this->get_text_update_key(); + if ( isset( $this->lexical_updates[ $update_key ] ) ) { + return $this->lexical_updates[ $update_key ]['text']; + } + + return $this->get_token_text(); + } + + /** + * Replaces the current text token without applying HTML escaping. + * + * Text regions may contain CSS, HTML, block markup, another structured + * language, or ordinary prose. The caller owns that nested grammar, so this + * method records the supplied bytes exactly. + * + * @param string $text Replacement text. + * @return bool Whether the update was enqueued. + */ + public function set_modifiable_text( string $text ): bool { + if ( + self::TOKEN_TEXT !== $this->token_type || + null === $this->token_starts_at || + null === $this->token_length + ) { + return false; + } + + $this->lexical_updates[ $this->get_text_update_key() ] = array( + 'start' => $this->token_starts_at, + 'length' => $this->token_length, + 'text' => $text, + ); + + return true; + } + + /** + * Returns the source text with all enqueued lexical updates applied. + * + * @return string Updated text. + */ + public function get_updated_text(): string { + if ( empty( $this->lexical_updates ) ) { + return $this->text; + } + + $updates = array_values( $this->lexical_updates ); + usort( + $updates, + static function ( array $a, array $b ): int { + return $a['start'] - $b['start']; + } + ); + + $output = ''; + $bytes_already_copied = 0; + foreach ( $updates as $update ) { + if ( $update['start'] < $bytes_already_copied ) { + continue; + } + + $output .= substr( + $this->text, + $bytes_already_copied, + $update['start'] - $bytes_already_copied + ); + $output .= $update['text']; + + $bytes_already_copied = $update['start'] + $update['length']; + } + + $output .= substr( $this->text, $bytes_already_copied ); + + return $output; + } + + /** + * Returns the updated source text. + * + * @return string Updated text. + */ + public function __toString(): string { + return $this->get_updated_text(); + } + + /** + * Scans a shortcode candidate at a given byte offset. + * + * @param int $start Candidate start offset. + * @return array|false Parsed token metadata, or false for plain text. + * @phpstan-return array{ + * start: int, + * length: int, + * tag_name_start: int, + * tag_name_length: int, + * is_closing: bool, + * self_closing: bool, + * escaped_opening: bool, + * escaped_closing: bool, + * attributes: array + * }|false + */ + private function scan_shortcode_at( int $start ) { + $this->scan_at_after_failure = null; + + $at = $start + 1; + if ( $at >= $this->length ) { + return false; + } + + $escaped_opening = false; + if ( '[' === $this->text[ $at ] ) { + $escaped_opening = true; + ++$at; + } + + $is_closing = false; + if ( $at < $this->length && '/' === $this->text[ $at ] ) { + $is_closing = true; + ++$at; + } + + $tag_name_start = $at; + while ( + $at < $this->length && + $this->is_shortcode_tag_name_byte( $this->text[ $at ] ) + ) { + ++$at; + } + + $tag_name_length = $at - $tag_name_start; + if ( 0 === $tag_name_length ) { + return false; + } + + if ( + $at < $this->length && + ! $this->is_shortcode_whitespace_at( $at ) && + '"' !== $this->text[ $at ] && + "'" !== $this->text[ $at ] && + '/' !== $this->text[ $at ] && + ']' !== $this->text[ $at ] + ) { + return false; + } + + if ( $is_closing ) { + $at = $this->skip_shortcode_whitespace( $at, $this->length ); + if ( $at >= $this->length || ']' !== $this->text[ $at ] ) { + return false; + } + + ++$at; + $escaped_closing = $at < $this->length && ']' === $this->text[ $at ]; + if ( $escaped_closing ) { + ++$at; + } + + return array( + 'start' => $start, + 'length' => $at - $start, + 'tag_name_start' => $tag_name_start, + 'tag_name_length' => $tag_name_length, + 'is_closing' => true, + 'self_closing' => false, + 'escaped_opening' => $escaped_opening, + 'escaped_closing' => $escaped_closing, + 'attributes' => array(), + ); + } + + $attributes_start = $at; + $first_quote_at = null; + $quote = null; + while ( $at < $this->length ) { + $byte = $this->text[ $at ]; + + if ( null !== $quote ) { + if ( '\\' === $byte && $at + 1 < $this->length ) { + $at += 2; + continue; + } + + if ( $byte === $quote ) { + $quote = null; + } + + ++$at; + continue; + } + + if ( '"' === $byte || "'" === $byte ) { + if ( null === $first_quote_at ) { + $first_quote_at = $at; + } + $quote = $byte; + ++$at; + continue; + } + + if ( ']' !== $byte ) { + ++$at; + continue; + } + + $attributes_end = $at; + $self_closing = ( + $attributes_end > $attributes_start && + '/' === $this->text[ $attributes_end - 1 ] + ); + if ( $self_closing ) { + --$attributes_end; + } + + ++$at; + $escaped_closing = $at < $this->length && ']' === $this->text[ $at ]; + if ( $escaped_closing ) { + ++$at; + } + + return array( + 'start' => $start, + 'length' => $at - $start, + 'tag_name_start' => $tag_name_start, + 'tag_name_length' => $tag_name_length, + 'is_closing' => false, + 'self_closing' => $self_closing, + 'escaped_opening' => $escaped_opening, + 'escaped_closing' => $escaped_closing, + 'attributes' => $this->parse_attributes( + $attributes_start, + $attributes_end + ), + ); + } + + if ( null !== $first_quote_at ) { + /* + * No candidate before this quote can close before it: this scan would + * already have stopped at that closing bracket. Resume inside the + * quoted region so a standalone candidate there can still be recovered + * without rescanning the entire preceding suffix. + */ + $this->scan_at_after_failure = $first_quote_at + 1; + } + + return false; + } + + /** + * Parses attributes from a complete shortcode opener. + * + * @param int $start Attribute-list start offset. + * @param int $end Attribute-list end offset. + * @return array Parsed attribute metadata. + */ + private function parse_attributes( int $start, int $end ): array { + $attributes = array(); + $at = $start; + + while ( $at < $end ) { + $at = $this->skip_shortcode_whitespace( $at, $end ); + if ( $at >= $end ) { + break; + } + + $attribute_start = $at; + $quote = $this->text[ $at ]; + if ( '"' === $quote || "'" === $quote ) { + $value_start = $at + 1; + $value_end = $this->find_quote_end( $value_start, $end, $quote ); + if ( false === $value_end ) { + break; + } + + $at = $value_end + 1; + $attributes[] = array( + 'name' => null, + 'comparable_name' => null, + 'start' => $attribute_start, + 'length' => $at - $attribute_start, + 'value_start' => $value_start, + 'value_length' => $value_end - $value_start, + 'quote' => $quote, + ); + continue; + } + + $word_start = $at; + while ( + $at < $end && + ! $this->is_shortcode_whitespace_at( $at ) && + '=' !== $this->text[ $at ] && + '"' !== $this->text[ $at ] && + "'" !== $this->text[ $at ] + ) { + ++$at; + } + + $word_end = $at; + if ( $word_start === $word_end ) { + ++$at; + continue; + } + + $after_word = $this->skip_shortcode_whitespace( $at, $end ); + if ( $after_word >= $end || '=' !== $this->text[ $after_word ] ) { + $attributes[] = array( + 'name' => null, + 'comparable_name' => null, + 'start' => $attribute_start, + 'length' => $word_end - $attribute_start, + 'value_start' => $word_start, + 'value_length' => $word_end - $word_start, + 'quote' => null, + ); + + $at = $word_end; + continue; + } + + $name = substr( $this->text, $word_start, $word_end - $word_start ); + $at = $this->skip_shortcode_whitespace( $after_word + 1, $end ); + + if ( $at < $end && ( '"' === $this->text[ $at ] || "'" === $this->text[ $at ] ) ) { + $quote = $this->text[ $at ]; + $value_start = $at + 1; + $value_end = $this->find_quote_end( $value_start, $end, $quote ); + if ( false === $value_end ) { + break; + } + $at = $value_end + 1; + } else { + $quote = null; + $value_start = $at; + while ( $at < $end && ! $this->is_shortcode_whitespace_at( $at ) ) { + ++$at; + } + $value_end = $at; + } + + $attributes[] = array( + 'name' => $name, + 'comparable_name' => strtolower( $name ), + 'start' => $attribute_start, + 'length' => $at - $attribute_start, + 'value_start' => $value_start, + 'value_length' => $value_end - $value_start, + 'quote' => $quote, + ); + } + + return $attributes; + } + + /** + * Finds an attribute's closing quote. + * + * Backslash-quoted values are accepted as a builder-tolerant extension. The + * tokenizer does not decode them. + * + * @param int $start Value start offset. + * @param int $end Attribute-list end offset. + * @param string $quote Quote delimiter. + * @return int|false Closing quote offset, or false. + */ + private function find_quote_end( int $start, int $end, string $quote ) { + $at = $start; + while ( $at < $end ) { + if ( '\\' === $this->text[ $at ] && $at + 1 < $end ) { + $at += 2; + continue; + } + + if ( $quote === $this->text[ $at ] ) { + return $at; + } + + ++$at; + } + + return false; + } + + /** + * Applies parsed shortcode metadata as the current token. + * + * @param array $shortcode Parsed shortcode metadata. + */ + private function set_shortcode_token( array $shortcode ): void { + $this->token_type = self::TOKEN_SHORTCODE; + $this->token_starts_at = $shortcode['start']; + $this->token_length = $shortcode['length']; + $this->tag_name_starts_at = $shortcode['tag_name_start']; + $this->tag_name_length = $shortcode['tag_name_length']; + $this->is_closing_tag = $shortcode['is_closing']; + $this->has_self_closing_flag = $shortcode['self_closing']; + $this->has_escaped_opening_bracket = $shortcode['escaped_opening']; + $this->has_escaped_closing_bracket = $shortcode['escaped_closing']; + $this->attributes = $shortcode['attributes']; + } + + /** + * Clears state belonging to the previous token. + */ + private function after_token(): void { + $this->token_type = null; + $this->token_starts_at = null; + $this->token_length = null; + $this->tag_name_starts_at = null; + $this->tag_name_length = null; + $this->is_closing_tag = false; + $this->has_self_closing_flag = false; + $this->has_escaped_opening_bracket = false; + $this->has_escaped_closing_bracket = false; + $this->attributes = array(); + $this->attribute_index = -1; + } + + /** + * Returns the current attribute metadata. + * + * @return array|null Current attribute metadata. + */ + private function get_current_attribute(): ?array { + if ( + $this->attribute_index < 0 || + $this->attribute_index >= count( $this->attributes ) + ) { + return null; + } + + return $this->attributes[ $this->attribute_index ]; + } + + /** + * Returns a stable lexical-update key for an attribute. + * + * @param int $attribute_index Attribute index in the current token. + * @return string Update key. + */ + private function get_attribute_update_key( int $attribute_index ): string { + return 'attribute:' . $this->token_starts_at . ':' . $attribute_index; + } + + /** + * Returns a stable lexical-update key for the current text token. + * + * @return string Update key. + */ + private function get_text_update_key(): string { + return 'text:' . $this->token_starts_at; + } + + /** + * Checks whether a replacement can remain unquoted. + * + * @param string $value Replacement value. + * @return bool Whether the value can remain unquoted. + */ + private function can_use_unquoted_attribute_value( string $value ): bool { + if ( '' === $value || false !== strpbrk( $value, " \t\v\f\r\n\"'[]" ) ) { + return false; + } + + return ( + false === strpos( $value, "\u{00A0}" ) && + false === strpos( $value, "\u{200B}" ) + ); + } + + /** + * Checks whether a byte may appear in a practical shortcode tag name. + * + * Builder and WordPress shortcode tags conventionally use ASCII letters, + * digits, underscores, hyphens, colons, and periods. Non-ASCII bytes are + * accepted so the tokenizer does not split UTF-8 names. Restricting the + * ASCII punctuation prevents CSS attribute selectors such as `[href^=]` + * from being reported as shortcode tokens. + * + * @param string $byte Byte to inspect. + * @return bool Whether the byte can occur in a tag name. + */ + private function is_shortcode_tag_name_byte( string $byte ): bool { + $ord = ord( $byte ); + if ( $ord >= 0x80 ) { + return true; + } + + return ( + ( $ord >= 0x30 && $ord <= 0x39 ) || + ( $ord >= 0x41 && $ord <= 0x5A ) || + ( $ord >= 0x61 && $ord <= 0x7A ) || + '_' === $byte || + '-' === $byte || + ':' === $byte || + '.' === $byte + ); + } + + /** + * Checks for shortcode whitespace at a byte offset. + * + * The shortcode_parse_atts() function normalizes U+00A0 NO-BREAK SPACE and U+200B + * ZERO WIDTH SPACE to ordinary spaces before parsing. Treating them as + * separators here provides the same attribute boundaries without changing + * the source bytes. + * + * @param int $at Byte offset. + * @return bool Whether whitespace begins at the offset. + */ + private function is_shortcode_whitespace_at( int $at ): bool { + if ( $at >= $this->length ) { + return false; + } + + if ( false !== strpos( " \t\v\f\r\n", $this->text[ $at ] ) ) { + return true; + } + + return ( + 0 === substr_compare( $this->text, "\u{00A0}", $at, 2 ) || + 0 === substr_compare( $this->text, "\u{200B}", $at, 3 ) + ); + } + + /** + * Advances past shortcode whitespace. + * + * @param int $at Starting byte offset. + * @param int $end Exclusive end offset. + * @return int First non-whitespace byte offset. + */ + private function skip_shortcode_whitespace( int $at, int $end ): int { + while ( $at < $end ) { + $ascii_length = strspn( $this->text, " \t\v\f\r\n", $at, $end - $at ); + if ( $ascii_length > 0 ) { + $at += $ascii_length; + continue; + } + + if ( 0 === substr_compare( $this->text, "\u{00A0}", $at, 2 ) ) { + $at += 2; + continue; + } + + if ( 0 === substr_compare( $this->text, "\u{200B}", $at, 3 ) ) { + $at += 3; + continue; + } + + break; + } + + return $at; + } +} diff --git a/components/DataLiberation/Tests/BlockMarkupUrlProcessorShortcodeTest.php b/components/DataLiberation/Tests/BlockMarkupUrlProcessorShortcodeTest.php new file mode 100644 index 000000000..f0c923674 --- /dev/null +++ b/components/DataLiberation/Tests/BlockMarkupUrlProcessorShortcodeTest.php @@ -0,0 +1,192 @@ +assertSame( $expected_url_count, $this->replace_old_base_url( $processor ) ); + $updated = $processor->get_updated_html(); + + $this->assertSame( $expected, $updated ); + $this->assertStringNotContainsString( '&', $updated ); + $this->assertStringNotContainsString( '"', $updated ); + $this->assertStringNotContainsString( '<', $updated ); + $this->assertStringNotContainsString( ''', $updated ); + } + + public static function shortcode_css_url_provider(): array { + return array( + 'Divi CSS preserves raw text bytes while updating multiple URLs' => array( + <<<'HTML' + +[et_pb_section custom_css_main_element='.hero::before { content: "<&>"; background: url(https://old.example/media/hero.jpg?width=1200&height=800) no-repeat; mask: url("https://old.example/media/hero.jpg?width=1200&height=800"); }'] +[/et_pb_section] + +HTML + , + <<<'HTML' + +[et_pb_section custom_css_main_element='.hero::before { content: "<&>"; background: url("https://new.example/media/hero.jpg?width=1200&height=800") no-repeat; mask: url("https://new.example/media/hero.jpg?width=1200&height=800"); }'] +[/et_pb_section] + +HTML + , + 2, + ), + 'WPBakery CSS switches delimiters instead of encoding quotes' => array( + <<<'HTML' + +[vc_row css=".vc_custom_1 { background-image: url(https://old.example/media/hero.jpg?width=1200&height=800); }"] +[/vc_row] + +HTML + , + <<<'HTML' + +[vc_row css='.vc_custom_1 { background-image: url("https://new.example/media/hero.jpg?width=1200&height=800"); }'] +[/vc_row] + +HTML + , + 1, + ), + ); + } + + public function test_rewrites_block_html_and_shortcode_urls_in_one_pass(): void { + $input = <<<'HTML' + +
+ + +[et_pb_image src="https://old.example/shortcode.jpg?width=800&height=600"] + +HTML; + $processor = new BlockMarkupUrlProcessor( $input, 'https://old.example/' ); + + $this->assertSame( 3, $this->replace_old_base_url( $processor ) ); + $this->assertSame( + <<<'HTML' + +
+ + +[et_pb_image src="https://new.example/shortcode.jpg?width=800&height=600"] + +HTML + , + $processor->get_updated_html() + ); + } + + public function test_rewrites_direct_shortcode_urls_from_multiple_site_builders(): void { + $input = <<<'HTML' +[fusion_builder_container background_image="https://old.example/avada.jpg?width=1600&quality=80"] +[vc_video link="https://old.example/wpbakery.mp4?autoplay=1&muted=1"][/vc_video] +[themify_button link="https://old.example/themify?iframe=true&width=100%"]Button[/themify_button] +[x_image href="https://old.example/cornerstone?slide=1&from=builder"] +HTML; + $processor = new BlockMarkupUrlProcessor( $input, 'https://old.example/' ); + + $this->assertSame( 4, $this->replace_old_base_url( $processor ) ); + $this->assertSame( + <<<'HTML' +[fusion_builder_container background_image="https://new.example/avada.jpg?width=1600&quality=80"] +[vc_video link="https://new.example/wpbakery.mp4?autoplay=1&muted=1"][/vc_video] +[themify_button link="https://new.example/themify?iframe=true&width=100%"]Button[/themify_button] +[x_image href="https://new.example/cornerstone?slide=1&from=builder"] +HTML + , + $processor->get_updated_html() + ); + } + + public function test_preserves_text_node_span_across_incremental_shortcode_updates(): void { + $input = <<<'HTML' +[vc_row css=" + background: url(https://old.example/first.jpg?x=1&y=2); + mask: url(https://old.example/second.svg?x=3&y=4); +"] +HTML; + $processor = new BlockMarkupUrlProcessor( $input, 'https://old.example/' ); + $new_base_url = WPURL::parse( 'https://new-and-longer.example/migrated/' ); + + $this->assertTrue( $processor->next_url() ); + $this->assertSame( 'https://old.example/first.jpg?x=1&y=2', $processor->get_raw_url() ); + $this->assertTrue( $processor->replace_base_url( $new_base_url ) ); + $this->assertSame( + <<<'HTML' +[vc_row css=' + background: url("https://new-and-longer.example/migrated/first.jpg?x=1&y=2"); + mask: url(https://old.example/second.svg?x=3&y=4); +'] +HTML + , + $processor->get_updated_html() + ); + + $this->assertTrue( $processor->next_url() ); + $this->assertSame( 'https://old.example/second.svg?x=3&y=4', $processor->get_raw_url() ); + $this->assertTrue( $processor->replace_base_url( $new_base_url ) ); + $this->assertSame( + <<<'HTML' +[vc_row css=' + background: url("https://new-and-longer.example/migrated/first.jpg?x=1&y=2"); + mask: url("https://new-and-longer.example/migrated/second.svg?x=3&y=4"); +'] +HTML + , + $processor->get_updated_html() + ); + } + + public function test_only_interprets_shortcodes_in_html_text_nodes(): void { + $input = <<<'HTML' + +
+[[button url="https://old.example/escaped"]] +[button url="https://old.example/real?one=1&two=2"] +
+ +HTML; + $processor = new BlockMarkupUrlProcessor( $input, 'https://old.example/' ); + + $this->assertSame( 1, $this->replace_old_base_url( $processor ) ); + $this->assertSame( + <<<'HTML' + +
+[[button url="https://old.example/escaped"]] +[button url="https://new.example/real?one=1&two=2"] +
+ +HTML + , + $processor->get_updated_html() + ); + } + + private function replace_old_base_url( BlockMarkupUrlProcessor $processor ): int { + $new_base_url = WPURL::parse( 'https://new.example/' ); + $updated = 0; + + while ( $processor->next_url() ) { + $this->assertTrue( $processor->replace_base_url( $new_base_url ) ); + ++$updated; + } + + return $updated; + } +} diff --git a/components/DataLiberation/Tests/ShortcodeProcessorTest.php b/components/DataLiberation/Tests/ShortcodeProcessorTest.php new file mode 100644 index 000000000..86ad3995c --- /dev/null +++ b/components/DataLiberation/Tests/ShortcodeProcessorTest.php @@ -0,0 +1,1730 @@ +assertSame( + array( + array( '#text', 'Before ', null, false, false ), + array( '#shortcode', '[gallery ids="1,2"]', 'gallery', false, false ), + array( '#text', ' middle ', null, false, false ), + array( '#shortcode', '[/gallery]', 'gallery', true, false ), + array( '#text', ' after.', null, false, false ), + ), + $this->collect_tokens( $processor ) + ); + } + + public function test_reports_token_byte_offsets_for_utf8_content(): void { + $input = 'Zażółć [et_pb_image src="https://example.com/żółw.jpg"]'; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_token() ); + $this->assertSame( '#text', $processor->get_token_type() ); + $this->assertSame( 'Zażółć ', $processor->get_token_text() ); + $this->assertSame( 0, $processor->get_token_start() ); + $this->assertSame( strlen( 'Zażółć ' ), $processor->get_token_length() ); + + $this->assertTrue( $processor->next_token() ); + $this->assertSame( '#shortcode', $processor->get_token_type() ); + $this->assertSame( strlen( 'Zażółć ' ), $processor->get_token_start() ); + $this->assertSame( + strlen( '[et_pb_image src="https://example.com/żółw.jpg"]' ), + $processor->get_token_length() + ); + } + + public function test_recognizes_opening_closing_self_closing_and_escaped_tokens(): void { + $processor = new ShortcodeProcessor( + '[one][/one][two /][three/][[four url="https://example.com"]]' + ); + + $tokens = array(); + while ( $processor->next_shortcode() ) { + $tokens[] = array( + $processor->get_tag(), + $processor->is_tag_closer(), + $processor->has_self_closing_flag(), + $processor->is_escaped(), + ); + } + + $this->assertSame( + array( + array( 'one', false, false, false ), + array( 'one', true, false, false ), + array( 'two', false, true, false ), + array( 'three', false, true, false ), + array( 'four', false, false, true ), + ), + $tokens + ); + } + + public function test_tokenizes_same_name_nesting_without_matching_entire_enclosing_macros(): void { + $processor = new ShortcodeProcessor( + '[row level="1"][row level="2"]Inner[/row][/row]' + ); + + $tokens = array(); + while ( $processor->next_shortcode( 'row' ) ) { + $tokens[] = array( + $processor->is_tag_closer(), + $processor->get_attribute( 'level' ), + ); + } + + $this->assertSame( + array( + array( false, '1' ), + array( false, '2' ), + array( true, null ), + array( true, null ), + ), + $tokens + ); + } + + public function test_next_shortcode_filters_by_name_prefix_and_closer_policy(): void { + $processor = new ShortcodeProcessor( + '[gallery][et_pb_section][/et_pb_section][vc_row][/vc_row]' + ); + + $tags = array(); + while ( + $processor->next_shortcode( + array( + 'tag_prefix' => 'et_pb_', + 'tag_closers' => 'skip', + ) + ) + ) { + $tags[] = $processor->get_tag(); + } + + $this->assertSame( array( 'et_pb_section' ), $tags ); + } + + public function test_next_shortcode_supports_match_offset(): void { + $processor = new ShortcodeProcessor( '[item id="1"][item id="2"][item id="3"]' ); + + $this->assertTrue( + $processor->next_shortcode( + array( + 'tag_name' => 'item', + 'match_offset' => 2, + ) + ) + ); + $this->assertSame( '2', $processor->get_attribute( 'id' ) ); + } + + public function test_parses_named_positional_empty_duplicate_and_non_ascii_spaced_attributes(): void { + $processor = new ShortcodeProcessor( + "[demo one=\"a b\"\ttwo='c' three=3 empty=\"\" \"positional value\" bare" + . "\u{00A0}duplicate=first\u{200B}DUPLICATE=last]" + ); + + $this->assertTrue( $processor->next_shortcode( 'demo' ) ); + + $attributes = array(); + while ( $processor->next_attribute() ) { + $attributes[] = array( + 'name' => $processor->get_attribute_name(), + 'value' => $processor->get_attribute_value(), + 'quote' => $processor->get_attribute_quote(), + ); + } + + $this->assertSame( + array( + array( 'name' => 'one', 'value' => 'a b', 'quote' => '"' ), + array( 'name' => 'two', 'value' => 'c', 'quote' => "'" ), + array( 'name' => 'three', 'value' => '3', 'quote' => null ), + array( 'name' => 'empty', 'value' => '', 'quote' => '"' ), + array( 'name' => null, 'value' => 'positional value', 'quote' => '"' ), + array( 'name' => null, 'value' => 'bare', 'quote' => null ), + array( 'name' => 'duplicate', 'value' => 'first', 'quote' => null ), + array( 'name' => 'DUPLICATE', 'value' => 'last', 'quote' => null ), + ), + $attributes + ); + + $this->assertSame( 'last', $processor->get_attribute( 'duplicate' ) ); + $this->assertSame( + array( 'duplicate' ), + $processor->get_attribute_names_with_prefix( 'dup' ) + ); + } + + public function test_quoted_attribute_may_contain_css_html_json_and_brackets(): void { + $css = '.x[data-label="<"]::before {' + . ' content: "[still-css]";' + . ' background: url(https://old.example/zażółć%20gęślą.jpg);' + . ' }'; + $json = '{"selector":"section-1","media":{"phone":["one","two"]}}'; + $input = "[et_pb_section custom_css_main_element='" . $css . "'" + . " ct_options='" . $json . "']"; + + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'et_pb_section' ) ); + $this->assertSame( $css, $processor->get_attribute( 'custom_css_main_element' ) ); + $this->assertSame( $json, $processor->get_attribute( 'ct_options' ) ); + $this->assertSame( $input, $processor->get_updated_text() ); + } + + public function test_attribute_updates_preserve_unrelated_bytes_and_existing_quotes(): void { + $input = "[et_pb_section background_image = 'https://old.example/a.jpg'" + . ' custom_css_main_element=".x::before { content: \'<\'; }" ]'; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'et_pb_section' ) ); + while ( $processor->next_attribute() ) { + if ( 'background_image' === $processor->get_attribute_name() ) { + $this->assertTrue( + $processor->set_attribute_value( 'https://new.example/a.jpg' ) + ); + } + } + + $this->assertSame( + str_replace( 'https://old.example', 'https://new.example', $input ), + $processor->get_updated_text() + ); + } + + public function test_multiple_updates_are_applied_by_original_byte_offset(): void { + $input = '[one url="https://old.example/one"][two url=https://old.example/two]'; + $processor = new ShortcodeProcessor( $input ); + + while ( $processor->next_shortcode() ) { + while ( $processor->next_attribute() ) { + if ( 'url' === $processor->get_attribute_name() ) { + $processor->set_attribute_value( + str_replace( + 'https://old.example', + 'https://new.example', + $processor->get_attribute_value() + ) + ); + } + } + } + + $this->assertSame( + str_replace( 'https://old.example', 'https://new.example', $input ), + (string) $processor + ); + } + + public function test_updating_unquoted_attribute_adds_quotes_only_when_required(): void { + $processor = new ShortcodeProcessor( '[button url=https://old.example/a]' ); + + $this->assertTrue( $processor->next_shortcode( 'button' ) ); + $this->assertTrue( $processor->next_attribute() ); + $this->assertTrue( + $processor->set_attribute_value( 'https://new.example/a path' ) + ); + + $this->assertSame( + '[button url="https://new.example/a path"]', + $processor->get_updated_text() + ); + } + + public function test_attribute_update_refuses_value_that_cannot_be_quoted_safely(): void { + $input = '[demo value=original]'; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'demo' ) ); + $this->assertTrue( $processor->next_attribute() ); + $this->assertFalse( $processor->set_attribute_value( 'both " and \' with space' ) ); + $this->assertSame( $input, $processor->get_updated_text() ); + } + + public function test_text_update_is_byte_preserving_and_does_not_apply_html_escaping(): void { + $input = '.x::before {' + . ' content: "<";' + . ' background: url(https://old.example/a.jpg);' + . ' }'; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_token() ); + $this->assertSame( '#text', $processor->get_token_type() ); + $this->assertTrue( + $processor->set_modifiable_text( + str_replace( 'https://old.example', 'https://new.example', $input ) + ) + ); + + $this->assertSame( + str_replace( 'https://old.example', 'https://new.example', $input ), + $processor->get_updated_text() + ); + $this->assertStringNotContainsString( '"', $processor->get_updated_text() ); + $this->assertStringNotContainsString( '<', $processor->get_updated_text() ); + } + + public function test_shortcode_updates_do_not_claim_neighboring_block_html_or_css_regions(): void { + $shortcode = "[et_pb_section background_image='https://old.example/shortcode.jpg'" + . " custom_css_main_element='.hero::before { content: \"<\";" + . " background: url(https://old.example/attribute.jpg); }']"; + $input = '' + . '

HTML

' + . '' + . '' + . $shortcode + . '[/et_pb_section]' + . '' + . ''; + $processor = new ShortcodeProcessor( $input ); + + while ( $processor->next_shortcode() ) { + while ( $processor->next_attribute() ) { + $value = $processor->get_attribute_value(); + if ( false !== strpos( $value, 'https://old.example' ) ) { + $processor->set_attribute_value( + str_replace( 'https://old.example', 'https://new.example', $value ) + ); + } + } + } + + $expected = str_replace( + $shortcode, + str_replace( 'https://old.example', 'https://new.example', $shortcode ), + $input + ); + $this->assertSame( $expected, $processor->get_updated_text() ); + $this->assertStringContainsString( + 'href="https://old.example/html"', + $processor->get_updated_text() + ); + $this->assertStringContainsString( + 'url(https://old.example/style.jpg)', + $processor->get_updated_text() + ); + $this->assertStringContainsString( 'content: "<"', $processor->get_updated_text() ); + } + + /** + * @dataProvider builder_shortcode_provider + */ + public function test_tokenizes_real_builder_shapes( + string $builder, + string $input, + array $expected_tags + ): void { + $processor = new ShortcodeProcessor( $input ); + $actual = array(); + + while ( $processor->next_shortcode() ) { + $actual[] = ( $processor->is_tag_closer() ? '/' : '' ) . $processor->get_tag(); + } + + $this->assertSame( $expected_tags, $actual, $builder ); + } + + public static function builder_shortcode_provider(): array { + return array( + 'Divi 4 with HTML and a nested third-party shortcode' => array( + 'Divi 4', + '[et_pb_section background_image="https://old.example/hero.jpg"' + . ' custom_css_main_element=".x::before { content: \'<\'; }"]' + . '[et_pb_row][et_pb_column type="4_4"]' + . '[et_pb_text]

[contact-form-7 id="10"]

[/et_pb_text]' + . '[/et_pb_column][/et_pb_row][/et_pb_section]', + array( + 'et_pb_section', + 'et_pb_row', + 'et_pb_column', + 'et_pb_text', + 'contact-form-7', + '/et_pb_text', + '/et_pb_column', + '/et_pb_row', + '/et_pb_section', + ), + ), + 'Divi 5 block markup with a legacy Divi 4 region' => array( + 'Divi 5 legacy region', + '

Before

' + . '' + . '[et_pb_section][et_pb_row][/et_pb_row][/et_pb_section]' + . '', + array( + 'et_pb_section', + 'et_pb_row', + '/et_pb_row', + '/et_pb_section', + ), + ), + 'Gutenberg Shortcode block' => array( + 'Gutenberg', + '

Before

' + . '[gallery ids="1,2"]', + array( 'gallery' ), + ), + 'WPBakery nested layout with CSS and Base64 raw HTML' => array( + 'WPBakery', + '[vc_row css=".vc_custom_1{background:url(https://old.example/a.jpg);}"]' + . '[vc_column][vc_raw_html]JTNDcCUzRUhlbGxvJTNDJTJGcCUzRQ==' + . '[/vc_raw_html][/vc_column][/vc_row]', + array( + 'vc_row', + 'vc_column', + 'vc_raw_html', + '/vc_raw_html', + '/vc_column', + '/vc_row', + ), + ), + 'Oxygen legacy shortcode tree with JSON options' => array( + 'Oxygen', + "[ct_section ct_options='{\"selector\":\"section-1\",\"original\":{" + . "\"background-image\":\"https://old.example/a.jpg\"}}']" + . '[ct_code_block ct_options=\'{"code-css":"LmF7Y29sb3I6cmVkO30="}\']' + . '[/ct_code_block][/ct_section]', + array( + 'ct_section', + 'ct_code_block', + '/ct_code_block', + '/ct_section', + ), + ), + 'Avada Fusion hierarchy' => array( + 'Avada', + '[fusion_builder_container background_image="https://old.example/a.jpg"]' + . '[fusion_builder_row][fusion_builder_column type="1_1"]' + . '[fusion_text]

Text

[/fusion_text]' + . '[/fusion_builder_column][/fusion_builder_row]' + . '[/fusion_builder_container]', + array( + 'fusion_builder_container', + 'fusion_builder_row', + 'fusion_builder_column', + 'fusion_text', + '/fusion_text', + '/fusion_builder_column', + '/fusion_builder_row', + '/fusion_builder_container', + ), + ), + 'Themify nested columns' => array( + 'Themify', + '[themify_col grid="2-1 first"]' + . '[themify_button link="https://old.example/a?x=1&y=2"]Text[/themify_button]' + . '[/themify_col]', + array( + 'themify_col', + 'themify_button', + '/themify_button', + '/themify_col', + ), + ), + ); + } + + /** + * @dataProvider nested_container_provider + */ + public function test_outer_non_shortcode_formats_are_left_for_their_own_processors( + string $builder, + string $input + ): void { + $processor = new ShortcodeProcessor( $input ); + + $this->assertFalse( $processor->next_shortcode(), $builder ); + $this->assertSame( $input, $processor->get_updated_text(), $builder ); + } + + public static function nested_container_provider(): array { + return array( + 'Elementor JSON' => array( + 'Elementor', + '{"id":"6a637978","elType":"widget","widgetType":"image",' + . '"settings":{"url":"https://old.example/a.jpg"},"elements":[]}', + ), + 'Beaver Builder serialized PHP' => array( + 'Beaver Builder', + 'a:1:{s:3:"url";s:29:"https://old.example/image.jpg";}', + ), + 'SiteOrigin serialized PHP' => array( + 'SiteOrigin', + 'a:1:{s:7:"widgets";a:1:{i:0;a:1:{s:5:"image";' + . 's:29:"https://old.example/image.jpg";}}}', + ), + 'Divi 5 block data without legacy shortcodes' => array( + 'Divi 5', + '', + ), + 'Oxygen JSON' => array( + 'Oxygen', + '{"component":{"name":"ct_section","options":{"original":{' + . '"background-image":"https://old.example/a.jpg"}}}}', + ), + ); + } + + /** + * @dataProvider malformed_candidate_provider + */ + public function test_malformed_candidates_remain_plain_text( string $input ): void { + $processor = new ShortcodeProcessor( $input ); + + $this->assertFalse( $processor->next_shortcode() ); + $this->assertSame( $input, $processor->get_updated_text() ); + } + + public static function malformed_candidate_provider(): array { + return array( + 'empty brackets' => array( 'Before [] after' ), + 'only a closer marker' => array( 'Before [/] after' ), + 'truncated tag' => array( 'Before [et_pb_section' ), + 'unclosed quoted attribute' => array( + 'Before [et_pb_section title="unterminated] after', + ), + 'HTML closing tag' => array( 'Before after' ), + 'CSS attribute selector' => array( + '.x[href^="https://old.example"] { color: red; }', + ), + ); + } + + public function test_bracketed_identifiers_require_caller_context(): void { + $css = '.x[hidden] { color: red; }'; + + $unfiltered = new ShortcodeProcessor( $css ); + $this->assertTrue( $unfiltered->next_shortcode() ); + $this->assertSame( 'hidden', $unfiltered->get_tag() ); + + $builder_filtered = new ShortcodeProcessor( $css ); + $this->assertFalse( + $builder_filtered->next_shortcode( + array( + 'tag_prefix' => 'et_pb_', + ) + ) + ); + $this->assertSame( $css, $builder_filtered->get_updated_text() ); + } + + public function test_rewrites_url_inside_divi_css_attribute_without_html_encoding(): void { + $old_url = 'https://old.example/%C5%BC%C3%B3%C5%82%C4%87-g%C4%99%C5%9Bl%C4%85.jpg'; + $new_url = 'https://new.example/%C5%BC%C3%B3%C5%82%C4%87-g%C4%99%C5%9Bl%C4%85.jpg'; + $css = '.x::before { content: "<"; background: url(' . $old_url + . ') no-repeat center center fixed; }'; + $input = "[et_pb_section custom_css_main_element='" . $css . "']Hello[/et_pb_section]"; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'et_pb_section' ) ); + $this->assertTrue( $processor->next_attribute() ); + $this->assertSame( 'custom_css_main_element', $processor->get_attribute_name() ); + + $css_processor = new CSSURLProcessor( $processor->get_attribute_value() ); + $this->assertTrue( $css_processor->next_url() ); + $this->assertSame( $old_url, $css_processor->get_raw_url() ); + $this->assertTrue( $css_processor->set_raw_url( $new_url ) ); + + $updated_css = $css_processor->get_updated_css(); + $this->assertTrue( $processor->set_attribute_value( $updated_css ) ); + + $updated = $processor->get_updated_text(); + $expected_css = str_replace( + 'url(' . $old_url . ')', + 'url("' . $new_url . '")', + $css + ); + + $this->assertSame( str_replace( $css, $expected_css, $input ), $updated ); + // The CSS is not double-encoded the way an HTML text-node serializer would. + $this->assertStringContainsString( 'content: "<"', $updated ); + $this->assertStringNotContainsString( '"', $updated ); + $this->assertStringNotContainsString( '<', $updated ); + $this->assertStringNotContainsString( ''', $updated ); + // The closing quote is not slurped into the URL and percent-encoded. + $this->assertStringNotContainsString( '%22', $updated ); + $this->assertStringContainsString( + '") no-repeat center center fixed', + $updated + ); + $this->assertStringNotContainsString( $old_url, $updated ); + } + + public function test_shortcode_syntax_inside_a_quoted_attribute_is_not_a_separate_token(): void { + $processor = new ShortcodeProcessor( '[outer inner="[gallery ids=\'1,2\']"]text[/outer]' ); + + $tags = array(); + while ( $processor->next_shortcode() ) { + $tags[] = ( $processor->is_tag_closer() ? '/' : '' ) . $processor->get_tag(); + } + + $this->assertSame( array( 'outer', '/outer' ), $tags ); + } + + + /** + * @dataProvider attribute_update_round_trip_provider + */ + public function test_attribute_updates_round_trip_without_changing_token_semantics( + string $input, + string $attribute_name, + string $replacement, + string $expected + ): void { + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $was_self_closing = $processor->has_self_closing_flag(); + $this->assertTrue( $this->move_to_attribute( $processor, $attribute_name ) ); + $this->assertTrue( $processor->set_attribute_value( $replacement ) ); + $this->assertSame( $expected, $processor->get_updated_text() ); + + $reparsed = new ShortcodeProcessor( $processor->get_updated_text() ); + $this->assertTrue( $reparsed->next_shortcode( 'tag' ) ); + $this->assertSame( $was_self_closing, $reparsed->has_self_closing_flag() ); + $this->assertSame( $replacement, $reparsed->get_attribute( $attribute_name ) ); + } + + public static function attribute_update_round_trip_provider(): array { + return array( + 'terminal slash on final attribute is quoted' => array( + '[tag url=old]', + 'url', + 'https://example.com/', + '[tag url="https://example.com/"]', + ), + 'terminal slash before another attribute remains unquoted' => array( + '[tag url=old other=1]', + 'url', + 'https://example.com/', + '[tag url=https://example.com/ other=1]', + ), + 'terminal slash before an existing self-closing flag remains unquoted' => array( + '[tag url=old /]', + 'url', + 'https://example.com/', + '[tag url=https://example.com/ /]', + ), + 'empty replacement is quoted' => array( + '[tag value=old]', + 'value', + '', + '[tag value=""]', + ), + 'vertical tab requires quotes' => array( + '[tag value=old]', + 'value', + "one\vtwo", + "[tag value=\"one\vtwo\"]", + ), + 'ordinary whitespace requires quotes' => array( + '[tag value=old]', + 'value', + 'two words', + '[tag value="two words"]', + ), + 'tab requires quotes' => array( + '[tag value=old]', + 'value', + "one\ttwo", + "[tag value=\"one\ttwo\"]", + ), + 'form feed requires quotes' => array( + '[tag value=old]', + 'value', + "one\ftwo", + "[tag value=\"one\ftwo\"]", + ), + 'carriage return requires quotes' => array( + '[tag value=old]', + 'value', + "one\rtwo", + "[tag value=\"one\rtwo\"]", + ), + 'line feed requires quotes' => array( + '[tag value=old]', + 'value', + "one\ntwo", + "[tag value=\"one\ntwo\"]", + ), + 'opening bracket requires quotes' => array( + '[tag value=old]', + 'value', + 'one[two', + '[tag value="one[two"]', + ), + 'closing bracket requires quotes' => array( + '[tag value=old]', + 'value', + 'one]two', + '[tag value="one]two"]', + ), + 'double quote selects single-quoted representation' => array( + '[tag value=old]', + 'value', + 'say "hi"', + '[tag value=\'say "hi"\']', + ), + 'single quote selects double-quoted representation' => array( + '[tag value=old]', + 'value', + "it's", + '[tag value="it\'s"]', + ), + 'non-breaking space requires quotes' => array( + '[tag value=old]', + 'value', + "one\u{00A0}two", + "[tag value=\"one\u{00A0}two\"]", + ), + 'zero-width space requires quotes' => array( + '[tag value=old]', + 'value', + "one\u{200B}two", + "[tag value=\"one\u{200B}two\"]", + ), + ); + } + + public function test_positional_terminal_slash_update_preserves_value_and_token_type(): void { + $processor = new ShortcodeProcessor( '[tag old]' ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertTrue( $processor->next_attribute() ); + $this->assertNull( $processor->get_attribute_name() ); + $this->assertTrue( $processor->set_attribute_value( 'https://example.com/' ) ); + $this->assertSame( '[tag "https://example.com/"]', $processor->get_updated_text() ); + + $reparsed = new ShortcodeProcessor( $processor->get_updated_text() ); + $this->assertTrue( $reparsed->next_shortcode( 'tag' ) ); + $this->assertFalse( $reparsed->has_self_closing_flag() ); + $this->assertTrue( $reparsed->next_attribute() ); + $this->assertNull( $reparsed->get_attribute_name() ); + $this->assertSame( 'https://example.com/', $reparsed->get_attribute_value() ); + } + + public function test_realistic_backslash_quoted_css_with_both_quote_delimiters_is_refused(): void { + $input = '[tag css=".x::before { content: \\"it\'s\\";' + . ' background: url(https://old.example/a.jpg); }"]'; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertTrue( $this->move_to_attribute( $processor, 'css' ) ); + $updated_css = str_replace( + 'https://old.example', + 'https://new.example', + $processor->get_attribute_value() + ); + + $this->assertFalse( $processor->set_attribute_value( $updated_css ) ); + $this->assertSame( $input, $processor->get_updated_text() ); + } + + public function test_updated_attribute_value_is_logical_while_quote_and_spans_describe_source(): void { + $input = '[tag value="old"]'; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertTrue( $processor->next_attribute() ); + $source_start = $processor->get_attribute_start(); + $source_length = $processor->get_attribute_length(); + + $this->assertTrue( $processor->set_attribute_value( 'say "hi"' ) ); + $this->assertSame( 'say "hi"', $processor->get_attribute_value() ); + $this->assertSame( 'say "hi"', $processor->get_attribute( 'value' ) ); + $this->assertSame( '"', $processor->get_attribute_quote() ); + $this->assertSame( $source_start, $processor->get_attribute_start() ); + $this->assertSame( $source_length, $processor->get_attribute_length() ); + $this->assertSame( $input, $processor->get_token_text() ); + $this->assertSame( '[tag value=\'say "hi"\']', $processor->get_updated_text() ); + } + + public function test_last_successful_update_wins_and_a_later_refusal_preserves_it(): void { + $processor = new ShortcodeProcessor( '[tag value=old]' ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertTrue( $processor->next_attribute() ); + $this->assertTrue( $processor->set_attribute_value( 'first' ) ); + $this->assertTrue( $processor->set_attribute_value( 'second' ) ); + $this->assertSame( 'second', $processor->get_attribute_value() ); + $this->assertFalse( $processor->set_attribute_value( 'both " and \' with space' ) ); + $this->assertSame( 'second', $processor->get_attribute_value() ); + $this->assertSame( '[tag value=second]', $processor->get_updated_text() ); + } + + public function test_length_changing_text_and_attribute_updates_use_original_byte_offsets(): void { + $input = 'α[tag a=1 b="22"]MID[tag c=333]ω'; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_token() ); + $this->assertTrue( $processor->set_modifiable_text( 'A-MUCH-LONGER-PREFIX' ) ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertTrue( $this->move_to_attribute( $processor, 'a' ) ); + $this->assertTrue( $processor->set_attribute_value( '123456789' ) ); + $this->assertTrue( $this->move_to_attribute( $processor, 'b' ) ); + $this->assertTrue( $processor->set_attribute_value( '' ) ); + + $this->assertTrue( $processor->next_token() ); + $this->assertSame( 'MID', $processor->get_modifiable_text() ); + $this->assertTrue( $processor->set_modifiable_text( '' ) ); + $this->assertSame( + 'A-MUCH-LONGER-PREFIX[tag a=123456789 b=""][tag c=333]ω', + $processor->get_updated_text() + ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertTrue( $this->move_to_attribute( $processor, 'c' ) ); + $this->assertTrue( $processor->set_attribute_value( 'two words' ) ); + + $this->assertTrue( $processor->next_token() ); + $this->assertTrue( $processor->set_modifiable_text( 'TAIL' ) ); + + $expected = 'A-MUCH-LONGER-PREFIX[tag a=123456789 b=""]' + . '[tag c="two words"]TAIL'; + $this->assertSame( $expected, $processor->get_updated_text() ); + $this->assertSame( $expected, (string) $processor ); + $this->assertSame( $expected, $processor->get_updated_text() ); + } + + public function test_text_getters_distinguish_original_and_updated_bytes(): void { + $processor = new ShortcodeProcessor( 'original' ); + + $this->assertTrue( $processor->next_token() ); + $this->assertSame( 'original', $processor->get_token_text() ); + $this->assertSame( 'original', $processor->get_modifiable_text() ); + $this->assertTrue( $processor->set_modifiable_text( 'updated' ) ); + $this->assertSame( 'original', $processor->get_token_text() ); + $this->assertSame( 'updated', $processor->get_modifiable_text() ); + $this->assertSame( 'updated', $processor->get_updated_text() ); + } + + public function test_text_replacement_is_not_rescanned_in_the_current_pass(): void { + $processor = new ShortcodeProcessor( 'plain text' ); + + $this->assertTrue( $processor->next_token() ); + $this->assertTrue( $processor->set_modifiable_text( '[tag]' ) ); + $this->assertFalse( $processor->next_token() ); + $this->assertSame( '[tag]', $processor->get_updated_text() ); + + $reparsed = new ShortcodeProcessor( $processor->get_updated_text() ); + $this->assertTrue( $reparsed->next_shortcode( 'tag' ) ); + } + + public function test_setters_reject_the_wrong_token_and_attribute_states(): void { + $processor = new ShortcodeProcessor( 'text[tag value=1][/tag]' ); + + $this->assertTrue( $processor->next_token() ); + $this->assertFalse( $processor->set_attribute_value( 'nope' ) ); + $this->assertTrue( $processor->set_modifiable_text( 'updated text' ) ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertNull( $processor->get_modifiable_text() ); + $this->assertFalse( $processor->set_modifiable_text( 'nope' ) ); + $this->assertFalse( $processor->set_attribute_value( 'nope' ) ); + + $this->assertTrue( + $processor->next_shortcode( + array( + 'tag_name' => 'tag', + 'tag_closers' => 'visit', + ) + ) + ); + $this->assertTrue( $processor->is_tag_closer() ); + $this->assertFalse( $processor->set_attribute_value( 'nope' ) ); + $this->assertFalse( $processor->set_modifiable_text( 'nope' ) ); + } + + public function test_public_state_is_null_before_scanning_and_after_exhaustion(): void { + $processor = new ShortcodeProcessor( '' ); + + $this->assertNull( $processor->get_token_type() ); + $this->assertNull( $processor->get_token_text() ); + $this->assertNull( $processor->get_tag() ); + $this->assertFalse( $processor->is_tag_closer() ); + $this->assertFalse( $processor->has_self_closing_flag() ); + $this->assertFalse( $processor->is_escaped() ); + $this->assertNull( $processor->get_token_start() ); + $this->assertNull( $processor->get_token_length() ); + $this->assertNull( $processor->get_attribute_name() ); + $this->assertNull( $processor->get_attribute_value() ); + $this->assertNull( $processor->get_attribute_quote() ); + $this->assertNull( $processor->get_attribute_start() ); + $this->assertNull( $processor->get_attribute_length() ); + $this->assertNull( $processor->get_modifiable_text() ); + $this->assertNull( $processor->get_attribute_names_with_prefix( '' ) ); + $this->assertFalse( $processor->next_attribute() ); + $this->assertFalse( $processor->set_attribute_value( 'nope' ) ); + $this->assertFalse( $processor->set_modifiable_text( 'nope' ) ); + $this->assertFalse( $processor->next_token() ); + + $processor = new ShortcodeProcessor( '[tag]' ); + $this->assertTrue( $processor->next_token() ); + $this->assertFalse( $processor->set_modifiable_text( 'nope' ) ); + $this->assertFalse( $processor->next_token() ); + $this->assertNull( $processor->get_token_type() ); + $this->assertNull( $processor->get_token_text() ); + $this->assertNull( $processor->get_tag() ); + $this->assertNull( $processor->get_token_start() ); + $this->assertNull( $processor->get_token_length() ); + } + + public function test_failed_attribute_advancement_invalidates_the_attribute_cursor(): void { + $processor = new ShortcodeProcessor( '[tag one=1 two=2]' ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertTrue( $processor->next_attribute() ); + $this->assertTrue( $processor->next_attribute() ); + $this->assertSame( 'two', $processor->get_attribute_name() ); + $this->assertFalse( $processor->next_attribute() ); + $this->assertNull( $processor->get_attribute_name() ); + $this->assertNull( $processor->get_attribute_value() ); + $this->assertNull( $processor->get_attribute_quote() ); + $this->assertNull( $processor->get_attribute_start() ); + $this->assertNull( $processor->get_attribute_length() ); + $this->assertFalse( $processor->set_attribute_value( 'nope' ) ); + } + + public function test_attribute_and_token_offsets_are_absolute_byte_spans(): void { + $prefix = "\xEF\xBB\xBFZażółć 😀"; + $shortcode = "[tag a = 'żółw' ]"; + $input = $prefix . $shortcode . "\r\n"; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertSame( strlen( $prefix ), $processor->get_token_start() ); + $this->assertSame( strlen( $shortcode ), $processor->get_token_length() ); + $this->assertSame( $shortcode, $processor->get_token_text() ); + + $this->assertTrue( $processor->next_attribute() ); + $attribute_text = "a = 'żółw'"; + $attribute_at = strpos( $input, $attribute_text ); + $this->assertSame( $attribute_at, $processor->get_attribute_start() ); + $this->assertSame( strlen( $attribute_text ), $processor->get_attribute_length() ); + $this->assertSame( + $attribute_text, + substr( + $input, + $processor->get_attribute_start(), + $processor->get_attribute_length() + ) + ); + $this->assertSame( 'żółw', $processor->get_attribute_value() ); + + $this->assertTrue( $processor->next_token() ); + $this->assertSame( "\r\n", $processor->get_token_text() ); + $this->assertSame( strlen( $prefix . $shortcode ), $processor->get_token_start() ); + } + + public function test_attribute_name_prefix_queries_cover_empty_missing_duplicate_and_invalid_states(): void { + $processor = new ShortcodeProcessor( + '[tag DATA-One=1 data-two=2 data-one=3 bare][/tag]' + ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertSame( + array( 'data-one', 'data-two' ), + $processor->get_attribute_names_with_prefix( 'DATA-' ) + ); + $this->assertSame( + array( 'data-one', 'data-two' ), + $processor->get_attribute_names_with_prefix( '' ) + ); + $this->assertSame( array(), $processor->get_attribute_names_with_prefix( 'missing-' ) ); + $this->assertSame( '3', $processor->get_attribute( 'DATA-ONE' ) ); + + $this->assertTrue( + $processor->next_shortcode( + array( + 'tag_name' => 'tag', + 'tag_closers' => 'visit', + ) + ) + ); + $this->assertTrue( $processor->is_tag_closer() ); + $this->assertNull( $processor->get_attribute_names_with_prefix( '' ) ); + } + + public function test_query_filters_are_combined_before_applying_match_offset(): void { + $processor = new ShortcodeProcessor( + '[a id=1][/a][[a id=2]][a id=3][b id=4]' + ); + + $this->assertTrue( + $processor->next_shortcode( + array( + 'tag_name' => 'a', + 'tag_prefix' => 'a', + 'tag_closers' => 'skip', + 'escaped' => false, + 'match_offset' => 2, + ) + ) + ); + $this->assertSame( '[a id=3]', $processor->get_token_text() ); + $this->assertSame( '3', $processor->get_attribute( 'id' ) ); + } + + public function test_escaped_query_selects_only_complete_escaped_tokens(): void { + $processor = new ShortcodeProcessor( '[a][[a]][[a][a]]' ); + + $this->assertTrue( + $processor->next_shortcode( + array( + 'tag_name' => 'a', + 'escaped' => true, + ) + ) + ); + $this->assertSame( '[[a]]', $processor->get_token_text() ); + $this->assertTrue( $processor->is_escaped() ); + $this->assertFalse( + $processor->next_shortcode( + array( + 'tag_name' => 'a', + 'escaped' => true, + ) + ) + ); + } + + public function test_repeated_match_offset_queries_consume_each_skipped_match(): void { + $processor = new ShortcodeProcessor( + '[a id=1][a id=2][a id=3][a id=4][a id=5]' + ); + $query = array( + 'tag_name' => 'a', + 'match_offset' => 2, + ); + + $this->assertTrue( $processor->next_shortcode( $query ) ); + $this->assertSame( '2', $processor->get_attribute( 'id' ) ); + $this->assertTrue( $processor->next_shortcode( $query ) ); + $this->assertSame( '4', $processor->get_attribute( 'id' ) ); + $this->assertFalse( $processor->next_shortcode( $query ) ); + } + + public function test_nonmatching_query_consumes_the_stream(): void { + $processor = new ShortcodeProcessor( '[a][b]' ); + + $this->assertFalse( $processor->next_shortcode( 'missing' ) ); + $this->assertFalse( $processor->next_shortcode( 'a' ) ); + $this->assertNull( $processor->get_token_type() ); + } + + public function test_invalid_query_type_applies_no_filter(): void { + $processor = new ShortcodeProcessor( '[a]' ); + + $this->assertTrue( $processor->next_shortcode( 123 ) ); + $this->assertSame( 'a', $processor->get_tag() ); + } + + /** + * @dataProvider match_offset_coercion_provider + */ + public function test_match_offset_coercion_is_explicit( $match_offset, string $expected_id ): void { + $processor = new ShortcodeProcessor( '[a id=1][a id=2][a id=3]' ); + + $this->assertTrue( + $processor->next_shortcode( + array( + 'tag_name' => 'a', + 'match_offset' => $match_offset, + ) + ) + ); + $this->assertSame( $expected_id, $processor->get_attribute( 'id' ) ); + } + + public static function match_offset_coercion_provider(): array { + return array( + 'zero falls back to first' => array( 0, '1' ), + 'negative falls back to first' => array( -2, '1' ), + 'numeric string is cast' => array( '2', '2' ), + 'float is truncated' => array( 2.9, '2' ), + ); + } + + public function test_tag_queries_are_case_sensitive_and_name_prefix_constraints_intersect(): void { + $processor = new ShortcodeProcessor( '[A][a][alpha]' ); + + $this->assertTrue( $processor->next_shortcode( 'a' ) ); + $this->assertSame( '[a]', $processor->get_token_text() ); + + $processor = new ShortcodeProcessor( '[a][alpha]' ); + $this->assertFalse( + $processor->next_shortcode( + array( + 'tag_name' => 'a', + 'tag_prefix' => 'b', + ) + ) + ); + } + + public function test_registry_dependent_period_and_colon_boundaries_use_the_longest_practical_name(): void { + $processor = new ShortcodeProcessor( '[foo.bar][foo:bar]' ); + + $this->assertFalse( $processor->next_shortcode( 'foo' ) ); + + $processor = new ShortcodeProcessor( '[foo.bar][foo:bar]' ); + $this->assertTrue( $processor->next_shortcode( 'foo.bar' ) ); + $this->assertSame( '[foo.bar]', $processor->get_token_text() ); + $this->assertTrue( $processor->next_shortcode( 'foo:bar' ) ); + $this->assertSame( '[foo:bar]', $processor->get_token_text() ); + } + + public function test_unknown_closer_policy_uses_default_visit_behavior(): void { + $processor = new ShortcodeProcessor( '[/a][a]' ); + + $this->assertTrue( + $processor->next_shortcode( + array( + 'tag_closers' => 'unknown', + ) + ) + ); + $this->assertTrue( $processor->is_tag_closer() ); + } + + /** + * @dataProvider escaped_token_provider + */ + public function test_bracket_runs_have_explicit_token_and_escape_semantics( + string $input, + array $expected + ): void { + $this->assertSame( + $expected, + $this->collect_detailed_tokens( new ShortcodeProcessor( $input ) ) + ); + } + + public static function escaped_token_provider(): array { + return array( + 'complete escaping' => array( + '[[tag]]', + array( + array( '#shortcode', '[[tag]]', 'tag', false, false, true ), + ), + ), + 'triple brackets leave surrounding text' => array( + '[[[tag]]]', + array( + array( '#text', '[', null, false, false, false ), + array( '#shortcode', '[[tag]]', 'tag', false, false, true ), + array( '#text', ']', null, false, false, false ), + ), + ), + 'opening bracket doubled only' => array( + '[[tag]', + array( + array( '#shortcode', '[[tag]', 'tag', false, false, false ), + ), + ), + 'closing bracket doubled only' => array( + '[tag]]', + array( + array( '#shortcode', '[tag]]', 'tag', false, false, false ), + ), + ), + 'escaped self-closing token' => array( + '[[tag /]]', + array( + array( '#shortcode', '[[tag /]]', 'tag', false, true, true ), + ), + ), + 'escaped closing token' => array( + '[[/tag]]', + array( + array( '#shortcode', '[[/tag]]', 'tag', true, false, true ), + ), + ), + ); + } + + public function test_core_style_escaped_enclosing_shortcode_is_not_a_complete_escaped_token(): void { + $input = '[[outer url=old]content[/outer]]'; + $processor = new ShortcodeProcessor( $input ); + + $this->assertSame( + array( + array( '#shortcode', '[[outer url=old]', 'outer', false, false, false ), + array( '#text', 'content', null, false, false, false ), + array( '#shortcode', '[/outer]]', 'outer', true, false, false ), + ), + $this->collect_detailed_tokens( $processor ) + ); + + $escaped = new ShortcodeProcessor( $input ); + $this->assertFalse( + $escaped->next_shortcode( + array( + 'escaped' => true, + ) + ) + ); + + $active = new ShortcodeProcessor( $input ); + $this->assertTrue( + $active->next_shortcode( + array( + 'escaped' => false, + ) + ) + ); + $this->assertSame( '[[outer url=old]', $active->get_token_text() ); + } + + /** + * @dataProvider practical_tag_name_provider + */ + public function test_practical_tag_name_subset_is_explicit( + string $input, + ?string $expected_tag + ): void { + $processor = new ShortcodeProcessor( $input ); + + if ( null === $expected_tag ) { + $this->assertFalse( $processor->next_shortcode() ); + $this->assertSame( $input, $processor->get_updated_text() ); + return; + } + + $this->assertTrue( $processor->next_shortcode() ); + $this->assertSame( $expected_tag, $processor->get_tag() ); + } + + public static function practical_tag_name_provider(): array { + return array( + 'plain ASCII' => array( '[plain]', 'plain' ), + 'numeric' => array( '[123]', '123' ), + 'colon' => array( '[namespace:tag]', 'namespace:tag' ), + 'period' => array( '[tag.name]', 'tag.name' ), + 'non-ASCII' => array( '[żółw]', 'żółw' ), + 'hyphen' => array( '[contact-form-7]', 'contact-form-7' ), + 'Core-valid exclamation rejected' => array( '[good!]', null ), + 'Core-valid punctuation rejected' => array( + '[unreserved!#$%()*+,-.;?@^_{|}~chars]', + null, + ), + 'legacy equals syntax rejected' => array( + '[tag=https://wordpress.org/]', + null, + ), + ); + } + + /** + * @dataProvider self_closing_syntax_provider + */ + public function test_self_closing_syntax_matches_the_immediate_solidus_rule( + string $input, + bool $expected_self_closing, + array $expected_positional_attributes + ): void { + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertSame( $expected_self_closing, $processor->has_self_closing_flag() ); + + $actual = array(); + while ( $processor->next_attribute() ) { + if ( null === $processor->get_attribute_name() ) { + $actual[] = $processor->get_attribute_value(); + } + } + $this->assertSame( $expected_positional_attributes, $actual ); + } + + public static function self_closing_syntax_provider(): array { + return array( + 'no whitespace' => array( '[tag/]', true, array() ), + 'whitespace before solidus' => array( '[tag /]', true, array() ), + 'whitespace after solidus' => array( '[tag / ]', false, array( '/' ) ), + 'newline after solidus' => array( "[tag /\n]", false, array( '/' ) ), + 'non-breaking space after solidus' => array( + "[tag /\u{00A0}]", + false, + array( '/' ), + ), + 'one positional solidus plus flag' => array( '[tag //]', true, array( '/' ) ), + 'solidus inside positional value' => array( + '[tag /path]', + false, + array( '/path' ), + ), + ); + } + + public function test_closing_tokens_tolerate_whitespace_but_reject_attributes(): void { + $processor = new ShortcodeProcessor( "[/tag ][/tag\n][/tag attr=bad]" ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertSame( '[/tag ]', $processor->get_token_text() ); + $this->assertTrue( $processor->is_tag_closer() ); + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertSame( "[/tag\n]", $processor->get_token_text() ); + $this->assertTrue( $processor->is_tag_closer() ); + $this->assertFalse( $processor->next_shortcode( 'tag' ) ); + } + + public function test_all_shortcode_whitespace_bytes_separate_attributes(): void { + $input = "[tag a=1 b=2\tc=3\fd=4\re=5\nf=6\vg=7" + . "\u{00A0}h=8\u{200B}i=9]"; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + foreach ( range( 'a', 'i' ) as $index => $name ) { + $this->assertSame( (string) ( $index + 1 ), $processor->get_attribute( $name ) ); + } + } + + /** + * @dataProvider tolerant_attribute_contract_provider + */ + public function test_tolerant_attribute_contracts_are_explicit( + string $input, + array $expected + ): void { + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $actual = array(); + while ( $processor->next_attribute() ) { + $actual[] = array( + $processor->get_attribute_name(), + $processor->get_attribute_value(), + $processor->get_attribute_quote(), + ); + } + $this->assertSame( $expected, $actual ); + } + + public static function tolerant_attribute_contract_provider(): array { + return array( + 'hyphenated names' => array( + '[tag -foo=x foo-=y foo--bar=z]', + array( + array( '-foo', 'x', null ), + array( 'foo-', 'y', null ), + array( 'foo--bar', 'z', null ), + ), + ), + 'zero name and values' => array( + '[tag 0=x zero=0 0 "0"]', + array( + array( '0', 'x', null ), + array( 'zero', '0', null ), + array( null, '0', null ), + array( null, '0', '"' ), + ), + ), + 'punctuation in attribute names' => array( + '[tag foo.bar=x foo:bar=y @x=z]', + array( + array( 'foo.bar', 'x', null ), + array( 'foo:bar', 'y', null ), + array( '@x', 'z', null ), + ), + ), + 'adjacent quoted attributes' => array( + '[tag a="1"b="2"]', + array( + array( 'a', '1', '"' ), + array( 'b', '2', '"' ), + ), + ), + 'named empty value' => array( + '[tag a=]', + array( + array( 'a', '', null ), + ), + ), + 'quoted named and positional empty values' => array( + '[tag empty="" \'\' ""]', + array( + array( 'empty', '', '"' ), + array( null, '', "'" ), + array( null, '', '"' ), + ), + ), + ); + } + + public function test_backslash_quoted_values_are_returned_without_decoding(): void { + $input = "[tag value=\"one\\\"two\\\\three\" other=3]"; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertSame( "one\\\"two\\\\three", $processor->get_attribute( 'value' ) ); + $this->assertSame( '3', $processor->get_attribute( 'other' ) ); + $this->assertSame( $input, $processor->get_updated_text() ); + } + + public function test_single_quoted_backslash_value_is_returned_without_decoding(): void { + $input = "[tag value='one\\'two\\\\three' other=3]"; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertSame( "one\\'two\\\\three", $processor->get_attribute( 'value' ) ); + $this->assertSame( '3', $processor->get_attribute( 'other' ) ); + $this->assertSame( $input, $processor->get_updated_text() ); + } + + public function test_malformed_candidate_before_valid_shortcode_remains_in_text(): void { + $input = 'before [bad?] middle [good] after'; + $processor = new ShortcodeProcessor( $input ); + + $this->assertSame( + array( + array( '#text', 'before [bad?] middle ', null, false, false, false ), + array( '#shortcode', '[good]', 'good', false, false, false ), + array( '#text', ' after', null, false, false, false ), + ), + $this->collect_detailed_tokens( $processor ) + ); + } + + public function test_unclosed_quote_before_valid_shortcode_does_not_hide_the_later_token(): void { + $input = 'before [bad value="unterminated] middle [good] after'; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'good' ) ); + $this->assertSame( '[good]', $processor->get_token_text() ); + } + + public function test_candidate_inside_failed_matched_quote_region_is_recovered(): void { + $input = 'before [bad value="[good]"'; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'good' ) ); + $this->assertSame( '[good]', $processor->get_token_text() ); + } + + public function test_mismatched_unbalanced_and_raw_content_tokens_remain_independent(): void { + $processor = new ShortcodeProcessor( + '[a][b][/a][/c][raw]const flags = [true];[/raw]' + ); + $actual = array(); + + while ( $processor->next_shortcode() ) { + $actual[] = ( $processor->is_tag_closer() ? '/' : '' ) . $processor->get_tag(); + } + + $this->assertSame( + array( 'a', 'b', '/a', '/c', 'raw', 'true', '/raw' ), + $actual + ); + } + + /** + * @dataProvider context_ambiguity_provider + */ + public function test_context_free_bracket_ambiguities_are_explicit( + string $input, + array $expected_tags + ): void { + $processor = new ShortcodeProcessor( $input ); + $actual = array(); + + while ( $processor->next_shortcode() ) { + $actual[] = ( $processor->is_tag_closer() ? '/' : '' ) . $processor->get_tag(); + } + $this->assertSame( $expected_tags, $actual ); + + $builder_filtered = new ShortcodeProcessor( $input ); + $this->assertFalse( + $builder_filtered->next_shortcode( + array( + 'tag_prefix' => 'et_pb_', + ) + ) + ); + $this->assertSame( $input, $builder_filtered->get_updated_text() ); + } + + public static function context_ambiguity_provider(): array { + return array( + 'CSS selector' => array( + '.x[hidden] { color: red; }', + array( 'hidden' ), + ), + 'CSS string' => array( + '.x::before { content: "[gallery]"; }', + array( 'gallery' ), + ), + 'JSON arrays and string' => array( + '{"flags":[true],"count":[1],"label":"[gallery]"}', + array( 'true', '1', 'gallery' ), + ), + 'JavaScript array' => array( + '', + array( 'true' ), + ), + 'conditional HTML comment' => array( + '', + array( 'if', 'endif' ), + ), + 'Markdown link' => array( + '[label](https://example.com)', + array( 'label' ), + ), + 'regular expression character class' => array( + '/[a-z]/', + array( 'a-z' ), + ), + 'serialized PHP string' => array( + 'a:1:{s:4:"text";s:9:"[gallery]";}', + array( 'gallery' ), + ), + 'Gutenberg block JSON' => array( + '', + array( 'gallery' ), + ), + 'HTML attribute' => array( + 'Link', + array( 'url' ), + ), + ); + } + + /** + * @dataProvider builder_attribute_provider + */ + public function test_builder_attributes_can_be_rewritten_with_unequal_length_values( + string $input, + string $tag, + string $attribute_name, + string $old_value, + string $new_value + ): void { + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( $tag ) ); + $this->assertSame( $old_value, $processor->get_attribute( $attribute_name ) ); + $this->assertTrue( $this->move_to_attribute( $processor, $attribute_name ) ); + $this->assertTrue( $processor->set_attribute_value( $new_value ) ); + + $expected = str_replace( $old_value, $new_value, $input ); + $this->assertSame( $expected, $processor->get_updated_text() ); + + $reparsed = new ShortcodeProcessor( $processor->get_updated_text() ); + $this->assertTrue( $reparsed->next_shortcode( $tag ) ); + $this->assertSame( $new_value, $reparsed->get_attribute( $attribute_name ) ); + } + + public static function builder_attribute_provider(): array { + $old_url = 'https://old.example/a.jpg'; + $new_url = 'https://new-and-longer.example/assets/a.jpg'; + + $divi_old = '.x[data-state="open"] { background:url(' . $old_url . '); }'; + $divi_new = str_replace( $old_url, $new_url, $divi_old ); + + $wpbakery_old = ".x[data-state='open'] { background:url(" . $old_url . '); }'; + $wpbakery_new = str_replace( $old_url, $new_url, $wpbakery_old ); + + $oxygen_old = '{"selector":"section-1","background-image":"' . $old_url . '"}'; + $oxygen_new = str_replace( $old_url, $new_url, $oxygen_old ); + + return array( + 'Divi CSS attribute' => array( + "[et_pb_section custom_css_main_element='" . $divi_old . "']", + 'et_pb_section', + 'custom_css_main_element', + $divi_old, + $divi_new, + ), + 'WPBakery CSS attribute' => array( + '[vc_row css="' . $wpbakery_old . '"]', + 'vc_row', + 'css', + $wpbakery_old, + $wpbakery_new, + ), + 'Oxygen JSON attribute' => array( + "[ct_section ct_options='" . $oxygen_old . "']", + 'ct_section', + 'ct_options', + $oxygen_old, + $oxygen_new, + ), + 'Avada URL attribute' => array( + '[fusion_builder_container background_image="' . $old_url . '"]', + 'fusion_builder_container', + 'background_image', + $old_url, + $new_url, + ), + 'Themify URL attribute' => array( + '[themify_button link="' . $old_url . '?x=1&y=2"]', + 'themify_button', + 'link', + $old_url . '?x=1&y=2', + $new_url . '?x=1&y=2', + ), + ); + } + + /** + * @dataProvider token_stream_invariant_provider + */ + public function test_token_stream_spans_cover_every_original_byte( string $input ): void { + $processor = new ShortcodeProcessor( $input ); + $at = 0; + $rebuilt = ''; + + while ( $processor->next_token() ) { + $this->assertSame( $at, $processor->get_token_start() ); + $this->assertGreaterThan( 0, $processor->get_token_length() ); + $this->assertSame( + substr( $input, $processor->get_token_start(), $processor->get_token_length() ), + $processor->get_token_text() + ); + $rebuilt .= $processor->get_token_text(); + $at += $processor->get_token_length(); + } + + $this->assertSame( strlen( $input ), $at ); + $this->assertSame( $input, $rebuilt ); + $this->assertSame( $input, $processor->get_updated_text() ); + $this->assertNull( $processor->get_token_type() ); + } + + public static function token_stream_invariant_provider(): array { + return array( + 'empty' => array( '' ), + 'plain text' => array( 'plain text' ), + 'mixed valid tokens' => array( 'α[tag a="żółw"]MID[/tag]ω' ), + 'malformed before valid' => array( '[bad?] before [good] after' ), + 'escaped and partial escaped' => array( '[[tag]][[tag][tag]]' ), + 'CRLF and Unicode whitespace' => array( + "before\r\n[tag\u{00A0}a=1\u{200B}b=2]\r\nafter", + ), + 'binary bytes' => array( "\xFFbefore[tag value=\"\xFE\"]after\x00" ), + 'context ambiguity' => array( '{"flags":[true],"label":"[gallery]"}' ), + ); + } + + public function test_short_fuzz_corpus_preserves_every_input_byte(): void { + $alphabet = array( '[', ']', '/', '"', "'", '=', 'a', ' ', '\\', "\0", "\xFF" ); + + foreach ( $alphabet as $first ) { + foreach ( $alphabet as $second ) { + foreach ( $alphabet as $third ) { + $input = $first . $second . $third; + $processor = new ShortcodeProcessor( $input ); + $at = 0; + $rebuilt = ''; + + while ( $processor->next_token() ) { + $this->assertSame( $at, $processor->get_token_start() ); + $rebuilt .= $processor->get_token_text(); + $at += $processor->get_token_length(); + } + + $this->assertSame( strlen( $input ), $at ); + $this->assertSame( $input, $rebuilt ); + $this->assertSame( $input, $processor->get_updated_text() ); + } + } + } + } + + public function test_large_quoted_attribute_with_many_brackets_is_one_linear_token(): void { + $value = str_repeat( '[not-a-token]', 5000 ); + $input = '[tag value="' . $value . '"]'; + $processor = new ShortcodeProcessor( $input ); + + $this->assertTrue( $processor->next_shortcode( 'tag' ) ); + $this->assertSame( $input, $processor->get_token_text() ); + $this->assertSame( $value, $processor->get_attribute( 'value' ) ); + $this->assertFalse( $processor->next_shortcode() ); + } + + /** + * @dataProvider malformed_candidate_storm_provider + */ + public function test_malformed_candidate_storm_does_not_rescan_the_remaining_suffix( + string $input + ): void { + $started = microtime( true ); + $processor = new ShortcodeProcessor( $input ); + + $this->assertFalse( $processor->next_shortcode() ); + $elapsed = microtime( true ) - $started; + + $this->assertLessThan( + 2.0, + $elapsed, + 'Malformed candidates should be processed in approximately linear time.' + ); + $this->assertSame( $input, $processor->get_updated_text() ); + } + + public static function malformed_candidate_storm_provider(): array { + $prefix = str_repeat( '[a ', 12000 ); + + return array( + 'no closing bracket' => array( $prefix ), + 'closing bracket inside an unmatched quote' => array( $prefix . '"]' ), + 'closing bracket inside a matched quoted region' => array( $prefix . '"]"' ), + ); + } + + private function move_to_attribute( + ShortcodeProcessor $processor, + string $attribute_name + ): bool { + while ( $processor->next_attribute() ) { + if ( $attribute_name === $processor->get_attribute_name() ) { + return true; + } + } + + return false; + } + + private function collect_detailed_tokens( ShortcodeProcessor $processor ): array { + $tokens = array(); + + while ( $processor->next_token() ) { + $tokens[] = array( + $processor->get_token_type(), + $processor->get_token_text(), + $processor->get_tag(), + $processor->is_tag_closer(), + $processor->has_self_closing_flag(), + $processor->is_escaped(), + ); + } + + return $tokens; + } + private function collect_tokens( ShortcodeProcessor $processor ): array { + $tokens = array(); + + while ( $processor->next_token() ) { + $tokens[] = array( + $processor->get_token_type(), + $processor->get_token_text(), + $processor->get_tag(), + $processor->is_tag_closer(), + $processor->is_escaped(), + ); + } + + return $tokens; + } +} diff --git a/components/HTML/PHP/class-wp-html-php-tag-processor.php b/components/HTML/PHP/class-wp-html-php-tag-processor.php index d35ecc346..74743920f 100644 --- a/components/HTML/PHP/class-wp-html-php-tag-processor.php +++ b/components/HTML/PHP/class-wp-html-php-tag-processor.php @@ -3805,6 +3805,57 @@ static function ( $tag_match ) { return false; } + /** + * Returns the exact source bytes for the current text node. + * + * This bypasses HTML character-reference decoding and input-stream + * normalization. It is intended for subclasses that hand the text node to + * another parser whose grammar owns those bytes. + * + * @return string Raw text-node bytes, or an empty string on another token. + */ + protected function get_modifiable_text_raw(): string { + if ( self::STATE_TEXT_NODE !== $this->parser_state ) { + return ''; + } + + if ( isset( $this->lexical_updates['modifiable text'] ) ) { + return $this->lexical_updates['modifiable text']->text; + } + + return substr( $this->html, $this->text_starts_at, $this->text_length ); + } + + /** + * Replaces the current text node with exact source bytes. + * + * Unlike set_modifiable_text(), this method performs no HTML escaping. A + * subclass must only use it after a nested parser has preserved the text + * node's grammar and changed a safe lexical span within it. + * + * @param string $raw_content Replacement source bytes. + * @return bool Whether the text was able to update. + */ + protected function set_modifiable_text_raw( string $raw_content ): bool { + if ( self::STATE_TEXT_NODE !== $this->parser_state ) { + return false; + } + + $replaced_length = isset( $this->lexical_updates['modifiable text'] ) + ? $this->lexical_updates['modifiable text']->length + : $this->text_length; + + $this->lexical_updates['modifiable text'] = new WP_HTML_Text_Replacement( + $this->text_starts_at, + $replaced_length, + $raw_content + ); + $this->text_length = strlen( $raw_content ); + $this->token_length = $this->text_length; + + return true; + } + /** * Updates or creates a new attribute on the currently matched tag with the passed value. *