diff --git a/.github/workflows/update_top_level_domains.yml b/.github/workflows/update_top_level_domains.yml new file mode 100644 index 00000000..1d77eae1 --- /dev/null +++ b/.github/workflows/update_top_level_domains.yml @@ -0,0 +1,40 @@ +name: update_top_level_domains + +on: + schedule: + # Runs "At 06:00 on the 1st of every month" + - cron: '0 6 1 * *' + workflow_dispatch: + +permissions: + contents: read + +jobs: + update_top_level_domains: + runs-on: ubuntu-latest + steps: + - name: 📚 Git Checkout + uses: actions/checkout@v6 + + - name: 🎯 Install Dart + uses: dart-lang/setup-dart@v1 + + # Leaves the file as it is when the list of domains has not changed, so + # no pull request is opened then. + - name: 🌐 Regenerate Top-Level Domains + run: dart run scripts/generate_top_level_domains.dart + + # Opened with the bot's token so the pull request's checks run, which + # they don't for one opened with the workflow's own token. + - name: 📤 Open Pull Request + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.BOT_GITHUB_API_TOKEN }} + branch: chore/update-top-level-domains + delete-branch: true + commit-message: 'chore(repo): update the IANA top-level domains' + title: 'chore(repo): update the IANA top-level domains' + body: | + Regenerates the top-level domains `findLinks` recognizes from the IANA root zone list with `melos run generate:tlds`. + committer: Stream SDK Bot <60655709+Stream-SDK-Bot@users.noreply.github.com> + author: Stream SDK Bot <60655709+Stream-SDK-Bot@users.noreply.github.com> diff --git a/STYLE_GUIDE.md b/STYLE_GUIDE.md index 45e2644c..df435280 100644 --- a/STYLE_GUIDE.md +++ b/STYLE_GUIDE.md @@ -368,6 +368,9 @@ Sub-tasks: - `melos run generate:icons` — regenerates the icon font from SVGs in `assets_source/icons/`. → [Icons](#icons) - `melos run gen-l10n` — regenerates localization ARB output. +- `melos run generate:tlds` — regenerates the top-level domains the link finder + recognizes from the IANA root zone list. It downloads the list, so it is not + part of `generate:all`. ## Documentation diff --git a/melos.yaml b/melos.yaml index b1a9274e..108b8623 100644 --- a/melos.yaml +++ b/melos.yaml @@ -52,6 +52,7 @@ command: # List of all the dev_dependencies used in the project. dev_dependencies: alchemist: ^0.13.0 + benchmark_harness: ^2.4.0 build_runner: ^2.10.5 fake_async: ^1.3.3 json_serializable: ^6.9.5 @@ -138,6 +139,10 @@ scripts: run: melos exec -c 1 --file-exists="stream_icons.yaml" -- "dart run \$MELOS_ROOT_PATH/scripts/generate_icons.dart" description: Generate icon font and Dart classes from SVG files. + generate:tlds: + run: dart run scripts/generate_top_level_domains.dart + description: Regenerate the top-level domains the link finder recognizes from the IANA root zone list. + generate:pigeon: run: | melos exec -c 1 --file-exists="pigeons/messages.dart" -- " diff --git a/packages/stream_core/CHANGELOG.md b/packages/stream_core/CHANGELOG.md index 89672cf2..3152a9ff 100644 --- a/packages/stream_core/CHANGELOG.md +++ b/packages/stream_core/CHANGELOG.md @@ -1,3 +1,10 @@ +## Upcoming + +### ✨ Features + +- Added `findLinks` and `findLinkAt`, which find the web links in a text, with + or without a scheme, such as `https://example.com` or `example.com`. + ## 0.5.1+1 ### 🔄 Changed diff --git a/packages/stream_core/benchmark/link_finder_benchmark.dart b/packages/stream_core/benchmark/link_finder_benchmark.dart new file mode 100644 index 00000000..bdec7f0d --- /dev/null +++ b/packages/stream_core/benchmark/link_finder_benchmark.dart @@ -0,0 +1,57 @@ +/// Link Finder Benchmark +/// +/// Measures one `findLinks` call over typical messages, long texts in several +/// scripts, and inputs built to make a link scanner slow. Each score is the +/// time of a single call, in microseconds. +/// +/// Usage (run from `packages/stream_core`): +/// dart run benchmark/link_finder_benchmark.dart +/// +/// For numbers close to a release build, compile it ahead of time first: +/// dart compile exe benchmark/link_finder_benchmark.dart -o benchmark.exe +/// ./benchmark.exe +library; + +import 'package:benchmark_harness/benchmark_harness.dart'; +import 'package:stream_core/stream_core.dart'; + +void main() { + for (final (name, text) in _inputs) { + _FindLinksBenchmark(name, text).report(); + } +} + +final _inputs = <(String, String)>[ + ('short message', 'Some link: youtube.com'), + ( + 'typical message', + 'Hey! The release notes are at getstream.io/chat/docs and the demo is ' + 'on youtube.com/watch?v=abc. Ping me at me@x.io if anything breaks 🙂', + ), + ('prose with links, 3.6k', 'See getstream.io/chat or https://x.io/a?b=c, or mail me@x.io. ' * 60), + ('plain words, 5k', 'hello ' * 850), + ('many links, 5k', 'a.co ' * 1000), + ('Cyrillic, 5k', 'Привет мир. ' * 420), + ('Chinese mixed with ASCII, 5k', '中a' * 2500), + ('emoji, 2k', '🐼a.b ' * 400), + + // Inputs built to make a link scanner slow. + ('Chinese, ASCII and dots, 10k', '中a.' * 3400), + ('closing parentheses, 20k', ')' * 20000), + ('dots, 20k', '.' * 20000), + ('one-letter labels, 10k', 'a.' * 5000), +]; + +// Measures one `findLinks` call over [text]. +class _FindLinksBenchmark extends BenchmarkBase { + _FindLinksBenchmark(String name, this.text) : super('findLinks: $name'); + + final String text; + + @override + void run() => findLinks(text); + + // One call per measurement, so the score is the time of a single call. + @override + void exercise() => run(); +} diff --git a/packages/stream_core/lib/src/utils.dart b/packages/stream_core/lib/src/utils.dart index 06e0866f..fc0fd9cd 100644 --- a/packages/stream_core/lib/src/utils.dart +++ b/packages/stream_core/lib/src/utils.dart @@ -4,6 +4,7 @@ export 'utils/disposable.dart'; export 'utils/event_emitter.dart'; export 'utils/in_flight_cache.dart'; export 'utils/lifecycle_state_provider.dart'; +export 'utils/link_finder.dart'; export 'utils/list_extensions.dart'; export 'utils/network_state_provider.dart'; export 'utils/object.dart'; diff --git a/packages/stream_core/lib/src/utils/link_finder.dart b/packages/stream_core/lib/src/utils/link_finder.dart new file mode 100644 index 00000000..c0c70631 --- /dev/null +++ b/packages/stream_core/lib/src/utils/link_finder.dart @@ -0,0 +1,422 @@ +import 'package:equatable/equatable.dart'; + +part 'link_finder.tlds.g.dart'; + +/// A web link found in text by [findLinks] or [findLinkAt]. +class LinkMatch extends Equatable { + /// Creates a [LinkMatch] spanning [start] to [end] in the searched text. + const LinkMatch({ + required this.start, + required this.end, + required this.text, + required this.url, + }); + + /// The index of the first character of the link in the searched text. + final int start; + + /// The index just past the last character of the link in the searched text. + final int end; + + /// The link as it is written in the text, such as `example.com/docs`. + final String text; + + /// The address to open for the link. + /// + /// The link's own text when it has a scheme. A link written without one, + /// such as `example.com`, opens over `https://`. + /// + /// The address is not percent-encoded, so a caller parsing it into a [Uri] + /// decides how to encode it. + final String url; + + @override + List get props => [start, end, text, url]; + + @override + String toString() => 'LinkMatch($start, $end, $text, $url)'; +} + +/// Finds the web links in [text], in the order they appear. +/// +/// A link is an address with an `http://` or `https://` scheme, or a domain +/// written without one, such as `example.com`, `www.example.com/path` or +/// `docs.example.co.uk:8080/a?b=c`. An address without a scheme counts only +/// when it ends in a top-level domain from the IANA root zone, so most file +/// names, such as `config.json`, are not links. A file name whose extension is +/// also a top-level domain, such as `README.md`, is. +/// +/// Email addresses, and punctuation that ends the sentence around a link, are +/// left out. +/// +/// ```dart +/// final links = findLinks('Docs at example.com/docs, or mail me@x.io.'); +/// print(links.single.url); // https://example.com/docs +/// ``` +List findLinks(String text) { + final links = []; + var index = 0; + while (index < text.length) { + final link = _findLinkAt(text, index); + if (link == null) { + index += 1; + continue; + } + + links.add(link); + index = link.end; + } + + return links; +} + +/// The web link starting at [start] in [text], or null if none starts there. +/// +/// Finds the same links as [findLinks], but only one starting exactly at +/// [start]. A link starts a word, so a position in the middle of a word or an +/// email address finds nothing. +/// +/// A [start] at the end of [text] finds nothing. Throws a [RangeError] if +/// [start] is negative or past the end of [text]. +LinkMatch? findLinkAt(String text, int start) { + RangeError.checkValueInInterval(start, 0, text.length, 'start'); + if (start == text.length) return null; + + return _findLinkAt(text, start); +} + +LinkMatch? _findLinkAt(String text, int start) { + final codeUnit = text.codeUnitAt(start); + final mayHaveScheme = codeUnit == _kLowercaseH || codeUnit == _kUppercaseH; + final hasScheme = mayHaveScheme && _scheme.matchAsPrefix(text, start) != null; + if (hasScheme && !_startsSchemeLink(text, start)) return null; + if (!hasScheme && !_startsWord(text, start)) return null; + + final end = hasScheme ? _matchWithScheme(text, start) : _matchWithoutScheme(text, start); + if (end == null) return null; + + final linkEnd = _trimTrailingPunctuation(text, start, end); + + // An address can be the start of an email address, as in `anna.li@gmail.com`. + if (_continuesAsEmailAddress(text, linkEnd)) return null; + + final link = text.substring(start, linkEnd); + final url = hasScheme ? link : 'https://$link'; + return LinkMatch(start: start, end: linkEnd, text: link, url: url); +} + +const _kAt = 0x40; +const _kHyphen = 0x2D; +const _kUnderscore = 0x5F; +const _kDot = 0x2E; +const _kLowercaseH = 0x68; +const _kUppercaseH = 0x48; +const _kFirstNonAscii = 0x80; + +// The longest host name DNS allows, in characters. +const _kMaxHostLength = 253; + +// A domain label of up to 63 letters, marks, digits, hyphens and underscores, +// starting with a letter or digit and ending with a letter, mark or digit. +const _kLabelPattern = r'[\p{L}\p{N}](?:[\p{L}\p{M}\p{N}_\-]{0,61}[\p{L}\p{M}\p{N}])?'; +const _kAsciiLabelPattern = r'[a-zA-Z0-9](?:[a-zA-Z0-9_\-]{0,61}[a-zA-Z0-9])?'; +const _kOctetPattern = r'(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)'; + +final _scheme = RegExp('https?://', caseSensitive: false); +final _userInfo = RegExp(r'[^\s/?#@\[\]]+@'); +final _labels = RegExp('$_kLabelPattern(?:\\.$_kLabelPattern)*', unicode: true); +final _asciiLabels = RegExp('$_kAsciiLabelPattern(?:\\.$_kAsciiLabelPattern)*'); +final _ipAddress = RegExp('$_kOctetPattern(?:\\.$_kOctetPattern){3}'); +final _port = RegExp(r':\d{1,5}'); +final _ipv6Address = RegExp(r'\[[0-9a-fA-F:.]+\]'); +final _wordCharacter = RegExp(r'[\p{L}\p{M}\p{N}_]', unicode: true); +final _noSpaceScript = RegExp( + r'[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}' + r'\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]', + unicode: true, +); + +// Characters that can't come right before a link, as they would make the link +// the tail of a word, an address or another link. +const _kInvalidPrecedingCodeUnits = { + 0x40, // @ + 0x2E, // . + 0x2D, // - + 0x5F, // _ + 0x2F, // / + 0x3A, // : + 0x23, // # + 0x25, // % + 0x26, // & + 0x3D, // = + 0x2B, // + + 0x5C, // \ +}; + +// Punctuation that ends the sentence around a link rather than the link. +const _kTrailingPunctuationCodeUnits = { + 0x3F, // ? + 0x21, // ! + 0x2E, // . + 0x2C, // , + 0x3A, // : + 0x3B, // ; + 0x2A, // * + 0x5F, // _ + 0x7E, // ~ + 0x27, // ' + 0x22, // " + 0x40, // @ + 0x2019, // ’ + 0x201D, // ” + 0x00BB, // » +}; + +// Characters a path, query or fragment starts with: / ? #. +const _kPathStarts = {0x2F, 0x3F, 0x23}; + +// Characters that can't be in an address: < > " ` |. +const _kPathTerminators = {0x3C, 0x3E, 0x22, 0x60, 0x7C}; + +// Symbols an email address can have before its `@`: . _ % + -. +const _kEmailLocalPartSymbols = {0x2E, 0x5F, 0x25, 0x2B, 0x2D}; + +// Opening brackets in a path, mapped to the bracket that closes each: ( [ {. +const _kClosingBracketFor = {0x28: 0x29, 0x5B: 0x5D, 0x7B: 0x7D}; + +// Closing brackets in a path: ) ] }. +const _kClosingBrackets = {0x29, 0x5D, 0x7D}; + +bool _startsWord(String text, int start) { + final codeUnit = text.codeUnitAt(start); + if (codeUnit == _kUnderscore || _isLowSurrogate(codeUnit)) return false; + if (!_isWordCharacterAt(text, start)) return false; + if (start == 0) return true; + + final previousStart = _previousCharacterStart(text, start); + final previous = text.codeUnitAt(previousStart); + final isInvalidPreceding = _kInvalidPrecedingCodeUnits.contains(previous); + if (isInvalidPreceding && !_followsEllipsis(text, start)) return false; + if (!_isWordCharacterAt(text, previousStart)) return true; + + // Scripts written without spaces run words together, so a link may follow + // one directly: in `请访问google.com`, the link starts at `google.com`. + return _isAsciiAlphanumeric(codeUnit) && _isNoSpaceScriptAt(text, previousStart); +} + +// Whether the `.` right before [start] ends an ellipsis, as in +// `see...example.com`. +bool _followsEllipsis(String text, int start) { + if (start < 2) return false; + return text.codeUnitAt(start - 1) == _kDot && text.codeUnitAt(start - 2) == _kDot; +} + +// An address with a scheme is a link wherever it doesn't continue a word, as in +// `(https://example.com)` or `_https://example.com_`. +bool _startsSchemeLink(String text, int start) { + if (start == 0) return true; + + final previousStart = _previousCharacterStart(text, start); + if (text.codeUnitAt(previousStart) == _kUnderscore) return true; + return !_isWordCharacterAt(text, previousStart); +} + +// The end of the `http://` or `https://` address starting at [start], or null. +int? _matchWithScheme(String text, int start) { + final scheme = _scheme.matchAsPrefix(text, start)!; + final hostStart = _userInfo.matchAsPrefix(text, scheme.end)?.end ?? scheme.end; + final ipv6End = _ipv6Address.matchAsPrefix(text, hostStart)?.end; + final hostEnd = ipv6End ?? _matchLabelsEnd(_labels, text, hostStart); + if (hostEnd == null) return null; + + return _matchPortAndPath(text, hostEnd); +} + +// The end of the address without a scheme starting at [start], or null. +int? _matchWithoutScheme(String text, int start) { + // A link that runs on from a script written without spaces, as in + // `请访问google.com`, has an ASCII host, so the text in that script after + // it is not read as more of the host. + final previousStart = start > 0 ? _previousCharacterStart(text, start) : -1; + final runsOn = previousStart >= 0 && _isWordCharacterAt(text, previousStart); + final labelsPattern = runsOn ? _asciiLabels : _labels; + final labelsEnd = _matchLabelsEnd(labelsPattern, text, start); + if (labelsEnd == null) return null; + + final ipEnd = _ipAddress.matchAsPrefix(text, start)?.end; + if (ipEnd == labelsEnd) return _matchPortAndPath(text, labelsEnd); + + final host = text.substring(start, labelsEnd); + if (_runsOnFromNoSpaceScript(host)) return null; + + final parts = host.split('.'); + var partEnd = labelsEnd; + for (var count = parts.length; count >= 2; count--) { + final topLevelDomain = parts[count - 1]; + final partStart = partEnd - topLevelDomain.length; + for (final length in _topLevelDomainLengths(topLevelDomain)) { + final end = _matchPortAndPath(text, partStart + length); + if (end != null) return end; + } + partEnd = partStart - 1; + } + + return null; +} + +// The end of the dot-separated [labels] of a host starting at [start], or null +// when there are none or they are longer than a host name can be. +int? _matchLabelsEnd(RegExp labels, String text, int start) { + final end = labels.matchAsPrefix(text, start)?.end; + if (end == null || end - start > _kMaxHostLength) return null; + + return end; +} + +// The lengths at which [label] can end in a known top-level domain, longest +// first: the whole label, or the part before a hyphen or before text in a +// script written without spaces, as in `google.com了解更多`. +Iterable _topLevelDomainLengths(String label) sync* { + for (var length = label.length; length >= 2; length--) { + if (length < label.length && !_isLabelBreak(label, length)) continue; + if (_isTopLevelDomain(label.substring(0, length))) yield length; + } +} + +bool _isLabelBreak(String label, int index) { + return label.codeUnitAt(index) == _kHyphen || _runsIntoNoSpaceScript(label, index); +} + +bool _isTopLevelDomain(String label) => _kIanaTopLevelDomains.contains(label.toLowerCase()); + +// Whether the first label of [host] runs on from a script written without +// spaces into ASCII, as in `请访问google.com`. The link then starts later, +// at the ASCII part. +bool _runsOnFromNoSpaceScript(String host) { + final dot = host.indexOf('.'); + final firstLabel = dot == -1 ? host : host.substring(0, dot); + final codeUnits = firstLabel.codeUnits; + if (codeUnits.every((codeUnit) => codeUnit < _kFirstNonAscii)) return false; + + return codeUnits.any(_isAsciiAlphanumeric) && _noSpaceScript.hasMatch(firstLabel); +} + +// Whether ASCII before [index] runs straight into a script written without +// spaces at [index]. +bool _runsIntoNoSpaceScript(String text, int index) { + return _isAsciiAlphanumeric(text.codeUnitAt(index - 1)) && _isNoSpaceScriptAt(text, index); +} + +// The end of the address with its optional port and path after a host ending +// at [hostEnd], or null when the host does not end a word. +int? _matchPortAndPath(String text, int hostEnd) { + final portEnd = _port.matchAsPrefix(text, hostEnd)?.end; + if (portEnd != null && !_endsWord(text, portEnd)) return hostEnd; + if (portEnd == null && !_endsWord(text, hostEnd)) return null; + + return _matchPath(text, portEnd ?? hostEnd); +} + +// The end of the path, query or fragment starting at [start], or [start] when +// there is none. +// +// It runs to whitespace or a character that can't be in an address, and stops +// at a closing bracket that closes nothing in it, as in `(see example.com/a)`. +int _matchPath(String text, int start) { + if (start >= text.length || !_kPathStarts.contains(text.codeUnitAt(start))) return start; + + final expectedClosers = []; + var index = start; + while (index < text.length) { + final codeUnit = text.codeUnitAt(index); + if (_isWhitespace(codeUnit) || _kPathTerminators.contains(codeUnit)) break; + + if (_kClosingBracketFor[codeUnit] case final closer?) { + expectedClosers.add(closer); + } else if (_kClosingBrackets.contains(codeUnit)) { + if (expectedClosers.isEmpty || expectedClosers.last != codeUnit) break; + expectedClosers.removeLast(); + } + index += 1; + } + + return index; +} + +// Whether the text from [index] runs on to the `@` of an email address, as the +// `+tag@gmail.com` of `anna.li+tag@gmail.com` does. +bool _continuesAsEmailAddress(String text, int index) { + var atIndex = index; + while (atIndex < text.length && _isEmailLocalPartCodeUnit(text.codeUnitAt(atIndex))) { + atIndex += 1; + } + + if (atIndex + 1 >= text.length || text.codeUnitAt(atIndex) != _kAt) return false; + return _isWordCharacterAt(text, atIndex + 1); +} + +bool _isEmailLocalPartCodeUnit(int codeUnit) { + return _isAsciiAlphanumeric(codeUnit) || _kEmailLocalPartSymbols.contains(codeUnit); +} + +bool _endsWord(String text, int end) { + if (end >= text.length) return true; + return !_isWordCharacterAt(text, end) || _runsIntoNoSpaceScript(text, end); +} + +// The end of the link from [start] to [end] once the punctuation that ends the +// sentence around it is left out. +int _trimTrailingPunctuation(String text, int start, int end) { + var trimmedEnd = end; + while (trimmedEnd > start) { + final last = text.codeUnitAt(trimmedEnd - 1); + if (!_kTrailingPunctuationCodeUnits.contains(last)) break; + trimmedEnd -= 1; + } + + return trimmedEnd; +} + +// Whether [codeUnit] is a control character, a space or a Unicode White_Space +// character. +bool _isWhitespace(int codeUnit) { + return codeUnit <= 0x20 || + codeUnit == 0x85 || + codeUnit == 0xA0 || + codeUnit == 0x1680 || + (codeUnit >= 0x2000 && codeUnit <= 0x200A) || + codeUnit == 0x2028 || + codeUnit == 0x2029 || + codeUnit == 0x202F || + codeUnit == 0x205F || + codeUnit == 0x3000 || + codeUnit == 0xFEFF; +} + +bool _isAsciiAlphanumeric(int codeUnit) { + return (codeUnit >= 0x30 && codeUnit <= 0x39) || + (codeUnit >= 0x41 && codeUnit <= 0x5A) || + (codeUnit >= 0x61 && codeUnit <= 0x7A); +} + +bool _isWordCharacterAt(String text, int index) { + final codeUnit = text.codeUnitAt(index); + if (codeUnit >= _kFirstNonAscii) return _wordCharacter.matchAsPrefix(text, index) != null; + return _isAsciiAlphanumeric(codeUnit) || codeUnit == _kUnderscore; +} + +bool _isNoSpaceScriptAt(String text, int index) { + if (text.codeUnitAt(index) < _kFirstNonAscii) return false; + return _noSpaceScript.matchAsPrefix(text, index) != null; +} + +// The index of the character that ends right before [index]. +int _previousCharacterStart(String text, int index) { + final isLowSurrogate = _isLowSurrogate(text.codeUnitAt(index - 1)); + if (!isLowSurrogate || index < 2) return index - 1; + return _isHighSurrogate(text.codeUnitAt(index - 2)) ? index - 2 : index - 1; +} + +bool _isHighSurrogate(int codeUnit) => codeUnit >= 0xD800 && codeUnit <= 0xDBFF; + +bool _isLowSurrogate(int codeUnit) => codeUnit >= 0xDC00 && codeUnit <= 0xDFFF; diff --git a/packages/stream_core/lib/src/utils/link_finder.tlds.g.dart b/packages/stream_core/lib/src/utils/link_finder.tlds.g.dart new file mode 100644 index 00000000..b7065354 --- /dev/null +++ b/packages/stream_core/lib/src/utils/link_finder.tlds.g.dart @@ -0,0 +1,1602 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND +// +// Generated by `melos run generate:tlds` from +// https://data.iana.org/TLD/tlds-alpha-by-domain.txt +// Version 2026102700, Last Updated Tue Oct 27 07:07:01 2026 UTC + +part of 'link_finder.dart'; + +// The top-level domains in the IANA root zone, lowercase. +// +// Internationalized domains are in both their ASCII and Unicode forms, each +// commented with the other. +const _kIanaTopLevelDomains = { + 'aaa', + 'aarp', + 'abb', + 'abbott', + 'abbvie', + 'abc', + 'able', + 'abogado', + 'abudhabi', + 'ac', + 'academy', + 'accenture', + 'accountant', + 'accountants', + 'aco', + 'actor', + 'ad', + 'ads', + 'adult', + 'ae', + 'aeg', + 'aero', + 'aetna', + 'af', + 'afl', + 'africa', + 'ag', + 'agakhan', + 'agency', + 'ai', + 'aig', + 'airbus', + 'airforce', + 'airtel', + 'akdn', + 'al', + 'alibaba', + 'alipay', + 'allfinanz', + 'allstate', + 'ally', + 'alsace', + 'alstom', + 'am', + 'amazon', + 'americanexpress', + 'americanfamily', + 'amex', + 'amfam', + 'amica', + 'amsterdam', + 'analytics', + 'android', + 'anquan', + 'anz', + 'ao', + 'aol', + 'apartments', + 'app', + 'apple', + 'aq', + 'aquarelle', + 'ar', + 'arab', + 'aramco', + 'archi', + 'army', + 'arpa', + 'art', + 'arte', + 'as', + 'asda', + 'asia', + 'associates', + 'at', + 'athleta', + 'attorney', + 'au', + 'auction', + 'audi', + 'audible', + 'audio', + 'auspost', + 'author', + 'auto', + 'autos', + 'aw', + 'aws', + 'ax', + 'axa', + 'az', + 'azure', + 'ba', + 'baby', + 'baidu', + 'banamex', + 'band', + 'bank', + 'bar', + 'barcelona', + 'barclaycard', + 'barclays', + 'barefoot', + 'bargains', + 'baseball', + 'basketball', + 'bauhaus', + 'bayern', + 'bb', + 'bbc', + 'bbt', + 'bbva', + 'bcg', + 'bcn', + 'bd', + 'be', + 'beats', + 'beauty', + 'beer', + 'berlin', + 'best', + 'bestbuy', + 'bet', + 'bf', + 'bg', + 'bh', + 'bharti', + 'bi', + 'bible', + 'bid', + 'bike', + 'bing', + 'bingo', + 'bio', + 'biz', + 'bj', + 'black', + 'blackfriday', + 'blockbuster', + 'blog', + 'bloomberg', + 'blue', + 'bm', + 'bms', + 'bmw', + 'bn', + 'bnpparibas', + 'bo', + 'boats', + 'boehringer', + 'bofa', + 'bom', + 'bond', + 'boo', + 'book', + 'booking', + 'bosch', + 'bostik', + 'boston', + 'bot', + 'boutique', + 'box', + 'br', + 'bradesco', + 'bridgestone', + 'broadway', + 'broker', + 'brother', + 'brussels', + 'bs', + 'bt', + 'build', + 'builders', + 'business', + 'buy', + 'buzz', + 'bv', + 'bw', + 'by', + 'bz', + 'bzh', + 'ca', + 'cab', + 'cafe', + 'cal', + 'call', + 'calvinklein', + 'cam', + 'camera', + 'camp', + 'canon', + 'capetown', + 'capital', + 'capitalone', + 'car', + 'caravan', + 'cards', + 'care', + 'career', + 'careers', + 'cars', + 'casa', + 'case', + 'cash', + 'casino', + 'cat', + 'catering', + 'catholic', + 'cba', + 'cbn', + 'cbre', + 'cc', + 'cd', + 'center', + 'ceo', + 'cern', + 'cf', + 'cfa', + 'cfd', + 'cg', + 'ch', + 'chanel', + 'channel', + 'charity', + 'chase', + 'chat', + 'cheap', + 'chintai', + 'christmas', + 'chrome', + 'church', + 'ci', + 'cipriani', + 'circle', + 'cisco', + 'citadel', + 'citi', + 'citic', + 'city', + 'ck', + 'cl', + 'claims', + 'cleaning', + 'click', + 'clinic', + 'clinique', + 'clothing', + 'cloud', + 'club', + 'clubmed', + 'cm', + 'cn', + 'co', + 'coach', + 'codes', + 'coffee', + 'college', + 'cologne', + 'com', + 'commbank', + 'community', + 'company', + 'compare', + 'computer', + 'comsec', + 'condos', + 'construction', + 'consulting', + 'contact', + 'contractors', + 'cooking', + 'cool', + 'coop', + 'corsica', + 'country', + 'coupon', + 'coupons', + 'courses', + 'cpa', + 'cr', + 'credit', + 'creditcard', + 'creditunion', + 'cricket', + 'crown', + 'crs', + 'cruise', + 'cruises', + 'cu', + 'cuisinella', + 'cv', + 'cw', + 'cx', + 'cy', + 'cymru', + 'cyou', + 'cz', + 'dad', + 'dance', + 'data', + 'date', + 'dating', + 'datsun', + 'day', + 'dclk', + 'dds', + 'de', + 'deal', + 'dealer', + 'deals', + 'degree', + 'delivery', + 'dell', + 'deloitte', + 'delta', + 'democrat', + 'dental', + 'dentist', + 'desi', + 'design', + 'dev', + 'dhl', + 'diamonds', + 'diet', + 'digital', + 'direct', + 'directory', + 'discount', + 'discover', + 'dish', + 'diy', + 'dj', + 'dk', + 'dm', + 'dnp', + 'do', + 'docs', + 'doctor', + 'dog', + 'domains', + 'dot', + 'download', + 'drive', + 'dtv', + 'dubai', + 'dupont', + 'durban', + 'dvag', + 'dvr', + 'dz', + 'earth', + 'eat', + 'ec', + 'eco', + 'edeka', + 'edu', + 'education', + 'ee', + 'eg', + 'email', + 'emerck', + 'energy', + 'engineer', + 'engineering', + 'enterprises', + 'epson', + 'equipment', + 'er', + 'ericsson', + 'erni', + 'es', + 'esq', + 'estate', + 'et', + 'eu', + 'eurovision', + 'eus', + 'events', + 'exchange', + 'expert', + 'exposed', + 'express', + 'extraspace', + 'fage', + 'fail', + 'fairwinds', + 'faith', + 'family', + 'fan', + 'fans', + 'farm', + 'farmers', + 'fashion', + 'fast', + 'fedex', + 'feedback', + 'ferrari', + 'ferrero', + 'fi', + 'fidelity', + 'fido', + 'film', + 'final', + 'finance', + 'financial', + 'fire', + 'firestone', + 'firmdale', + 'fish', + 'fishing', + 'fit', + 'fitness', + 'fj', + 'fk', + 'flickr', + 'flights', + 'flir', + 'florist', + 'flowers', + 'fly', + 'fm', + 'fo', + 'foo', + 'food', + 'football', + 'ford', + 'forex', + 'forsale', + 'forum', + 'foundation', + 'fox', + 'fr', + 'free', + 'fresenius', + 'frl', + 'frogans', + 'frontier', + 'ftr', + 'fujitsu', + 'fun', + 'fund', + 'furniture', + 'futbol', + 'fyi', + 'ga', + 'gal', + 'gallery', + 'gallo', + 'gallup', + 'game', + 'games', + 'gap', + 'garden', + 'gay', + 'gb', + 'gbiz', + 'gd', + 'gdn', + 'ge', + 'gea', + 'gent', + 'genting', + 'george', + 'gf', + 'gg', + 'ggee', + 'gh', + 'gi', + 'gift', + 'gifts', + 'gives', + 'giving', + 'gl', + 'glass', + 'gle', + 'global', + 'globo', + 'gm', + 'gmail', + 'gmbh', + 'gmo', + 'gmx', + 'gn', + 'godaddy', + 'gold', + 'goldpoint', + 'golf', + 'goodyear', + 'goog', + 'google', + 'gop', + 'got', + 'gov', + 'gp', + 'gq', + 'gr', + 'grainger', + 'graphics', + 'gratis', + 'green', + 'gripe', + 'grocery', + 'group', + 'gs', + 'gt', + 'gu', + 'gucci', + 'guge', + 'guide', + 'guitars', + 'guru', + 'gw', + 'gy', + 'hair', + 'hamburg', + 'hangout', + 'haus', + 'hbo', + 'hdfc', + 'hdfcbank', + 'health', + 'healthcare', + 'help', + 'helsinki', + 'here', + 'hermes', + 'hiphop', + 'hisamitsu', + 'hitachi', + 'hiv', + 'hk', + 'hkt', + 'hm', + 'hn', + 'hockey', + 'holdings', + 'holiday', + 'homedepot', + 'homegoods', + 'homes', + 'homesense', + 'honda', + 'horse', + 'hospital', + 'host', + 'hosting', + 'hot', + 'hotels', + 'hotmail', + 'house', + 'how', + 'hr', + 'hsbc', + 'ht', + 'hu', + 'hughes', + 'hyatt', + 'hyundai', + 'ibm', + 'icbc', + 'ice', + 'icu', + 'id', + 'ie', + 'ieee', + 'ifm', + 'ikano', + 'il', + 'im', + 'imamat', + 'imdb', + 'immo', + 'immobilien', + 'in', + 'inc', + 'industries', + 'infiniti', + 'info', + 'ing', + 'ink', + 'institute', + 'insurance', + 'insure', + 'int', + 'international', + 'intuit', + 'investments', + 'io', + 'ipiranga', + 'iq', + 'ir', + 'irish', + 'is', + 'ismaili', + 'ist', + 'istanbul', + 'it', + 'itau', + 'itv', + 'jaguar', + 'java', + 'jcb', + 'je', + 'jeep', + 'jetzt', + 'jewelry', + 'jio', + 'jll', + 'jm', + 'jmp', + 'jnj', + 'jo', + 'jobs', + 'joburg', + 'jot', + 'joy', + 'jp', + 'jpmorgan', + 'jprs', + 'juegos', + 'kaufen', + 'kddi', + 'ke', + 'kerryhotels', + 'kerryproperties', + 'kfh', + 'kg', + 'kh', + 'ki', + 'kia', + 'kids', + 'kim', + 'kindle', + 'kitchen', + 'kiwi', + 'km', + 'kn', + 'koeln', + 'komatsu', + 'kosher', + 'kp', + 'kpmg', + 'kpn', + 'kr', + 'krd', + 'kred', + 'kuokgroup', + 'kw', + 'ky', + 'kyoto', + 'kz', + 'la', + 'lacaixa', + 'lamborghini', + 'lamer', + 'land', + 'landrover', + 'lanxess', + 'lasalle', + 'lat', + 'latino', + 'latrobe', + 'law', + 'lawyer', + 'lb', + 'lc', + 'lds', + 'lease', + 'leclerc', + 'lefrak', + 'legal', + 'lego', + 'lexus', + 'lgbt', + 'li', + 'lidl', + 'life', + 'lifeinsurance', + 'lifestyle', + 'lighting', + 'like', + 'lilly', + 'limited', + 'limo', + 'lincoln', + 'link', + 'live', + 'living', + 'lk', + 'llc', + 'llp', + 'loan', + 'loans', + 'locker', + 'locus', + 'lol', + 'london', + 'lotte', + 'lotto', + 'love', + 'lpl', + 'lplfinancial', + 'lr', + 'ls', + 'lt', + 'ltd', + 'ltda', + 'lu', + 'lundbeck', + 'luxe', + 'luxury', + 'lv', + 'ly', + 'ma', + 'madrid', + 'maif', + 'maison', + 'makeup', + 'man', + 'management', + 'mango', + 'map', + 'market', + 'marketing', + 'markets', + 'marriott', + 'marshalls', + 'mattel', + 'mba', + 'mc', + 'mckinsey', + 'md', + 'me', + 'med', + 'media', + 'meet', + 'melbourne', + 'meme', + 'memorial', + 'men', + 'menu', + 'merck', + 'merckmsd', + 'mg', + 'mh', + 'miami', + 'microsoft', + 'mil', + 'mini', + 'mint', + 'mit', + 'mitsubishi', + 'mk', + 'ml', + 'mlb', + 'mls', + 'mm', + 'mma', + 'mn', + 'mo', + 'mobi', + 'mobile', + 'moda', + 'moe', + 'moi', + 'mom', + 'monash', + 'money', + 'monster', + 'mormon', + 'mortgage', + 'moscow', + 'moto', + 'motorcycles', + 'mov', + 'movie', + 'mp', + 'mq', + 'mr', + 'ms', + 'msd', + 'mt', + 'mtn', + 'mtr', + 'mu', + 'museum', + 'music', + 'mv', + 'mw', + 'mx', + 'my', + 'mz', + 'na', + 'nab', + 'nagoya', + 'name', + 'navy', + 'nba', + 'nc', + 'ne', + 'nec', + 'net', + 'netbank', + 'netflix', + 'network', + 'neustar', + 'new', + 'news', + 'next', + 'nextdirect', + 'nexus', + 'nf', + 'nfl', + 'ng', + 'ngo', + 'nhk', + 'ni', + 'nico', + 'nike', + 'nikon', + 'ninja', + 'nissan', + 'nissay', + 'nl', + 'no', + 'nokia', + 'norton', + 'now', + 'nowruz', + 'nowtv', + 'np', + 'nr', + 'nra', + 'nrw', + 'ntt', + 'nu', + 'nyc', + 'nz', + 'obi', + 'observer', + 'office', + 'okinawa', + 'olayan', + 'olayangroup', + 'ollo', + 'om', + 'omega', + 'one', + 'ong', + 'onl', + 'online', + 'ooo', + 'open', + 'oracle', + 'orange', + 'org', + 'organic', + 'origins', + 'osaka', + 'otsuka', + 'ott', + 'ovh', + 'pa', + 'page', + 'panasonic', + 'paris', + 'pars', + 'partners', + 'parts', + 'party', + 'pay', + 'pccw', + 'pe', + 'pet', + 'pf', + 'pfizer', + 'pg', + 'ph', + 'pharmacy', + 'phd', + 'philips', + 'phone', + 'photo', + 'photography', + 'photos', + 'physio', + 'pics', + 'pictet', + 'pictures', + 'pid', + 'pin', + 'ping', + 'pink', + 'pioneer', + 'pizza', + 'pk', + 'pl', + 'place', + 'play', + 'playstation', + 'plumbing', + 'plus', + 'pm', + 'pn', + 'pnc', + 'pohl', + 'poker', + 'politie', + 'porn', + 'post', + 'pr', + 'praxi', + 'press', + 'prime', + 'pro', + 'prod', + 'productions', + 'prof', + 'progressive', + 'promo', + 'properties', + 'property', + 'protection', + 'pru', + 'prudential', + 'ps', + 'pt', + 'pub', + 'pw', + 'pwc', + 'py', + 'qa', + 'qpon', + 'quebec', + 'quest', + 'racing', + 'radio', + 're', + 'read', + 'realestate', + 'realtor', + 'realty', + 'recipes', + 'red', + 'redumbrella', + 'rehab', + 'reise', + 'reisen', + 'reit', + 'reliance', + 'ren', + 'rent', + 'rentals', + 'repair', + 'report', + 'republican', + 'rest', + 'restaurant', + 'review', + 'reviews', + 'rexroth', + 'rich', + 'richardli', + 'ricoh', + 'ril', + 'rio', + 'rip', + 'ro', + 'rocks', + 'rodeo', + 'rogers', + 'room', + 'rs', + 'rsvp', + 'ru', + 'rugby', + 'ruhr', + 'run', + 'rw', + 'rwe', + 'ryukyu', + 'sa', + 'saarland', + 'safe', + 'safety', + 'sakura', + 'sale', + 'salon', + 'samsclub', + 'samsung', + 'sandvik', + 'sandvikcoromant', + 'sanofi', + 'sap', + 'sarl', + 'sas', + 'save', + 'saxo', + 'sb', + 'sbi', + 'sbs', + 'sc', + 'scb', + 'schaeffler', + 'schmidt', + 'scholarships', + 'school', + 'schule', + 'schwarz', + 'science', + 'scot', + 'sd', + 'se', + 'search', + 'seat', + 'secure', + 'security', + 'seek', + 'select', + 'sener', + 'services', + 'seven', + 'sew', + 'sex', + 'sexy', + 'sfr', + 'sg', + 'sh', + 'shangrila', + 'sharp', + 'shell', + 'shia', + 'shiksha', + 'shoes', + 'shop', + 'shopping', + 'shouji', + 'show', + 'si', + 'silk', + 'sina', + 'singles', + 'site', + 'sj', + 'sk', + 'ski', + 'skin', + 'sky', + 'skype', + 'sl', + 'sling', + 'sm', + 'smart', + 'smile', + 'sn', + 'sncf', + 'so', + 'soccer', + 'social', + 'softbank', + 'software', + 'sohu', + 'solar', + 'solutions', + 'song', + 'sony', + 'soy', + 'spa', + 'space', + 'sport', + 'spot', + 'sr', + 'srl', + 'ss', + 'st', + 'stada', + 'staples', + 'star', + 'statebank', + 'statefarm', + 'stc', + 'stcgroup', + 'stockholm', + 'storage', + 'store', + 'stream', + 'studio', + 'study', + 'style', + 'su', + 'sucks', + 'supplies', + 'supply', + 'support', + 'surf', + 'surgery', + 'suzuki', + 'sv', + 'swatch', + 'swiss', + 'sx', + 'sy', + 'sydney', + 'systems', + 'sz', + 'tab', + 'taipei', + 'talk', + 'taobao', + 'target', + 'tatamotors', + 'tatar', + 'tattoo', + 'tax', + 'taxi', + 'tc', + 'tci', + 'td', + 'tdk', + 'team', + 'tech', + 'technology', + 'tel', + 'temasek', + 'tennis', + 'teva', + 'tf', + 'tg', + 'th', + 'thd', + 'theater', + 'theatre', + 'tiaa', + 'tickets', + 'tienda', + 'tips', + 'tires', + 'tirol', + 'tj', + 'tjmaxx', + 'tjx', + 'tk', + 'tkmaxx', + 'tl', + 'tm', + 'tmall', + 'tn', + 'to', + 'today', + 'tokyo', + 'tools', + 'top', + 'toray', + 'toshiba', + 'total', + 'tours', + 'town', + 'toyota', + 'toys', + 'tr', + 'trade', + 'trading', + 'training', + 'travel', + 'travelers', + 'travelersinsurance', + 'trust', + 'trv', + 'tt', + 'tube', + 'tui', + 'tunes', + 'tushu', + 'tv', + 'tvs', + 'tw', + 'tz', + 'ua', + 'ubank', + 'ubs', + 'ug', + 'uk', + 'unicom', + 'university', + 'uno', + 'uol', + 'ups', + 'us', + 'uy', + 'uz', + 'va', + 'vacations', + 'vana', + 'vanguard', + 'vc', + 've', + 'vegas', + 'ventures', + 'verisign', + 'vermögensberater', // xn--vermgensberater-ctb + 'vermögensberatung', // xn--vermgensberatung-pwb + 'versicherung', + 'vet', + 'vg', + 'vi', + 'viajes', + 'video', + 'vig', + 'viking', + 'villas', + 'vin', + 'vip', + 'virgin', + 'visa', + 'vision', + 'viva', + 'vivo', + 'vlaanderen', + 'vn', + 'vodka', + 'volvo', + 'vote', + 'voting', + 'voto', + 'voyage', + 'vu', + 'wales', + 'walmart', + 'walter', + 'wang', + 'wanggou', + 'watch', + 'watches', + 'weather', + 'weatherchannel', + 'web', + 'webcam', + 'weber', + 'website', + 'wed', + 'wedding', + 'weibo', + 'weir', + 'wf', + 'whoswho', + 'wien', + 'wiki', + 'williamhill', + 'win', + 'windows', + 'wine', + 'winners', + 'wme', + 'woodside', + 'work', + 'works', + 'world', + 'wow', + 'ws', + 'wtc', + 'wtf', + 'xbox', + 'xerox', + 'xihuan', + 'xin', + 'xn--11b4c3d', // कॉम + 'xn--1ck2e1b', // セール + 'xn--1qqw23a', // 佛山 + 'xn--2scrj9c', // ಭಾರತ + 'xn--30rr7y', // 慈善 + 'xn--3bst00m', // 集团 + 'xn--3ds443g', // 在线 + 'xn--3e0b707e', // 한국 + 'xn--3hcrj9c', // ଭାରତ + 'xn--3pxu8k', // 点看 + 'xn--42c2d9a', // คอม + 'xn--45br5cyl', // ভাৰত + 'xn--45brj9c', // ভারত + 'xn--45q11c', // 八卦 + 'xn--4dbrk0ce', // ישראל + 'xn--4gbrim', // موقع + 'xn--54b7fta0cc', // বাংলা + 'xn--55qw42g', // 公益 + 'xn--55qx5d', // 公司 + 'xn--5su34j936bgsg', // 香格里拉 + 'xn--5tzm5g', // 网站 + 'xn--6frz82g', // 移动 + 'xn--6qq986b3xl', // 我爱你 + 'xn--80adxhks', // москва + 'xn--80ao21a', // қаз + 'xn--80aqecdr1a', // католик + 'xn--80asehdb', // онлайн + 'xn--80aswg', // сайт + 'xn--8y0a063a', // 联通 + 'xn--90a3ac', // срб + 'xn--90ae', // бг + 'xn--90ais', // бел + 'xn--9dbq2a', // קום + 'xn--9et52u', // 时尚 + 'xn--9krt00a', // 微博 + 'xn--b4w605ferd', // 淡马锡 + 'xn--bck1b9a5dre4c', // ファッション + 'xn--c1avg', // орг + 'xn--c2br7g', // नेट + 'xn--cck2b3b', // ストア + 'xn--cckwcxetd', // アマゾン + 'xn--cg4bki', // 삼성 + 'xn--clchc0ea0b2g2a9gcd', // சிங்கப்பூர் + 'xn--czr694b', // 商标 + 'xn--czrs0t', // 商店 + 'xn--czru2d', // 商城 + 'xn--d1acj3b', // дети + 'xn--d1alf', // мкд + 'xn--e1a4c', // ею + 'xn--eckvdtc9d', // ポイント + 'xn--efvy88h', // 新闻 + 'xn--fct429k', // 家電 + 'xn--fhbei', // كوم + 'xn--fiq228c5hs', // 中文网 + 'xn--fiq64b', // 中信 + 'xn--fiqs8s', // 中国 + 'xn--fiqz9s', // 中國 + 'xn--fjq720a', // 娱乐 + 'xn--flw351e', // 谷歌 + 'xn--fpcrj9c3d', // భారత్ + 'xn--fzc2c9e2c', // ලංකා + 'xn--fzys8d69uvgm', // 電訊盈科 + 'xn--g2xx48c', // 购物 + 'xn--gckr3f0f', // クラウド + 'xn--gecrj9c', // ભારત + 'xn--gk3at1e', // 通販 + 'xn--h2breg3eve', // भारतम् + 'xn--h2brj9c', // भारत + 'xn--h2brj9c8c', // भारोत + 'xn--hxt814e', // 网店 + 'xn--i1b6b1a6a2e', // संगठन + 'xn--imr513n', // 餐厅 + 'xn--io0a7i', // 网络 + 'xn--j1aef', // ком + 'xn--j1amh', // укр + 'xn--j6w193g', // 香港 + 'xn--jlq480n2rg', // 亚马逊 + 'xn--jvr189m', // 食品 + 'xn--kcrx77d1x4a', // 飞利浦 + 'xn--kprw13d', // 台湾 + 'xn--kpry57d', // 台灣 + 'xn--kput3i', // 手机 + 'xn--l1acc', // мон + 'xn--lgbbat1ad8j', // الجزائر + 'xn--mgb9awbf', // عمان + 'xn--mgba3a3ejt', // ارامكو + 'xn--mgba3a4f16a', // ایران + 'xn--mgba7c0bbn0a', // العليان + 'xn--mgbaam7a8h', // امارات + 'xn--mgbab2bd', // بازار + 'xn--mgbah1a3hjkrd', // موريتانيا + 'xn--mgbai9azgqp6j', // پاکستان + 'xn--mgbayh7gpa', // الاردن + 'xn--mgbbh1a', // بارت + 'xn--mgbbh1a71e', // بھارت + 'xn--mgbc0a9azcg', // المغرب + 'xn--mgbca7dzdo', // ابوظبي + 'xn--mgbcpq6gpa1a', // البحرين + 'xn--mgberp4a5d4ar', // السعودية + 'xn--mgbgu82a', // ڀارت + 'xn--mgbi4ecexp', // كاثوليك + 'xn--mgbpl2fh', // سودان + 'xn--mgbt3dhd', // همراه + 'xn--mgbtx2b', // عراق + 'xn--mgbx4cd0ab', // مليسيا + 'xn--mix891f', // 澳門 + 'xn--mk1bu44c', // 닷컴 + 'xn--mxtq1m', // 政府 + 'xn--ngbc5azd', // شبكة + 'xn--ngbe9e0a', // بيتك + 'xn--ngbrx', // عرب + 'xn--node', // გე + 'xn--nqv7f', // 机构 + 'xn--nqv7fs00ema', // 组织机构 + 'xn--nyqy26a', // 健康 + 'xn--o3cw4h', // ไทย + 'xn--ogbpf8fl', // سورية + 'xn--otu796d', // 招聘 + 'xn--p1acf', // рус + 'xn--p1ai', // рф + 'xn--pgbs0dh', // تونس + 'xn--pssy2u', // 大拿 + 'xn--q7ce6a', // ລາວ + 'xn--q9jyb4c', // みんな + 'xn--qcka1pmc', // グーグル + 'xn--qxa6a', // ευ + 'xn--qxam', // ελ + 'xn--rhqv96g', // 世界 + 'xn--rovu88b', // 書籍 + 'xn--rvc1e0am3e', // ഭാരതം + 'xn--s9brj9c', // ਭਾਰਤ + 'xn--ses554g', // 网址 + 'xn--t60b56a', // 닷넷 + 'xn--tckwe', // コム + 'xn--tiq49xqyj', // 天主教 + 'xn--unup4y', // 游戏 + 'xn--vermgensberater-ctb', // vermögensberater + 'xn--vermgensberatung-pwb', // vermögensberatung + 'xn--vhquv', // 企业 + 'xn--vuq861b', // 信息 + 'xn--w4r85el8fhu5dnra', // 嘉里大酒店 + 'xn--w4rs40l', // 嘉里 + 'xn--wgbh1c', // مصر + 'xn--wgbl6a', // قطر + 'xn--xhq521b', // 广东 + 'xn--xkc2al3hye2a', // இலங்கை + 'xn--xkc2dl3a5ee0h', // இந்தியா + 'xn--y9a3aq', // հայ + 'xn--yfro4i67o', // 新加坡 + 'xn--ygbi2ammx', // فلسطين + 'xn--zfr164b', // 政务 + 'xxx', + 'xyz', + 'yachts', + 'yahoo', + 'yamaxun', + 'yandex', + 'ye', + 'yodobashi', + 'yoga', + 'yokohama', + 'you', + 'youtube', + 'yt', + 'yun', + 'za', + 'zappos', + 'zara', + 'zero', + 'zip', + 'zm', + 'zone', + 'zuerich', + 'zw', + 'ελ', // xn--qxam + 'ευ', // xn--qxa6a + 'бг', // xn--90ae + 'бел', // xn--90ais + 'дети', // xn--d1acj3b + 'ею', // xn--e1a4c + 'католик', // xn--80aqecdr1a + 'ком', // xn--j1aef + 'мкд', // xn--d1alf + 'мон', // xn--l1acc + 'москва', // xn--80adxhks + 'онлайн', // xn--80asehdb + 'орг', // xn--c1avg + 'рус', // xn--p1acf + 'рф', // xn--p1ai + 'сайт', // xn--80aswg + 'срб', // xn--90a3ac + 'укр', // xn--j1amh + 'қаз', // xn--80ao21a + 'հայ', // xn--y9a3aq + 'ישראל', // xn--4dbrk0ce + 'קום', // xn--9dbq2a + 'ابوظبي', // xn--mgbca7dzdo + 'ارامكو', // xn--mgba3a3ejt + 'الاردن', // xn--mgbayh7gpa + 'البحرين', // xn--mgbcpq6gpa1a + 'الجزائر', // xn--lgbbat1ad8j + 'السعودية', // xn--mgberp4a5d4ar + 'العليان', // xn--mgba7c0bbn0a + 'المغرب', // xn--mgbc0a9azcg + 'امارات', // xn--mgbaam7a8h + 'ایران', // xn--mgba3a4f16a + 'بارت', // xn--mgbbh1a + 'بازار', // xn--mgbab2bd + 'بيتك', // xn--ngbe9e0a + 'بھارت', // xn--mgbbh1a71e + 'تونس', // xn--pgbs0dh + 'سودان', // xn--mgbpl2fh + 'سورية', // xn--ogbpf8fl + 'شبكة', // xn--ngbc5azd + 'عراق', // xn--mgbtx2b + 'عرب', // xn--ngbrx + 'عمان', // xn--mgb9awbf + 'فلسطين', // xn--ygbi2ammx + 'قطر', // xn--wgbl6a + 'كاثوليك', // xn--mgbi4ecexp + 'كوم', // xn--fhbei + 'مصر', // xn--wgbh1c + 'مليسيا', // xn--mgbx4cd0ab + 'موريتانيا', // xn--mgbah1a3hjkrd + 'موقع', // xn--4gbrim + 'همراه', // xn--mgbt3dhd + 'پاکستان', // xn--mgbai9azgqp6j + 'ڀارت', // xn--mgbgu82a + 'कॉम', // xn--11b4c3d + 'नेट', // xn--c2br7g + 'भारत', // xn--h2brj9c + 'भारतम्', // xn--h2breg3eve + 'भारोत', // xn--h2brj9c8c + 'संगठन', // xn--i1b6b1a6a2e + 'বাংলা', // xn--54b7fta0cc + 'ভারত', // xn--45brj9c + 'ভাৰত', // xn--45br5cyl + 'ਭਾਰਤ', // xn--s9brj9c + 'ભારત', // xn--gecrj9c + 'ଭାରତ', // xn--3hcrj9c + 'இந்தியா', // xn--xkc2dl3a5ee0h + 'இலங்கை', // xn--xkc2al3hye2a + 'சிங்கப்பூர்', // xn--clchc0ea0b2g2a9gcd + 'భారత్', // xn--fpcrj9c3d + 'ಭಾರತ', // xn--2scrj9c + 'ഭാരതം', // xn--rvc1e0am3e + 'ලංකා', // xn--fzc2c9e2c + 'คอม', // xn--42c2d9a + 'ไทย', // xn--o3cw4h + 'ລາວ', // xn--q7ce6a + 'გე', // xn--node + 'みんな', // xn--q9jyb4c + 'アマゾン', // xn--cckwcxetd + 'クラウド', // xn--gckr3f0f + 'グーグル', // xn--qcka1pmc + 'コム', // xn--tckwe + 'ストア', // xn--cck2b3b + 'セール', // xn--1ck2e1b + 'ファッション', // xn--bck1b9a5dre4c + 'ポイント', // xn--eckvdtc9d + '世界', // xn--rhqv96g + '中信', // xn--fiq64b + '中国', // xn--fiqs8s + '中國', // xn--fiqz9s + '中文网', // xn--fiq228c5hs + '亚马逊', // xn--jlq480n2rg + '企业', // xn--vhquv + '佛山', // xn--1qqw23a + '信息', // xn--vuq861b + '健康', // xn--nyqy26a + '八卦', // xn--45q11c + '公司', // xn--55qx5d + '公益', // xn--55qw42g + '台湾', // xn--kprw13d + '台灣', // xn--kpry57d + '商城', // xn--czru2d + '商店', // xn--czrs0t + '商标', // xn--czr694b + '嘉里', // xn--w4rs40l + '嘉里大酒店', // xn--w4r85el8fhu5dnra + '在线', // xn--3ds443g + '大拿', // xn--pssy2u + '天主教', // xn--tiq49xqyj + '娱乐', // xn--fjq720a + '家電', // xn--fct429k + '广东', // xn--xhq521b + '微博', // xn--9krt00a + '慈善', // xn--30rr7y + '我爱你', // xn--6qq986b3xl + '手机', // xn--kput3i + '招聘', // xn--otu796d + '政务', // xn--zfr164b + '政府', // xn--mxtq1m + '新加坡', // xn--yfro4i67o + '新闻', // xn--efvy88h + '时尚', // xn--9et52u + '書籍', // xn--rovu88b + '机构', // xn--nqv7f + '淡马锡', // xn--b4w605ferd + '游戏', // xn--unup4y + '澳門', // xn--mix891f + '点看', // xn--3pxu8k + '移动', // xn--6frz82g + '组织机构', // xn--nqv7fs00ema + '网址', // xn--ses554g + '网店', // xn--hxt814e + '网站', // xn--5tzm5g + '网络', // xn--io0a7i + '联通', // xn--8y0a063a + '谷歌', // xn--flw351e + '购物', // xn--g2xx48c + '通販', // xn--gk3at1e + '集团', // xn--3bst00m + '電訊盈科', // xn--fzys8d69uvgm + '飞利浦', // xn--kcrx77d1x4a + '食品', // xn--jvr189m + '餐厅', // xn--imr513n + '香格里拉', // xn--5su34j936bgsg + '香港', // xn--j6w193g + '닷넷', // xn--t60b56a + '닷컴', // xn--mk1bu44c + '삼성', // xn--cg4bki + '한국', // xn--3e0b707e +}; diff --git a/packages/stream_core/pubspec.yaml b/packages/stream_core/pubspec.yaml index 37527908..9b47c76c 100644 --- a/packages/stream_core/pubspec.yaml +++ b/packages/stream_core/pubspec.yaml @@ -37,6 +37,7 @@ dependencies: web_socket_channel: ^3.0.1 dev_dependencies: + benchmark_harness: ^2.4.0 build_runner: ^2.10.5 fake_async: ^1.3.3 json_serializable: ^6.9.5 diff --git a/packages/stream_core/test/utils/link_finder_test.dart b/packages/stream_core/test/utils/link_finder_test.dart new file mode 100644 index 00000000..4ce944e0 --- /dev/null +++ b/packages/stream_core/test/utils/link_finder_test.dart @@ -0,0 +1,161 @@ +import 'package:stream_core/stream_core.dart'; +import 'package:test/test.dart'; + +void main() { + final cases = <(String description, String text, List links)>[ + // Addresses without a scheme. + ('finds a domain', 'Some link: youtube.com', ['youtube.com']), + ('finds a domain with a path', 'Docs at getstream.io/chat', ['getstream.io/chat']), + ('finds a subdomain with a port and query', 'docs.example.co.uk:8080/a?b=c', ['docs.example.co.uk:8080/a?b=c']), + ('finds a www address', 'See www.google.com', ['www.google.com']), + ('finds an uppercase top-level domain', 'GOOGLE.COM and Example.Com', ['GOOGLE.COM', 'Example.Com']), + ('finds an IP address', 'Open 192.168.0.1:8080/x', ['192.168.0.1:8080/x']), + ('finds a domain that starts with an IP address', 'Open 1.2.3.4.nip.io', ['1.2.3.4.nip.io']), + ('finds a punycode domain', 'xn--80ak6aa92e.com', ['xn--80ak6aa92e.com']), + ('finds an internationalized domain', 'пример.рф bücher.de 例子.中国', ['пример.рф', 'bücher.de', '例子.中国']), + ('finds a top-level domain in punycode', 'пример.xn--p1ai', ['пример.xn--p1ai']), + ('finds a file name whose extension is a top-level domain', 'Open README.md', ['README.md']), + + // Addresses with a scheme. + ('finds an http address', 'http://getstream.io/chat', ['http://getstream.io/chat']), + ('finds an https address with any host', 'https://localhost:3000/path', ['https://localhost:3000/path']), + ('finds an uppercase scheme', 'HTTPS://example.com', ['HTTPS://example.com']), + ('finds a host that starts with an IP address', 'http://127.0.0.1.nip.io/x', ['http://127.0.0.1.nip.io/x']), + ('finds credentials before the host', 'https://user:pw@example.com/x', ['https://user:pw@example.com/x']), + + // Text that is not a link. + ('leaves out an unknown top-level domain', 'config.json and main.dart', []), + ('leaves out version numbers and abbreviations', 'v1.2.3, e.g. this', []), + ('leaves out an IP address with an octet out of range', '256.1.1.1', []), + ('leaves out a version number with more than four parts', 'version 1.2.3.4.5', []), + ('leaves out an unknown top-level domain in punycode', 'foo.xn--zzzz', []), + ('leaves out an email address', 'Mail anna.li@gmail.com', []), + ('leaves out the start of an email address with a tag', 'anna.li+tag@gmail.com', []), + ('leaves out the start of an email address with a percent sign', 'jo.me%x@y.com', []), + ('leaves out the start of an email address with a hyphen', 'abc.com-foo@x.com', []), + ('leaves out a mention', 'Hey @john.doe', []), + ('leaves out an address that continues a path', 'x/youtube.com', []), + ('leaves out a top-level domain that continues into a word', 'youtube.comx', []), + ('leaves out a label longer than 63 characters', '${'a' * 64}.com', []), + ('leaves out a host with a label longer than 63 characters', 'https://${'a' * 64}.com', []), + ('leaves out a domain longer than 253 characters', '${'a.' * 130}com', []), + ('leaves out a host longer than 253 characters', 'https://${'a.' * 130}com', []), + + // Where a link starts and ends. + ('finds a label of 63 characters', '${'a' * 63}.com', ['${'a' * 63}.com']), + ('ends a link at a hyphen after the top-level domain', 'youtube.com-foo', ['youtube.com']), + ('ends a link at a dot followed by an unknown label', 'youtube.com.x', ['youtube.com']), + ('leaves out the punctuation ending a sentence', 'See youtube.com/docs. Really?', ['youtube.com/docs']), + ('leaves out an at sign after a domain', 'Ping youtube.com@ now', ['youtube.com']), + ('leaves out a closing parenthesis around a link', '(see youtube.com/foo)', ['youtube.com/foo']), + ( + 'keeps a closing parenthesis opened in the link', + 'en.wikipedia.org/wiki/Foo_(bar)', + ['en.wikipedia.org/wiki/Foo_(bar)'], + ), + ( + 'leaves out quotes around a link', + '"youtube.com" “youtube.com” «youtube.com»', + ['youtube.com', 'youtube.com', 'youtube.com'], + ), + ('leaves out emphasis around a link', '*youtube.com* ~youtube.com~', ['youtube.com', 'youtube.com']), + ( + 'leaves out opening punctuation before a link', + '¿youtube.com? see—youtube.com (youtube.com) 「youtube.com」', + ['youtube.com', 'youtube.com', 'youtube.com', 'youtube.com'], + ), + ('leaves out brackets around a link', 'see [youtube.com]', ['youtube.com']), + ('starts a link after a non-breaking space', 'see\u00A0youtube.com', ['youtube.com']), + ('starts a link after text in a script written without spaces', '请访问google.com了解更多', ['google.com']), + ('reads only ASCII as the host after text in a script written without spaces', '访问example.中国', []), + ('starts a link after an ellipsis', 'te paso el enlace...idealista.com/piso', ['idealista.com/piso']), + ('leaves out a word after an ellipsis that is not an address', 'el finde...osea no', []), + ('starts an address with a scheme after punctuation', '_http://foo.com/bar_', ['http://foo.com/bar']), + + // Paths. + ('keeps brackets that open and close in a path', 'http://foo.com/path_[a]-{b}', ['http://foo.com/path_[a]-{b}']), + ('ends a path at a closing bracket that closes nothing', '(http://foo.com/bar)more', ['http://foo.com/bar']), + ('keeps symbols, emoji and other scripts in a path', r'http://foo.com/a0/$€¥🐼–', [r'http://foo.com/a0/$€¥🐼–']), + ( + 'keeps text in a script written without spaces in a path', + 'https://example.me/tmp/自殺でも?.png', + ['https://example.me/tmp/自殺でも?.png'], + ), + ('ends a path at a character that cannot be in an address', 'http://foo.bar/path|more', ['http://foo.bar/path']), + ('leaves out an at sign ending a path', 'http://foo.bar/path@', ['http://foo.bar/path']), + ('finds an IPv6 host after a scheme', 'http://[2001:db8::1]:8080/x', ['http://[2001:db8::1]:8080/x']), + ]; + + for (final (description, text, links) in cases) { + test('findLinks $description', () { + expect(_linkTexts(text), links); + }); + } + + test('findLinks opens an address without a scheme over https', () { + final link = findLinks('Docs at getstream.io/chat').single; + + expect(link.url, 'https://getstream.io/chat'); + }); + + test('findLinks keeps the scheme of an address that has one', () { + final link = findLinks('Docs at http://getstream.io/chat').single; + + expect(link.url, 'http://getstream.io/chat'); + }); + + test('findLinks reports where each link is in the text', () { + const text = 'See youtube.com and getstream.io.'; + + expect(findLinks(text), [ + _linkMatch(text, 'youtube.com'), + _linkMatch(text, 'getstream.io'), + ]); + }); + + test('findLinks finds the links after a long run of labels in a script written without spaces', () { + final text = '${'中a.' * 2000} youtube.com'; + + expect(_linkTexts(text), ['youtube.com']); + }); + + test('findLinks finds nothing in empty text', () { + expect(findLinks(''), isEmpty); + }); + + test('findLinkAt finds the link starting at the index', () { + const text = 'See youtube.com'; + + expect(findLinkAt(text, text.indexOf('youtube.com')), _linkMatch(text, 'youtube.com')); + }); + + test('findLinkAt finds nothing in the middle of a word', () { + const text = 'See youtube.com'; + + expect(findLinkAt(text, text.indexOf('tube.com')), isNull); + }); + + test('findLinkAt finds nothing at the end of the text', () { + const text = 'youtube.com'; + + expect(findLinkAt(text, text.length), isNull); + }); + + test('findLinkAt throws for an index past the end of the text', () { + const text = 'youtube.com'; + + expect(() => findLinkAt(text, text.length + 1), throwsRangeError); + }); + + test('findLinkAt throws for a negative index', () { + expect(() => findLinkAt('youtube.com', -1), throwsRangeError); + }); +} + +List _linkTexts(String text) => [for (final link in findLinks(text)) link.text]; + +// The link to [link] where it first appears in [text], opened over https. +LinkMatch _linkMatch(String text, String link) { + final start = text.indexOf(link); + return LinkMatch(start: start, end: start + link.length, text: link, url: 'https://$link'); +} diff --git a/packages/stream_core_flutter/CHANGELOG.md b/packages/stream_core_flutter/CHANGELOG.md index 5c59059e..d9229665 100644 --- a/packages/stream_core_flutter/CHANGELOG.md +++ b/packages/stream_core_flutter/CHANGELOG.md @@ -1,13 +1,26 @@ ## Upcoming +### ✨ Features + +- `StreamMessageText` now links a web address written without a scheme, such + as `example.com`, when it ends in a known top-level domain. It opens over + `https://`. + ### 🐛 Bug Fixes +- Fixed `StreamMessageText` not linking a web address right after a quote or + text in a script written without spaces, such as Chinese. - Fixed `StreamTextInput` ignoring typed text after the widgets around it, such as `leading` and `trailing`, changed while it had focus. - Fixed `StreamSnackbarHost` and `StreamSnackbarPopup` throwing in debug when they mount with a snackbar already queued on their messenger, for example after the widget holding them is rebuilt. +### 🔄 Changed + +- `StreamMessageText` opens a `www.` address over `https://` instead of + `http://`. + ## 0.5.3 ### ✨ Features diff --git a/packages/stream_core_flutter/lib/src/components/message/stream_message_text.dart b/packages/stream_core_flutter/lib/src/components/message/stream_message_text.dart index 976219c3..913bf6c4 100644 --- a/packages/stream_core_flutter/lib/src/components/message/stream_message_text.dart +++ b/packages/stream_core_flutter/lib/src/components/message/stream_message_text.dart @@ -1,6 +1,7 @@ import 'package:flutter/material.dart'; import 'package:flutter_markdown_plus/flutter_markdown_plus.dart'; import 'package:markdown/markdown.dart' as md; +import 'package:stream_core/stream_core.dart' show findLinkAt; import '../../factory/stream_component_factory.dart'; import '../../theme/components/stream_message_item_theme.dart'; @@ -81,6 +82,10 @@ final _emojiRegex = RegExp(r'\p{Emoji_Presentation}|\p{Emoji}\uFE0F', unicode: t /// Renders markdown text with themed styling. /// +/// Web addresses in the text are links. An address written without a scheme, +/// such as `example.com` or `www.example.com`, opens over `https://`, and is a +/// link only when it ends in a known top-level domain. +/// /// {@tool snippet} /// /// Basic markdown rendering: @@ -350,6 +355,7 @@ class DefaultStreamMessageText extends StatelessWidget { final effectiveInlineSyntaxes = [ _mentionSyntax, ...?props.inlineSyntaxes, + _linkSyntax, ]; // Base mention style — variant-specific overrides from [mentionStyles] @@ -393,6 +399,7 @@ class DefaultStreamMessageText extends StatelessWidget { // Stateless, so shared instead of recompiling its pattern on every build. static final _mentionSyntax = _StreamMentionSyntax(); + static final _linkSyntax = _StreamLinkSyntax(); // Shared across messages because building a sheet is costly. // The caller's sheet is merged after the lookup, not keyed on, because @@ -520,6 +527,64 @@ class _StreamMentionSyntax extends md.InlineSyntax { } } +// Links the web addresses found by [findLinkAt], with or without a scheme, +// and keeps an address in the label of a link as part of that link, as in +// `[example.com](https://example.com/docs)`. +// +// Other links, such as email addresses, are left to the markdown autolinks. +class _StreamLinkSyntax extends md.LinkSyntax { + @override + bool tryMatch(md.InlineParser parser, [int? startMatchPos]) { + if (super.tryMatch(parser, startMatchPos)) return true; + + final start = startMatchPos ?? parser.pos; + final link = findLinkAt(parser.source, start); + if (link == null) return false; + + final anchor = md.Element.text('a', link.text); + anchor.attributes['href'] = link.url; + + parser + ..writeText() + ..addNode(anchor) + ..consume(link.end - start); + return true; + } + + @override + md.Node createNode( + String destination, + String? title, { + required List Function() getChildren, + }) { + return super.createNode( + destination, + title, + getChildren: () => _withoutLinks(getChildren()), + ); + } + + // The [nodes] with every link in them replaced by its children. + static List _withoutLinks(List nodes) { + return [ + for (final node in nodes) + if (node is md.Element && node.tag == 'a') + ..._withoutLinks(node.children ?? const []) + else if (node is md.Element && node.children != null) + _withoutLinksIn(node) + else + node, + ]; + } + + // A copy of [element] with every link in its children replaced by its own + // children. + static md.Element _withoutLinksIn(md.Element element) { + final children = _withoutLinks(element.children!); + return md.Element(element.tag, children)..attributes.addAll(element.attributes); + } +} + // Renders `mention` elements as tappable styled text with pointer cursor. // // Per-element style resolution: reads the `type` attribute set by diff --git a/packages/stream_core_flutter/pubspec.yaml b/packages/stream_core_flutter/pubspec.yaml index 7aca1c5a..2986ead4 100644 --- a/packages/stream_core_flutter/pubspec.yaml +++ b/packages/stream_core_flutter/pubspec.yaml @@ -20,6 +20,7 @@ dependencies: path: ^1.9.0 path_provider: ^2.1.5 shimmer: ^3.0.0 + stream_core: ^0.5.1+1 svg_icon_widget: ^0.0.1+1 theme_extensions_builder_annotation: ^7.1.0 diff --git a/packages/stream_core_flutter/test/components/message/stream_message_text_links_test.dart b/packages/stream_core_flutter/test/components/message/stream_message_text_links_test.dart new file mode 100644 index 00000000..d064b245 --- /dev/null +++ b/packages/stream_core_flutter/test/components/message/stream_message_text_links_test.dart @@ -0,0 +1,149 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:markdown/markdown.dart' as md; +import 'package:stream_core_flutter/chat.dart'; + +void main() { + testWidgets('StreamMessageText opens a domain without a scheme over https', (tester) async { + expect(await _tapLink(tester, 'Some link: youtube.com', 'youtube.com'), 'https://youtube.com'); + }); + + testWidgets('StreamMessageText opens the path of a domain without a scheme', (tester) async { + expect(await _tapLink(tester, 'Docs at getstream.io/chat.', 'getstream.io/chat'), 'https://getstream.io/chat'); + }); + + testWidgets('StreamMessageText leaves an unknown top-level domain as plain text', (tester) async { + expect(await _tapLink(tester, 'Open config.json', 'config.json'), isNull); + }); + + testWidgets('StreamMessageText opens a www address over https', (tester) async { + expect(await _tapLink(tester, 'See www.google.com', 'www.google.com'), 'https://www.google.com'); + }); + + testWidgets('StreamMessageText opens an address with a scheme in quotes', (tester) async { + expect(await _tapLink(tester, 'Open "http://example.com/a" now', 'example.com'), 'http://example.com/a'); + }); + + testWidgets('StreamMessageText opens a domain in brackets that are not a link', (tester) async { + expect(await _tapLink(tester, 'See [youtube.com] for more', 'youtube.com'), 'https://youtube.com'); + }); + + testWidgets('StreamMessageText opens a domain after a closed bracket', (tester) async { + expect(await _tapLink(tester, '[1] youtube.com](https://example.com)', 'youtube.com'), 'https://youtube.com'); + }); + + testWidgets('StreamMessageText opens an email address as an email', (tester) async { + expect(await _tapLink(tester, 'Mail anna.li@gmail.com', 'anna.li'), 'mailto:anna.li@gmail.com'); + }); + + testWidgets('StreamMessageText opens a whole email address with a tag as an email', (tester) async { + expect(await _tapLink(tester, 'Mail anna.li+tag@gmail.com', 'anna.li'), 'mailto:anna.li+tag@gmail.com'); + }); + + testWidgets('StreamMessageText opens a markdown link to an email address', (tester) async { + expect( + await _tapLink(tester, '[Mail Anna](mailto:anna@example.com)', 'Mail Anna'), + 'mailto:anna@example.com', + ); + }); + + testWidgets('StreamMessageText keeps the target of a domain in a link label', (tester) async { + expect( + await _tapLink(tester, '[visit youtube.com](https://example.com)', 'youtube.com'), + 'https://example.com', + ); + }); + + testWidgets('StreamMessageText keeps the target of a domain in a link label with brackets', (tester) async { + expect( + await _tapLink(tester, '[youtube.com [docs]](https://example.com)', 'youtube.com'), + 'https://example.com', + ); + }); + + testWidgets('StreamMessageText keeps the target of a domain in a link label with an escaped bracket', (tester) async { + expect( + await _tapLink(tester, r'[a \] youtube.com](https://example.com)', 'youtube.com'), + 'https://example.com', + ); + }); + + testWidgets('StreamMessageText keeps the target of a domain in an emphasized link label', (tester) async { + expect( + await _tapLink(tester, '[*youtube.com*](https://example.com)', 'youtube.com'), + 'https://example.com', + ); + }); + + testWidgets('StreamMessageText keeps the target of a domain in a reference link label', (tester) async { + expect( + await _tapLink(tester, '[youtube.com][docs]\n\n[docs]: https://example.com', 'youtube.com'), + 'https://example.com', + ); + }); + + testWidgets('StreamMessageText keeps the target of a domain that names a link reference', (tester) async { + expect( + await _tapLink(tester, '[YouTube.com]\n\n[youtube.com]: https://example.com', 'YouTube.com'), + 'https://example.com', + ); + }); + + testWidgets('StreamMessageText leaves a domain in inline code as plain text', (tester) async { + expect(await _tapLink(tester, 'Run `youtube.com` now', 'youtube.com'), isNull); + }); + + testWidgets('StreamMessageText lets a custom syntax claim an address first', (tester) async { + expect( + await _tapLink( + tester, + 'Some link: youtube.com', + 'youtube.com', + inlineSyntaxes: [_FixedLinkSyntax('youtube.com', 'app://videos')], + ), + 'app://videos', + ); + }); +} + +// Renders [text] and returns the link opened by tapping [linkText], or null +// when the tap opens nothing. +Future _tapLink( + WidgetTester tester, + String text, + String linkText, { + List? inlineSyntaxes, +}) async { + String? opened; + await tester.pumpWidget( + MaterialApp( + home: Theme( + data: ThemeData(extensions: [StreamTheme()]), + child: StreamMessageLayout( + data: const StreamMessageLayoutData(), + child: StreamMessageText( + text, + inlineSyntaxes: inlineSyntaxes, + onTapLink: (_, href, _) => opened = href, + ), + ), + ), + ), + ); + + await tester.tapOnText(find.textRange.ofSubstring(linkText)); + return opened; +} + +// Links every [pattern] to [href]. +class _FixedLinkSyntax extends md.InlineSyntax { + _FixedLinkSyntax(String pattern, this.href) : super(RegExp.escape(pattern)); + + final String href; + + @override + bool onMatch(md.InlineParser parser, Match match) { + parser.addNode(md.Element.text('a', match[0]!)..attributes['href'] = href); + return true; + } +} diff --git a/scripts/generate_top_level_domains.dart b/scripts/generate_top_level_domains.dart new file mode 100644 index 00000000..6cfd10f4 --- /dev/null +++ b/scripts/generate_top_level_domains.dart @@ -0,0 +1,196 @@ +/// Top-Level Domain Generator +/// +/// Generates the set of top-level domains that `findLinks` and `findLinkAt` in +/// `stream_core` accept for an address written without a scheme, such as +/// `example.com`. +/// +/// Source: the IANA root zone list, +/// https://data.iana.org/TLD/tlds-alpha-by-domain.txt +/// +/// Output: `packages/stream_core/lib/src/utils/link_finder.tlds.g.dart`, a +/// part of `link_finder.dart` holding a private `const` set with one lowercase +/// domain per line. An internationalized domain +/// is listed in both its ASCII form (`xn--p1ai`) and its Unicode form (`рф`), +/// each commented with the other. +/// +/// Usage (from anywhere in the monorepo): +/// melos run generate:tlds +/// +/// Or directly, from the repository root: +/// dart run scripts/generate_top_level_domains.dart +/// +/// The `update_top_level_domains` workflow runs it monthly and opens a pull +/// request when the list has changed. Run it by hand to pick up a domain +/// sooner. It needs network access, so it is not part of `generate:all`. +/// +/// Steps: +/// 1. Downloads the list and checks that it starts with its version line +/// and that every entry is an ASCII domain. +/// 2. Decodes each `xn--` domain from punycode (RFC 3492). +/// 3. Sorts the domains and renders the set. +/// 4. Writes the file with the list's version in its header, or leaves it +/// as it is when the set has not changed, so a new list version alone +/// does not produce a diff. +/// +/// Throws, writing nothing, if the download fails or the list is not in the +/// expected format. +library; + +import 'dart:convert'; +import 'dart:io'; + +const _kSource = 'https://data.iana.org/TLD/tlds-alpha-by-domain.txt'; +const _kOutput = 'packages/stream_core/lib/src/utils/link_finder.tlds.g.dart'; +const _kVersionPrefix = '# Version '; +const _kPunycodePrefix = 'xn--'; + +final _asciiDomain = RegExp(r'^[a-z0-9-]+$'); + +Future main() async { + final lines = await _download(Uri.parse(_kSource)); + if (lines.isEmpty || !lines.first.startsWith(_kVersionPrefix)) { + throw FormatException('Expected the list to start with its version', lines.firstOrNull); + } + + final domains = [ + for (final line in lines.skip(1)) + if (line.trim().isNotEmpty) _validated(line.trim().toLowerCase()), + ]; + final entries = [for (final domain in domains) ..._entriesFor(domain)]..sort((a, b) => a.domain.compareTo(b.domain)); + + final file = File('${File.fromUri(Platform.script).parent.parent.path}/$_kOutput'); + final set = _renderSet(entries); + if (file.existsSync() && file.readAsStringSync().endsWith(set)) { + stdout.writeln('$_kOutput already has the ${domains.length} top-level domains'); + return; + } + + file + ..createSync(recursive: true) + ..writeAsStringSync(_renderHeader(lines.first.substring(2)) + set); + stdout.writeln('Wrote ${domains.length} top-level domains to $_kOutput'); +} + +// A domain in the set, and its other form when it is internationalized. +typedef _Entry = ({String domain, String? otherForm}); + +// The [domain] from the list, or a [FormatException] when it is not an ASCII +// domain, as when the download returns something other than the list. +String _validated(String domain) { + if (!_asciiDomain.hasMatch(domain)) throw FormatException('Expected an ASCII domain', domain); + return domain; +} + +// The entries for a domain from the list: one for an ASCII domain, and one +// per form for an internationalized one. +List<_Entry> _entriesFor(String ascii) { + if (!ascii.startsWith(_kPunycodePrefix)) return [(domain: ascii, otherForm: null)]; + + final unicode = _decodePunycode(ascii.substring(_kPunycodePrefix.length)); + return [(domain: ascii, otherForm: unicode), (domain: unicode, otherForm: ascii)]; +} + +Future> _download(Uri uri) async { + final client = HttpClient(); + try { + final response = await (await client.getUrl(uri)).close(); + if (response.statusCode != HttpStatus.ok) { + throw HttpException('GET $uri failed with ${response.statusCode}', uri: uri); + } + + return const LineSplitter().convert(await response.transform(utf8.decoder).join()); + } finally { + client.close(); + } +} + +// The header naming the source and the list version the set was built from. +String _renderHeader(String version) { + final buffer = StringBuffer() + ..writeln('// GENERATED CODE - DO NOT MODIFY BY HAND') + ..writeln('//') + ..writeln('// Generated by `melos run generate:tlds` from') + ..writeln('// $_kSource') + ..writeln('// $version') + ..writeln() + ..writeln("part of 'link_finder.dart';") + ..writeln(); + return buffer.toString(); +} + +// The set itself, compared with the existing file to skip a no-op write. +String _renderSet(List<_Entry> entries) { + final buffer = StringBuffer() + ..writeln('// The top-level domains in the IANA root zone, lowercase.') + ..writeln('//') + ..writeln('// Internationalized domains are in both their ASCII and Unicode forms, each') + ..writeln('// commented with the other.') + ..writeln('const _kIanaTopLevelDomains = {'); + + for (final (:domain, :otherForm) in entries) { + final comment = otherForm == null ? '' : ' // $otherForm'; + buffer.writeln(" '$domain',$comment"); + } + + buffer.writeln('};'); + return buffer.toString(); +} + +// RFC 3492 punycode decoding. +String _decodePunycode(String input) { + const base = 36; + const tMin = 1; + const tMax = 26; + const skew = 38; + const damp = 700; + const initialBias = 72; + const initialN = 128; + + int adapt(int delta, int numPoints, {required bool firstTime}) { + var scaled = firstTime ? delta ~/ damp : delta ~/ 2; + scaled += scaled ~/ numPoints; + var k = 0; + while (scaled > ((base - tMin) * tMax) ~/ 2) { + scaled ~/= base - tMin; + k += base; + } + return k + (base - tMin + 1) * scaled ~/ (scaled + skew); + } + + int digitValue(int codeUnit) { + if (codeUnit >= 0x30 && codeUnit <= 0x39) return codeUnit - 0x30 + 26; + if (codeUnit >= 0x61 && codeUnit <= 0x7A) return codeUnit - 0x61; + if (codeUnit >= 0x41 && codeUnit <= 0x5A) return codeUnit - 0x41; + throw FormatException('Invalid punycode digit', input); + } + + // The code points before the last `-` are copied as they are. With no `-`, + // every character is an encoded digit. + final delimiter = input.lastIndexOf('-'); + final output = [if (delimiter >= 0) ...input.substring(0, delimiter).codeUnits]; + + var n = initialN; + var i = 0; + var bias = initialBias; + var position = delimiter + 1; + + while (position < input.length) { + final oldI = i; + var weight = 1; + for (var k = base; ; k += base) { + if (position >= input.length) throw FormatException('Truncated punycode', input); + final digit = digitValue(input.codeUnitAt(position++)); + i += digit * weight; + final t = k <= bias ? tMin : (k >= bias + tMax ? tMax : k - bias); + if (digit < t) break; + weight *= base - t; + } + + bias = adapt(i - oldI, output.length + 1, firstTime: oldI == 0); + n += i ~/ (output.length + 1); + i %= output.length + 1; + output.insert(i++, n); + } + + return String.fromCharCodes(output); +}