gh-153494: Encode imaplib search criteria to the declared CHARSET

IMAP4.search(), sort(), thread() and the uid SORT/THREAD variants now encode
str search criteria to the declared charset, so international search text can
be passed as ordinary str.  A criterion passed as bytes is sent unchanged, for
use with a charset that Python has no codec for.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: serhiy-storchaka

Doc/library/imaplib.rst

@@ -584,6 +584,14 @@ An :class:`IMAP4` instance has the following methods:
    the ``UTF8=ACCEPT`` capability was enabled using the :meth:`enable`
    command.
 
+   A criterion passed as :class:`str` is encoded to *charset* (which must name
+   a codec known to Python); pass :class:`bytes` to send a criterion that is
+   already encoded, for example when *charset* is one that Python does not
+   support.
+
+   .. versionchanged:: next
+      ``str`` search criteria are encoded to *charset*.
+
    Example::
 
       # M is a connected IMAP4 instance...
@@ -651,8 +659,14 @@ An :class:`IMAP4` instance has the following methods:
    the interpretation of strings in the searching criteria.  It then returns the
    numbers of matching messages.
 
+   As with :meth:`search`, a *search_criterion* passed as :class:`str` is
+   encoded to *charset*; pass :class:`bytes` to send one already encoded.
+
    This is an ``IMAP4rev1`` extension command.
 
+   .. versionchanged:: next
+      ``str`` search criteria are encoded to *charset*.
+
 
 .. method:: IMAP4.starttls(ssl_context=None)
 
@@ -729,8 +743,14 @@ An :class:`IMAP4` instance has the following methods:
    returns the matching messages threaded according to the specified threading
    algorithm.
 
+   As with :meth:`search`, a *search_criterion* passed as :class:`str` is
+   encoded to *charset*; pass :class:`bytes` to send one already encoded.
+
    This is an ``IMAP4rev1`` extension command.
 
+   .. versionchanged:: next
+      ``str`` search criteria are encoded to *charset*.
+
 
 .. method:: IMAP4.uid(command, arg[, ...])
 

Lib/imaplib.py

@@ -920,11 +920,15 @@ def search(self, charset, *criteria):
 
         'data' is space separated list of matching message numbers.
         If UTF8 is enabled, charset MUST be None.
+
+        A 'criteria' passed as str is encoded to 'charset'; pass bytes to
+        send criteria that are already encoded.
         """
         name = 'SEARCH'
         if charset is not None:
             if self.utf8_enabled:
                 raise IMAP4.error("Non-None charset not valid in UTF8 mode")
+            criteria = self._encode_criteria(charset, criteria)
             typ, dat = self._simple_command(name,
                     'CHARSET', self._astring(charset), *criteria)
         else:
@@ -1000,6 +1004,7 @@ def sort(self, sort_criteria, charset, *search_criteria):
         #if not name in self.capabilities:      # Let the server decide!
         #       raise self.error('unimplemented extension command: %s' % name)
         sort_criteria = self._set_quote(sort_criteria)
+        search_criteria = self._encode_criteria(charset, search_criteria)
         if charset is not None:
             charset = self._astring(charset)
         typ, dat = self._simple_command(name, sort_criteria, charset, *search_criteria)
@@ -1068,6 +1073,7 @@ def thread(self, threading_algorithm, charset, *search_criteria):
         (type, [data]) = <instance>.thread(threading_algorithm, charset, search_criteria, ...)
         """
         name = 'THREAD'
+        search_criteria = self._encode_criteria(charset, search_criteria)
         if charset is not None:
             charset = self._astring(charset)
         typ, dat = self._simple_command(name, self._atom(threading_algorithm),
@@ -1106,12 +1112,14 @@ def uid(self, command, *args):
                     self._set_quote(flags))
         elif command == 'SORT':
             sort_criteria, charset, *search_criteria = args
+            search_criteria = self._encode_criteria(charset, search_criteria)
             if charset is not None:
                 charset = self._astring(charset)
             args = (self._set_quote(sort_criteria), charset,
                     *search_criteria)
         elif command == 'THREAD':
             threading_algorithm, charset, *search_criteria = args
+            search_criteria = self._encode_criteria(charset, search_criteria)
             if charset is not None:
                 charset = self._astring(charset)
             args = (self._atom(threading_algorithm), charset,
@@ -1524,6 +1532,17 @@ def _fetch_parts(self, arg):
             return arg
         return self._set_quote(arg)
 
+    def _encode_criteria(self, charset, criteria):
+        # Encode str search criteria to the declared CHARSET so the bytes on
+        # the wire match it.  bytes criteria are already encoded and pass
+        # through unchanged.  charset is None when no CHARSET is sent.
+        if charset is None:
+            return criteria
+        if isinstance(charset, (bytes, bytearray)):
+            charset = str(charset, 'ascii')
+        return tuple(c.encode(charset) if isinstance(c, str) else c
+                     for c in criteria)
+
     def _quote(self, arg):
         if isinstance(arg, str):
             arg = bytes(arg, self._encoding)

Lib/test/test_imaplib.py

@@ -205,6 +205,31 @@ def test_astring(self):
         self.assertEqual(m._astring('Entwürfe'), '"Entwürfe"'.encode())
         self.assertEqual(m._astring(b'Entw\xc3\xbcrfe'), b'"Entw\xc3\xbcrfe"')
 
+    def test_encode_criteria(self):
+        m = imaplib.IMAP4.__new__(imaplib.IMAP4)
+        enc = m._encode_criteria
+        # No charset: criteria are returned unchanged.
+        self.assertEqual(enc(None, ('TEXT', 'x')), ('TEXT', 'x'))
+        # str criteria are encoded to the charset; ASCII is charset-independent.
+        self.assertEqual(enc('UTF-8', ('TEXT', 'XXXXXX')), (b'TEXT', b'XXXXXX'))
+        # Non-ASCII text is encoded to the declared charset, including charsets
+        # other than ASCII, Latin-1 and UTF-8.
+        self.assertEqual(enc('UTF-8', ('"café"',)), ('"café"'.encode('utf-8'),))
+        self.assertEqual(enc('ISO-8859-1', ('"café"',)),
+                         ('"café"'.encode('latin-1'),))
+        self.assertEqual(enc('KOI8-U', ('"Київ"',)),
+                         ('"Київ"'.encode('koi8-u'),))
+        self.assertEqual(enc('SHIFT_JIS', ('"日本"',)),
+                         ('"日本"'.encode('shift_jis'),))
+        # bytes criteria are already encoded and pass through unchanged.
+        self.assertEqual(enc('SHIFT_JIS', (b'"already"',)), (b'"already"',))
+        # The charset name may itself be bytes.
+        self.assertEqual(enc(b'UTF-8', ('"café"',)), ('"café"'.encode('utf-8'),))
+        # A charset with no Python codec cannot encode str criteria; bytes
+        # criteria must be used with such server-only charsets.
+        self.assertRaises(LookupError, enc, 'NF_Z_62-010_(1973)', ('TEXT',))
+        self.assertEqual(enc('NF_Z_62-010_(1973)', (b'TEXT',)), (b'TEXT',))
+
     def test_astring_idempotent(self):
         # Quoting an already quoted argument should not change it, so that
         # quoting twice gives the same result as quoting once.
@@ -337,7 +362,11 @@ def handle(self):
                 except StopIteration:
                     self.continuation = None
                 continue
-            splitline = splitargs(line.decode().removesuffix('\r\n'))
+            self.server.line = line
+            # surrogateescape so a criterion encoded in a non-UTF-8 charset
+            # does not crash the handler; tests inspect server.line for bytes.
+            splitline = splitargs(line.decode('utf-8', 'surrogateescape')
+                                  .removesuffix('\r\n'))
             tag = splitline[0]
             cmd = splitline[1]
             args = splitline[2:]
@@ -1494,7 +1523,17 @@ def test_search(self):
         self.assertEqual(data, [b'43'])
         self.assertEqual(server.args, ['CHARSET', 'UTF-8', 'TEXT', 'XXXXXX'])
 
-        typ, data = client.search('NF_Z_62-010_(1973)', 'TEXT', 'XXXXXX')
+        # A non-ASCII str criterion is encoded to the declared charset (KOI8-U
+        # here, which is not UTF-8, so check the encoded bytes on the wire).
+        response[:] = ['* SEARCH 43']
+        typ, data = client.search('KOI8-U', 'SUBJECT', '"Київ"')
+        self.assertEqual(typ, 'OK')
+        self.assertIn(b'CHARSET KOI8-U ', server.line)
+        self.assertIn('"Київ"'.encode('koi8-u'), server.line)
+
+        # bytes criteria: NF_Z_62-010 has no Python codec, so this exercises
+        # charset-name quoting without criteria encoding.
+        typ, data = client.search('NF_Z_62-010_(1973)', b'TEXT', b'XXXXXX')
         self.assertEqual(typ, 'OK')
         self.assertEqual(server.args, ['CHARSET', '"NF_Z_62-010_(1973)"', 'TEXT', 'XXXXXX'])
 
@@ -1549,10 +1588,16 @@ def test_sort(self):
         self.assertEqual(data, [br''])
         self.assertEqual(server.args, ['(SUBJECT)', 'US-ASCII', 'TEXT', '"not in mailbox"'])
 
-        typ, data = client.sort('SUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"not in mailbox"')
+        typ, data = client.sort('SUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"not in mailbox"')
         self.assertEqual(typ, 'OK')
         self.assertEqual(server.args, ['(SUBJECT)', '"NF_Z_62-010_(1973)"', 'TEXT', '"not in mailbox"'])
 
+        # A non-ASCII str criterion is encoded to the declared charset.
+        response[:] = ['* SORT']
+        typ, data = client.sort('(SUBJECT)', 'KOI8-U', 'TEXT', '"Київ"')
+        self.assertEqual(typ, 'OK')
+        self.assertIn('"Київ"'.encode('koi8-u'), server.line)
+
     def test_uid_sort(self):
         response = []
         client, server = self._setup(make_simple_handler('UID', response,
@@ -1577,10 +1622,16 @@ def test_uid_sort(self):
         self.assertEqual(data, [br''])
         self.assertEqual(server.args, ['SORT', '(SUBJECT)', 'US-ASCII', 'TEXT', '"not in mailbox"'])
 
-        typ, data = client.uid('sort', 'SUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"not in mailbox"')
+        typ, data = client.uid('sort', 'SUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"not in mailbox"')
         self.assertEqual(typ, 'OK')
         self.assertEqual(server.args, ['SORT', '(SUBJECT)', '"NF_Z_62-010_(1973)"', 'TEXT', '"not in mailbox"'])
 
+        # A non-ASCII str criterion is encoded to the declared charset.
+        response[:] = ['* SORT']
+        typ, data = client.uid('sort', '(SUBJECT)', 'KOI8-U', 'TEXT', '"Київ"')
+        self.assertEqual(typ, 'OK')
+        self.assertIn('"Київ"'.encode('koi8-u'), server.line)
+
     def test_thread(self):
         response = []
         client, server = self._setup(make_simple_handler('THREAD', response))
@@ -1618,10 +1669,16 @@ def test_thread(self):
             b'(199)(200 202)(201)(203)(204)(205 206 207)(208)'])
         self.assertEqual(server.args, ['ORDEREDSUBJECT', 'US-ASCII', 'TEXT', '"gewp"'])
 
-        typ, data = client.thread('ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"gewp"')
+        typ, data = client.thread('ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"gewp"')
         self.assertEqual(typ, 'OK')
         self.assertEqual(server.args, ['ORDEREDSUBJECT', '"NF_Z_62-010_(1973)"', 'TEXT', '"gewp"'])
 
+        # A non-ASCII str criterion is encoded to the declared charset.
+        response[:] = ['* THREAD (1)']
+        typ, data = client.thread('ORDEREDSUBJECT', 'KOI8-U', 'TEXT', '"Київ"')
+        self.assertEqual(typ, 'OK')
+        self.assertIn('"Київ"'.encode('koi8-u'), server.line)
+
     def test_uid_thread(self):
         response = []
         client, server = self._setup(make_simple_handler('UID', response,
@@ -1660,10 +1717,16 @@ def test_uid_thread(self):
             b'(199)(200 202)(201)(203)(204)(205 206 207)(208)'])
         self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', 'US-ASCII', 'TEXT', '"gewp"'])
 
-        typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"gewp"')
+        typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"gewp"')
         self.assertEqual(typ, 'OK')
         self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', '"NF_Z_62-010_(1973)"', 'TEXT', '"gewp"'])
 
+        # A non-ASCII str criterion is encoded to the declared charset.
+        response[:] = ['* THREAD (1)']
+        typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'KOI8-U', 'TEXT', '"Київ"')
+        self.assertEqual(typ, 'OK')
+        self.assertIn('"Київ"'.encode('koi8-u'), server.line)
+
     def test_delete(self):
         client, server = self._setup(make_simple_handler('DELETE'))
         client.login('user', 'pass')

Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst

@@ -0,0 +1,6 @@
+:meth:`imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.sort` and
+:meth:`~imaplib.IMAP4.thread` (and the corresponding ``uid`` commands) now
+encode :class:`str` search criteria to the declared *charset*, so
+international search text can be passed as ordinary :class:`str`.  A criterion
+passed as :class:`bytes` is sent unchanged, for use with a charset that Python
+has no codec for.