Skip to content

Commit b7861ce

Browse files
committed
gh-152140: Reimplement _Extra to support .strip() and .iter()
Rework `_Extra` to keep the original `.strip()` behavior, to prevent altering existing callers. Add `.iter()` in place of `.strip()` to yield each subfield as tuples. This has better semantic and is proved to be more performant than subclassing `bytes` by benchmark. The method name change also allows feature detection for the behavior changes. Also refactor `ZipFile._decodeExtra` with this method.
1 parent 2ff724f commit b7861ce

3 files changed

Lines changed: 37 additions & 31 deletions

File tree

Lib/test/test_zipfile/test_core.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5840,7 +5840,7 @@ class StripExtraTests(unittest.TestCase):
58405840

58415841
ZIP64_EXTRA = 1
58425842

5843-
strip_extra = staticmethod(zipfile._strip_extra_fields)
5843+
strip_extra = staticmethod(zipfile._Extra.strip)
58445844

58455845
def test_no_data(self):
58465846
s = struct.Struct("<HH")

Lib/zipfile/__init__.py

Lines changed: 32 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,34 @@ class LargeZipFile(Exception):
194194

195195
_DD_SIGNATURE = 0x08074b50
196196

197+
198+
class _Extra:
199+
@classmethod
200+
def iter(cls, data):
201+
"""Iter through all subfields."""
202+
# early return for empty extra data
203+
if not data:
204+
return
205+
206+
pos, data_len = 0, len(data)
207+
while pos < data_len:
208+
try:
209+
xid, xlen = struct.unpack_from('<HH', data, pos)
210+
except struct.error:
211+
xid, xlen = None, 0
212+
yield data[pos:pos + 4 + xlen], xid, xlen
213+
pos += 4 + xlen
214+
215+
@classmethod
216+
def strip(cls, data, xids):
217+
"""Remove Extra fields with specified IDs."""
218+
result = b''
219+
for xfield, xid, _ in cls.iter(data):
220+
if xid not in xids:
221+
result += xfield
222+
return result
223+
224+
197225
def _check_zipfile(fp):
198226
try:
199227
endrec = _EndRecData(fp)
@@ -390,27 +418,6 @@ def _read_local_file_header(fp):
390418
raise BadZipFile("Bad magic number for file header")
391419
return fheader
392420

393-
def _strip_extra_fields(data, field_ids):
394-
"""Remove Extra fields with specified IDs and return a bytearray."""
395-
result = bytearray()
396-
397-
# early return for empty extra data
398-
if not data:
399-
return result
400-
401-
data_len = len(data)
402-
pos = 0
403-
while pos < data_len:
404-
try:
405-
xid, xlen = struct.unpack_from('<HH', data, pos)
406-
except struct.error:
407-
xid, xlen = None, 0
408-
if xid not in field_ids:
409-
result.extend(data[pos:pos + 4 + xlen])
410-
pos += 4 + xlen
411-
412-
return result
413-
414421

415422
class ZipInfo:
416423
"""Class with attributes describing each file in the ZIP archive."""
@@ -569,11 +576,9 @@ def _encodeFilenameFlags(self):
569576

570577
def _decodeExtra(self, filename_crc):
571578
# Try to decode the extra field.
572-
extra = self.extra
573579
unpack = struct.unpack
574-
while len(extra) >= 4:
575-
tp, ln = unpack('<HH', extra[:4])
576-
if ln+4 > len(extra):
580+
for extra, tp, ln in _Extra.iter(self.extra):
581+
if ln > len(extra):
577582
raise BadZipFile("Corrupt extra field %04x (size=%d)" % (tp, ln))
578583
if tp == 0x0001:
579584
data = extra[4:ln+4]
@@ -610,8 +615,6 @@ def _decodeExtra(self, filename_crc):
610615
except UnicodeDecodeError as e:
611616
raise BadZipFile('Corrupt unicode path extra field (0x7075): invalid utf-8 bytes') from e
612617

613-
extra = extra[ln+4:]
614-
615618
@classmethod
616619
def from_file(cls, filename, arcname=None, *, strict_timestamps=True):
617620
"""Construct an appropriate ZipInfo for a file on the filesystem.
@@ -2650,10 +2653,10 @@ def _write_end_record(self):
26502653
min_version = 0
26512654
if extra:
26522655
# Prepend a ZIP64 field to the extra's
2656+
extra_data = _Extra.strip(extra_data, (1,))
26532657
extra_data = struct.pack(
26542658
'<HH' + 'Q'*len(extra),
2655-
1, 8*len(extra), *extra
2656-
) + _strip_extra_fields(extra_data, (1,))
2659+
1, 8*len(extra), *extra) + extra_data
26572660

26582661
min_version = ZIP64_VERSION
26592662

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,4 @@
1-
Replace :class:`!_Extra` with :func:`!_strip_extra_fields` in the :mod:`zipfile` module.
1+
Rework :class:`!_Extra` in the :mod:`zipfile` module::
2+
* Improve performance for the mostly used :meth:`!_Extra.strip`.
3+
* Replace :meth:`!_Extra.split` with :meth:`!_Extra.iter` for better semantic and performance.
4+
* Make it a namespace class rather than a subclass of :class:`bytes`.

0 commit comments

Comments
 (0)