Skip to content

Commit 6860dc5

Browse files
committed
Rework and optimize _Extra
Make `_Extra` a normal class rather than a subclass of `bytes`, avoiding the performance penalty of copying the passed bytes upon each instantiation. Introduce `.iter()` in place of `.strip()` to yield each subfield info as a tuple. This provides better semantics and is more performant than subclassing `bytes`. The method name change also enables feature detection for these behavior changes. Stop using memoryview internally because each slice creates an overhead of around 184 bytes, making it inefficient for small bytes operations (each extra field is usually only several to tens of bytes). Remove `.read_one()` since it is now unused and obsolete.
1 parent 3a2147e commit 6860dc5

1 file changed

Lines changed: 30 additions & 31 deletions

File tree

Lib/zipfile/__init__.py

Lines changed: 30 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -195,53 +195,53 @@ class LargeZipFile(Exception):
195195
_DD_SIGNATURE = 0x08074b50
196196

197197

198-
class _Extra(bytes):
198+
class _Extra:
199199
FIELD_STRUCT = struct.Struct('<HH')
200200

201-
def __new__(cls, val, id=None):
202-
return super().__new__(cls, val)
203-
204-
def __init__(self, val, id=None):
205-
self.id = id
206-
207201
@classmethod
208-
def read_one(cls, raw, validate=False):
209-
try:
210-
xid, xlen = cls.FIELD_STRUCT.unpack_from(raw)
211-
except struct.error:
212-
xid, xlen = None, 0
213-
else:
214-
if validate and xlen + 4 > len(raw):
215-
raise BadZipFile("Corrupt extra field %04x (size=%d)" % (xid, xlen))
216-
return cls(raw[:4+xlen], xid), raw[4+xlen:]
202+
def iter(cls, data, validate=False):
203+
"""Iter through and yield each (field, id)."""
204+
# early return for empty extra data
205+
if not data:
206+
return
217207

218-
@classmethod
219-
def split(cls, data, validate=False):
220-
# use memoryview for zero-copy slices
221-
rest = memoryview(data)
222-
while rest:
223-
extra, rest = cls.read_one(rest, validate=validate)
224-
yield extra
208+
pos, data_len = 0, len(data)
209+
while pos < data_len:
210+
try:
211+
xid, xlen = cls.FIELD_STRUCT.unpack_from(data, pos)
212+
except struct.error:
213+
xid, xlen = None, 0
214+
else:
215+
if validate and pos + 4 + xlen > data_len:
216+
raise BadZipFile(
217+
"Corrupt extra field %04x (size=%d)" % (xid, xlen))
218+
yield data[pos:pos + 4 + xlen], xid
219+
pos += 4 + xlen
225220

226221
@classmethod
227222
def strip(cls, data, xids):
228223
"""Remove Extra fields with specified IDs."""
229224
return b''.join(
230225
ex
231-
for ex in cls.split(data)
232-
if ex.id not in xids
226+
for ex, xid in cls.iter(data)
227+
if xid not in xids
233228
)
234229

235230
@classmethod
236231
def update(cls, data, extra):
237232
"""Insert fields from extra and strip duplicates."""
233+
# early return for empty data
234+
if not data:
235+
return extra
236+
238237
extras = {
239-
ex.id: ex
240-
for ex in cls.split(extra, True)
241-
if ex.id is not None
238+
xid: ex
239+
for ex, xid in cls.iter(extra)
240+
if xid is not None
242241
}
243242
# New fields first since data may have a corrupted tail that renders
244-
# following fields inaccessible.
243+
# following fields inaccessible. (The caller is responsible for making
244+
# sure that extra is valid.)
245245
return b''.join(extras.values()) + cls.strip(data, extras)
246246

247247

@@ -600,8 +600,7 @@ def _encodeFilenameFlags(self):
600600
def _decodeExtra(self, filename_crc):
601601
# Try to decode the extra field.
602602
unpack = struct.unpack
603-
for extra in _Extra.split(self.extra, True):
604-
tp = extra.id
603+
for extra, tp in _Extra.iter(self.extra, True):
605604
if tp == 0x0001:
606605
data = extra[4:]
607606
# ZIP64 extension (large files and/or large archives)

0 commit comments

Comments
 (0)