From 1a52eaedce6f1d32cdb5ee18ecec74cfd82d5550 Mon Sep 17 00:00:00 2001 From: Valentin Samir Date: Sat, 15 Aug 2026 21:51:09 +0200 Subject: [PATCH 1/2] gh-83869: tarfile: compute next header offset using pax size for sparse file (GH-18562) In case of a sparse file, the tarinfo.size attribute is set to the sparse file expanded size (pax attribute GNU.sparse.size or GNU.sparse.size) and do not correspond to the actual size of the data block. The size of the data block is specified by the size pax header if present or by the ustar size header. Moreover, for GNU sparse 1.0 files, the data block start at the beginning of the sparse mapping and not after the sparse mapping and so the offset should be computed from here. Co-authored-by: Serhiy Storchaka --- Lib/tarfile.py | 11 +++- Lib/test/test_tarfile.py | 56 +++++++++++++++++++ ...0-02-19-16-35-52.gh-issue-83869.EPD_zn.rst | 6 ++ 3 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 592c4638c52c9b1..cee21bfc6fe5aa6 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1613,17 +1613,22 @@ def _proc_pax(self, tarfile): if self.type in (XHDTYPE, SOLARIS_XHDTYPE): # Patch the TarInfo object with the extended header info. next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors) - next.offset = self.offset if "size" in pax_headers: # If the extended header replaces the size field, # we need to recalculate the offset where the next # header starts. - offset = next.offset_data + offset = next.offset + BLOCKSIZE if next.isreg() or next.type not in SUPPORTED_TYPES: - offset += next._block(next.size) + try: + size = PAX_NUMBER_FIELDS["size"](pax_headers["size"]) + except ValueError: + size = 0 + offset += next._block(size) tarfile.offset = offset + next.offset = self.offset + return next def _proc_gnusparse_00(self, next, raw_headers): diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index bd544dfea51da32..c8c9b49fb6fc617 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1400,6 +1400,37 @@ def test_sparse_file_01(self): def test_sparse_file_10(self): self._test_sparse_file("gnu/sparse-1.0") + def test_sparse_file_10_pax_size(self): + # gh-83869: when the pax header replaces the size field, the offset + # of the next header must be computed from the size of the data in + # the archive, not from the apparent size of the sparse file. + data = b"payload!" * 4 + realsize = 1 << 20 + smap = b"1\n%d\n%d\n" % (realsize - len(data), len(data)) + smap += b"\0" * (-len(smap) % tarfile.BLOCKSIZE) + + sparse = tarfile.TarInfo("sparse") + sparse.size = len(smap) + len(data) + sparse.pax_headers = { + "GNU.sparse.major": "1", + "GNU.sparse.minor": "0", + "GNU.sparse.name": "sparse", + "GNU.sparse.realsize": str(realsize), + "size": str(sparse.size), + } + buf = sparse.tobuf(tarfile.PAX_FORMAT) + buf += smap + data + b"\0" * (-len(data) % tarfile.BLOCKSIZE) + + last = tarfile.TarInfo("last") + last.size = len(data) + buf += last.tobuf(tarfile.PAX_FORMAT) + buf += data + b"\0" * (-len(data) % tarfile.BLOCKSIZE) + buf += b"\0" * (tarfile.BLOCKSIZE * 2) + + with tarfile.open(fileobj=io.BytesIO(buf)) as tar: + self.assertEqual(tar.getnames(), ["sparse", "last"]) + self.assertEqual(tar.extractfile("last").read(), data) + @staticmethod def _fs_supports_holes(): # Return True if the platform knows the st_blocks stat attribute and @@ -1473,6 +1504,31 @@ def test_pax_global_headers(self): finally: tar.close() + def test_offset_after_global_header(self): + # gh-83869: a global header is a member of its own, the member which + # follows it keeps the offset of its own header. + rec = b"30 comment=global header here\n" + glob = tarfile.TarInfo("././@PaxHeader") + glob.type = tarfile.XGLTYPE + glob.size = len(rec) + buf = glob.tobuf(tarfile.USTAR_FORMAT) + buf += rec + b"\0" * (-len(rec) % tarfile.BLOCKSIZE) + + member = tarfile.TarInfo("member") + data = b"hello\n" + member.size = len(data) + offset = len(buf) + buf += member.tobuf(tarfile.USTAR_FORMAT) + buf += data + b"\0" * (-len(data) % tarfile.BLOCKSIZE) + buf += b"\0" * (tarfile.BLOCKSIZE * 2) + + with tarfile.open(fileobj=io.BytesIO(buf)) as tar: + tarinfo = tar.getmember("member") + self.assertEqual(tarinfo.offset, offset) + self.assertEqual(tarinfo.pax_headers.get("comment"), + "global header here") + self.assertEqual(tar.extractfile(tarinfo).read(), data) + def test_pax_number_fields(self): # All following number fields are read from the pax header. tar = tarfile.open(tarname, encoding="iso8859-1") diff --git a/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst b/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst new file mode 100644 index 000000000000000..efadbb289c3498c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst @@ -0,0 +1,6 @@ +Fix :mod:`tarfile` reading an archive with a GNU sparse 1.0 member whose +size is set in the pax extended header. +The offset of the next header was computed from the offset of the data, +which is already past the sparse map, and from the size of the member, +which can be the apparent size of the sparse file. +All following members were unreachable. From e3287f631f3c88ed80191aa222e7fc4ba91edd17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz?= Date: Sat, 15 Aug 2026 22:47:06 +0200 Subject: [PATCH 2/2] gh-155869: Fix data loss in dbm.dumb.reorganize() (GH-155872) --- Lib/dbm/dumb.py | 1 + Lib/test/test_dbm_dumb.py | 10 ++++++++++ .../2026-08-15-21-55-31.gh-issue-155869.yRUnQW.rst | 2 ++ 3 files changed, 13 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-08-15-21-55-31.gh-issue-155869.yRUnQW.rst diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py index c1c38da5101a572..a080f4e865508b0 100644 --- a/Lib/dbm/dumb.py +++ b/Lib/dbm/dumb.py @@ -311,6 +311,7 @@ def reorganize(self): reorganize_pos += blocks_occupied * _BLOCKSIZE f.truncate(reorganize_pos) + self._modified = True # Commit changes to index, which were not in-place. self._commit() diff --git a/Lib/test/test_dbm_dumb.py b/Lib/test/test_dbm_dumb.py index 672f9092207cf62..d977a81876df651 100644 --- a/Lib/test/test_dbm_dumb.py +++ b/Lib/test/test_dbm_dumb.py @@ -114,6 +114,16 @@ def test_write_write_read(self): with contextlib.closing(dumbdbm.open(_fname)) as f: self.assertEqual(f[b'1'], b'hello2') + def test_reorganize_persists_changed_offsets(self): + with dumbdbm.open(_fname, 'n') as f: + f[b'deleted'] = b'x' + f[b'retained'] = b'value' + del f[b'deleted'] + f.reorganize() + + with dumbdbm.open(_fname, 'r') as f: + self.assertEqual(f[b'retained'], b'value') + def test_str_read(self): self.init_db() with contextlib.closing(dumbdbm.open(_fname, 'r')) as f: diff --git a/Misc/NEWS.d/next/Library/2026-08-15-21-55-31.gh-issue-155869.yRUnQW.rst b/Misc/NEWS.d/next/Library/2026-08-15-21-55-31.gh-issue-155869.yRUnQW.rst new file mode 100644 index 000000000000000..19f76c176af0629 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-15-21-55-31.gh-issue-155869.yRUnQW.rst @@ -0,0 +1,2 @@ +Fix :meth:`!reorganize` in :mod:`dbm.dumb` failing to persist updated value +offsets, which could cause data loss after reopening the database.