From d5cf0dd4380420acdc26c954585d0f813007e91c Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sat, 29 Apr 2023 19:53:00 +0530 Subject: [PATCH 01/34] gh-103987: fix crash in mmap module --- Lib/test/test_mmap.py | 17 +++++++++++++++++ Modules/mmapmodule.c | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 213a44d56f37b3..9ac72d37fdb30c 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -264,6 +264,23 @@ def test_bad_file_desc(self): # Try opening a bad file descriptor... self.assertRaises(OSError, mmap.mmap, -2, 4096) + def test_if_crash(self): # test for issue gh-103987 + with open("TESTFN", "w+") as f: + f.write("foobar") + f.flush() + + class X(object): + def __index__(self): + m.close() + return 1 + + m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) + self.assertEqual(m[1], 111) + with self.assertRaises(ValueError): + m[X()] # should not crash + + m.close() + def test_tougher_find(self): # Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2, # searching for data with embedded \0 bytes didn't work. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index fe76ca6eafaa88..9358339aae9fc4 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -933,9 +933,9 @@ mmap_item(mmap_object *self, Py_ssize_t i) static PyObject * mmap_subscript(mmap_object *self, PyObject *item) { - CHECK_VALID(NULL); if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + CHECK_VALID(NULL); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) From af99baa476351b32353463bedd33cd271d1cccd0 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sat, 29 Apr 2023 20:51:34 +0530 Subject: [PATCH 02/34] fix test --- Lib/test/test_mmap.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 9ac72d37fdb30c..af82f3e49e8a1d 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -265,19 +265,22 @@ def test_bad_file_desc(self): self.assertRaises(OSError, mmap.mmap, -2, 4096) def test_if_crash(self): # test for issue gh-103987 - with open("TESTFN", "w+") as f: - f.write("foobar") + with open("TESTFN", "wb+") as f: + data = b'aabaac\x00deef\x00\x00aa\x00' + n = len(data) + f.write(data) f.flush() - class X(object): + class X: def __index__(self): m.close() return 1 - m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) - self.assertEqual(m[1], 111) - with self.assertRaises(ValueError): - m[X()] # should not crash + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + + self.assertEqual(m[1], 97) + with self.assertRaises(ValueError): + m[X()] # should not crash m.close() From 6b7af107b3380a42f94a8130eac116b61962bb58 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sat, 29 Apr 2023 23:42:56 +0530 Subject: [PATCH 03/34] fix test --- Lib/test/test_mmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index af82f3e49e8a1d..e59d58e18c2374 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -265,7 +265,7 @@ def test_bad_file_desc(self): self.assertRaises(OSError, mmap.mmap, -2, 4096) def test_if_crash(self): # test for issue gh-103987 - with open("TESTFN", "wb+") as f: + with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' n = len(data) f.write(data) From 6ab6a708275bcb8dfd345dfbe73388a2a73877b8 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sat, 29 Apr 2023 23:45:32 +0530 Subject: [PATCH 04/34] fix test --- Lib/test/test_mmap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index e59d58e18c2374..57509878188646 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -264,7 +264,8 @@ def test_bad_file_desc(self): # Try opening a bad file descriptor... self.assertRaises(OSError, mmap.mmap, -2, 4096) - def test_if_crash(self): # test for issue gh-103987 + def test_unexpected_mmap_close(self): + # See gh-103987 with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' n = len(data) From 50cc20b80aa3904f000b59c20e32a8c037d44c0e Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 29 Apr 2023 18:23:17 +0000 Subject: [PATCH 05/34] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst diff --git a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst new file mode 100644 index 00000000000000..4cbefe909e52e5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst @@ -0,0 +1 @@ +Fix `mmap_subscript` in mmapmodule.c to avoid segmentation fault From 4d26d53d456ecb8ae662ea2a750cade72eb8a5f1 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sat, 29 Apr 2023 23:56:44 +0530 Subject: [PATCH 06/34] fix doc --- .../next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst index 4cbefe909e52e5..4e243cf0c1675f 100644 --- a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst +++ b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst @@ -1 +1 @@ -Fix `mmap_subscript` in mmapmodule.c to avoid segmentation fault +Fix ``mmap_subscript`` in mmapmodule.c to avoid segmentation fault From d8a1365214f2c70a9955a1c735f47467de3abfb3 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sun, 30 Apr 2023 01:45:11 +0530 Subject: [PATCH 07/34] fix test --- Lib/test/test_mmap.py | 24 +++++++++++++++++++++++- Modules/mmapmodule.c | 1 + 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 57509878188646..9bed0dbe0da40e 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -264,7 +264,7 @@ def test_bad_file_desc(self): # Try opening a bad file descriptor... self.assertRaises(OSError, mmap.mmap, -2, 4096) - def test_unexpected_mmap_close(self): + def test_unexpected_mmap_close_scenario_1(self): # See gh-103987 with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' @@ -283,7 +283,29 @@ def __index__(self): with self.assertRaises(ValueError): m[X()] # should not crash + with self.assertRaises(ValueError): + m[X():5] # should not crash + + m.close() # first CHECK_VALID will pick and raise ValueError + with self.assertRaises(ValueError): + m[0:5] # should not crash + + + def test_unexpected_mmap_close_scenario_2(self): + + with open(TESTFN, "wb+") as f: + data = b'aabaac\x00deef\x00\x00aa\x00' + n = len(data) + f.write(data) + f.flush() + + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + m.close() + with self.assertRaises(ValueError): + m["abc"] # first CHECK_VALID will pick and raise ValueError + + def test_tougher_find(self): # Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2, diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 9358339aae9fc4..81130afbc95080 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -933,6 +933,7 @@ mmap_item(mmap_object *self, Py_ssize_t i) static PyObject * mmap_subscript(mmap_object *self, PyObject *item) { + CHECK_VALID(NULL); if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); CHECK_VALID(NULL); From 428338f2cd68a1725b68ffd4ffb7e6eaaf9444cb Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sun, 30 Apr 2023 10:11:45 +0530 Subject: [PATCH 08/34] add new test --- Lib/test/test_mmap.py | 8 ++++++++ Modules/mmapmodule.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 9bed0dbe0da40e..cad097bca221ab 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -286,6 +286,14 @@ def __index__(self): with self.assertRaises(ValueError): m[X():5] # should not crash + with self.assertRaises(ValueError): + m[X()] = 1 + m[X()] # should through ValueError + + with self.assertRaises(ValueError): + m[X():5] = b'abcde' + m[X():5] # should through ValueError + m.close() # first CHECK_VALID will pick and raise ValueError with self.assertRaises(ValueError): m[0:5] # should not crash diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 81130afbc95080..3d97a452cabaed 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -936,7 +936,6 @@ mmap_subscript(mmap_object *self, PyObject *item) CHECK_VALID(NULL); if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - CHECK_VALID(NULL); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) @@ -946,6 +945,7 @@ mmap_subscript(mmap_object *self, PyObject *item) "mmap index out of range"); return NULL; } + CHECK_VALID(NULL); return PyLong_FromLong(Py_CHARMASK(self->data[i])); } else if (PySlice_Check(item)) { From 0a8213f200f0d351622f901752e9bcd7b1494fee Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sun, 30 Apr 2023 12:06:01 +0530 Subject: [PATCH 09/34] fix test --- Lib/test/test_mmap.py | 40 ++++++++++++++++++++++++++++++---------- Modules/mmapmodule.c | 3 +++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index cad097bca221ab..e4b6ac7addbd01 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -283,23 +283,30 @@ def __index__(self): with self.assertRaises(ValueError): m[X()] # should not crash - with self.assertRaises(ValueError): - m[X():5] # should not crash - with self.assertRaises(ValueError): m[X()] = 1 m[X()] # should through ValueError - with self.assertRaises(ValueError): - m[X():5] = b'abcde' - m[X():5] # should through ValueError + def test_unexpected_mmap_close_scenario_2(self): + # See gh-103987 + with open(TESTFN, "wb+") as f: + data = b'aabaac\x00deef\x00\x00aa\x00' + n = len(data) + f.write(data) + f.flush() - m.close() # first CHECK_VALID will pick and raise ValueError - with self.assertRaises(ValueError): - m[0:5] # should not crash + class X: + def __index__(self): + m.close() + return 1 + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) - def test_unexpected_mmap_close_scenario_2(self): + self.assertEqual(m[1], 97) + with self.assertRaises(ValueError): + m[X():5] # should not crash + + def test_unexpected_mmap_close_scenario_3(self): with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' @@ -313,6 +320,19 @@ def test_unexpected_mmap_close_scenario_2(self): with self.assertRaises(ValueError): m["abc"] # first CHECK_VALID will pick and raise ValueError + def test_unexpected_mmap_close_scenario_4(self): + + with open(TESTFN, "wb+") as f: + data = b'aabaac\x00deef\x00\x00aa\x00' + n = len(data) + f.write(data) + f.flush() + + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + + m.close() # first CHECK_VALID will pick and raise ValueError + with self.assertRaises(ValueError): + m[0:5] # should not crash def test_tougher_find(self): diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 3d97a452cabaed..316a5b689dbc50 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -954,6 +954,7 @@ mmap_subscript(mmap_object *self, PyObject *item) if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + CHECK_VALID(NULL); slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (slicelen <= 0) @@ -969,6 +970,8 @@ mmap_subscript(mmap_object *self, PyObject *item) if (result_buf == NULL) return PyErr_NoMemory(); + CHECK_VALID(NULL); + for (cur = start, i = 0; i < slicelen; cur += step, i++) { result_buf[i] = self->data[cur]; From b460efea00eff46dc21597258bf85a48ac6060b4 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sun, 30 Apr 2023 13:44:00 +0530 Subject: [PATCH 10/34] add checks in mmap_ass_subscript --- Lib/test/test_mmap.py | 41 +++++++++++++++++++++++++++++++++++++++-- Modules/mmapmodule.c | 2 ++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index e4b6ac7addbd01..329d88133d44e2 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -307,7 +307,7 @@ def __index__(self): m[X():5] # should not crash def test_unexpected_mmap_close_scenario_3(self): - + # See gh-103987 with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' n = len(data) @@ -321,7 +321,7 @@ def test_unexpected_mmap_close_scenario_3(self): m["abc"] # first CHECK_VALID will pick and raise ValueError def test_unexpected_mmap_close_scenario_4(self): - + # See gh-103987 with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' n = len(data) @@ -334,6 +334,43 @@ def test_unexpected_mmap_close_scenario_4(self): with self.assertRaises(ValueError): m[0:5] # should not crash + def test_unexpected_mmap_close_scenario_5(self): + # See gh-103987 + with open(TESTFN, "wb+") as f: + data = b'aabaac\x00deef\x00\x00aa\x00' + n = len(data) + f.write(data) + f.flush() + + class X: + def __index__(self): + m.close() + return 1 + + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + + m.close() # first CHECK_VALID will pick and raise ValueError + with self.assertRaises(ValueError): + m[X()] = 1 # should not crash + + def test_unexpected_mmap_close_scenario_6(self): + # See gh-103987 + with open(TESTFN, "wb+") as f: + data = b'aabaac\x00deef\x00\x00aa\x00' + n = len(data) + f.write(data) + f.flush() + + class X: + def __index__(self): + m.close() + return 1 + + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + + m.close() # first CHECK_VALID will pick and raise ValueError + with self.assertRaises(ValueError): + m[X():5] = b'abcde' # should not crash def test_tougher_find(self): # Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2, diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 316a5b689dbc50..443fc750531c77 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1056,6 +1056,7 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) "in range(0, 256)"); return -1; } + CHECK_VALID(-1); self->data[i] = (char) v; return 0; } @@ -1081,6 +1082,7 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) return -1; } + CHECK_VALID(-1); if (slicelen == 0) { } else if (step == 1) { From 55bd26ef022a2358eee43bb6e7d59c98e5368c09 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sun, 30 Apr 2023 14:06:34 +0530 Subject: [PATCH 11/34] fix test --- Lib/test/test_mmap.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 329d88133d44e2..a98187d2c37191 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -283,9 +283,6 @@ def __index__(self): with self.assertRaises(ValueError): m[X()] # should not crash - with self.assertRaises(ValueError): - m[X()] = 1 - m[X()] # should through ValueError def test_unexpected_mmap_close_scenario_2(self): # See gh-103987 From 7adecc4e73dee35ffa23d0d84ca25cb972f9e3ca Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sun, 30 Apr 2023 17:45:23 +0530 Subject: [PATCH 12/34] fix test --- Lib/test/test_mmap.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index a98187d2c37191..eb467a4a4b664b 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -315,7 +315,7 @@ def test_unexpected_mmap_close_scenario_3(self): m.close() with self.assertRaises(ValueError): - m["abc"] # first CHECK_VALID will pick and raise ValueError + m["abc"] def test_unexpected_mmap_close_scenario_4(self): # See gh-103987 @@ -327,7 +327,7 @@ def test_unexpected_mmap_close_scenario_4(self): m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) - m.close() # first CHECK_VALID will pick and raise ValueError + m.close() with self.assertRaises(ValueError): m[0:5] # should not crash @@ -344,9 +344,9 @@ def __index__(self): m.close() return 1 - m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_WRITE) - m.close() # first CHECK_VALID will pick and raise ValueError + m.close() with self.assertRaises(ValueError): m[X()] = 1 # should not crash @@ -363,9 +363,9 @@ def __index__(self): m.close() return 1 - m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_WRITE) - m.close() # first CHECK_VALID will pick and raise ValueError + m.close() with self.assertRaises(ValueError): m[X():5] = b'abcde' # should not crash From d8b989760f208b751f8c0b2ffb1ce709ce1167ab Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Fri, 12 May 2023 19:07:17 +0800 Subject: [PATCH 13/34] Update Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst --- .../next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst index 4e243cf0c1675f..060c218da99ffb 100644 --- a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst +++ b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst @@ -1 +1 @@ -Fix ``mmap_subscript`` in mmapmodule.c to avoid segmentation fault +add more `CHECK_VALID`s in `mmapmodule.c` to avoid the file being invalidated by Python code. From 6d5019244d9a954356b70e81cf9181dbad2b5f84 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sat, 29 Apr 2023 19:53:00 +0530 Subject: [PATCH 14/34] gh-103987: fix crash in mmap module --- Lib/test/test_mmap.py | 17 +++++++++++++++++ Modules/mmapmodule.c | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 213a44d56f37b3..9ac72d37fdb30c 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -264,6 +264,23 @@ def test_bad_file_desc(self): # Try opening a bad file descriptor... self.assertRaises(OSError, mmap.mmap, -2, 4096) + def test_if_crash(self): # test for issue gh-103987 + with open("TESTFN", "w+") as f: + f.write("foobar") + f.flush() + + class X(object): + def __index__(self): + m.close() + return 1 + + m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) + self.assertEqual(m[1], 111) + with self.assertRaises(ValueError): + m[X()] # should not crash + + m.close() + def test_tougher_find(self): # Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2, # searching for data with embedded \0 bytes didn't work. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index a470dd3c2f3bba..eaacb847eb810b 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -933,9 +933,9 @@ mmap_item(mmap_object *self, Py_ssize_t i) static PyObject * mmap_subscript(mmap_object *self, PyObject *item) { - CHECK_VALID(NULL); if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + CHECK_VALID(NULL); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) From 3784382c087167bbaf1949293402e878e285bd3f Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sat, 29 Apr 2023 20:51:34 +0530 Subject: [PATCH 15/34] fix test --- Lib/test/test_mmap.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 9ac72d37fdb30c..af82f3e49e8a1d 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -265,19 +265,22 @@ def test_bad_file_desc(self): self.assertRaises(OSError, mmap.mmap, -2, 4096) def test_if_crash(self): # test for issue gh-103987 - with open("TESTFN", "w+") as f: - f.write("foobar") + with open("TESTFN", "wb+") as f: + data = b'aabaac\x00deef\x00\x00aa\x00' + n = len(data) + f.write(data) f.flush() - class X(object): + class X: def __index__(self): m.close() return 1 - m = mmap.mmap(f.fileno(), 6, access=mmap.ACCESS_READ) - self.assertEqual(m[1], 111) - with self.assertRaises(ValueError): - m[X()] # should not crash + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + + self.assertEqual(m[1], 97) + with self.assertRaises(ValueError): + m[X()] # should not crash m.close() From 5e192494728f5a46d9dfb4ced2076adebbbc83af Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sat, 29 Apr 2023 23:42:56 +0530 Subject: [PATCH 16/34] fix test --- Lib/test/test_mmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index af82f3e49e8a1d..e59d58e18c2374 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -265,7 +265,7 @@ def test_bad_file_desc(self): self.assertRaises(OSError, mmap.mmap, -2, 4096) def test_if_crash(self): # test for issue gh-103987 - with open("TESTFN", "wb+") as f: + with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' n = len(data) f.write(data) From b0d1fcda186d902077892c40c6d178f957cf3d00 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sat, 29 Apr 2023 23:45:32 +0530 Subject: [PATCH 17/34] fix test --- Lib/test/test_mmap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index e59d58e18c2374..57509878188646 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -264,7 +264,8 @@ def test_bad_file_desc(self): # Try opening a bad file descriptor... self.assertRaises(OSError, mmap.mmap, -2, 4096) - def test_if_crash(self): # test for issue gh-103987 + def test_unexpected_mmap_close(self): + # See gh-103987 with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' n = len(data) From 1482ee7da5e4b81b8cfc34454e1607f1cdc57dea Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 29 Apr 2023 18:23:17 +0000 Subject: [PATCH 18/34] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst diff --git a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst new file mode 100644 index 00000000000000..4cbefe909e52e5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst @@ -0,0 +1 @@ +Fix `mmap_subscript` in mmapmodule.c to avoid segmentation fault From 8c76209e0158551d8b84c81a75f31ee8e11b5d1c Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sat, 29 Apr 2023 23:56:44 +0530 Subject: [PATCH 19/34] fix doc --- .../next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst index 4cbefe909e52e5..4e243cf0c1675f 100644 --- a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst +++ b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst @@ -1 +1 @@ -Fix `mmap_subscript` in mmapmodule.c to avoid segmentation fault +Fix ``mmap_subscript`` in mmapmodule.c to avoid segmentation fault From 20182975fe9f7d6b2ca12fb72794821fac0a94c6 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sun, 30 Apr 2023 01:45:11 +0530 Subject: [PATCH 20/34] fix test --- Lib/test/test_mmap.py | 24 +++++++++++++++++++++++- Modules/mmapmodule.c | 1 + 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 57509878188646..9bed0dbe0da40e 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -264,7 +264,7 @@ def test_bad_file_desc(self): # Try opening a bad file descriptor... self.assertRaises(OSError, mmap.mmap, -2, 4096) - def test_unexpected_mmap_close(self): + def test_unexpected_mmap_close_scenario_1(self): # See gh-103987 with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' @@ -283,7 +283,29 @@ def __index__(self): with self.assertRaises(ValueError): m[X()] # should not crash + with self.assertRaises(ValueError): + m[X():5] # should not crash + + m.close() # first CHECK_VALID will pick and raise ValueError + with self.assertRaises(ValueError): + m[0:5] # should not crash + + + def test_unexpected_mmap_close_scenario_2(self): + + with open(TESTFN, "wb+") as f: + data = b'aabaac\x00deef\x00\x00aa\x00' + n = len(data) + f.write(data) + f.flush() + + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + m.close() + with self.assertRaises(ValueError): + m["abc"] # first CHECK_VALID will pick and raise ValueError + + def test_tougher_find(self): # Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2, diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index eaacb847eb810b..aff90be984ec02 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -933,6 +933,7 @@ mmap_item(mmap_object *self, Py_ssize_t i) static PyObject * mmap_subscript(mmap_object *self, PyObject *item) { + CHECK_VALID(NULL); if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); CHECK_VALID(NULL); From 2cea2322795716908d9c0e737a807f1147615de8 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sun, 30 Apr 2023 10:11:45 +0530 Subject: [PATCH 21/34] add new test --- Lib/test/test_mmap.py | 8 ++++++++ Modules/mmapmodule.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 9bed0dbe0da40e..cad097bca221ab 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -286,6 +286,14 @@ def __index__(self): with self.assertRaises(ValueError): m[X():5] # should not crash + with self.assertRaises(ValueError): + m[X()] = 1 + m[X()] # should through ValueError + + with self.assertRaises(ValueError): + m[X():5] = b'abcde' + m[X():5] # should through ValueError + m.close() # first CHECK_VALID will pick and raise ValueError with self.assertRaises(ValueError): m[0:5] # should not crash diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index aff90be984ec02..0cb620c0818154 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -936,7 +936,6 @@ mmap_subscript(mmap_object *self, PyObject *item) CHECK_VALID(NULL); if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - CHECK_VALID(NULL); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) @@ -946,6 +945,7 @@ mmap_subscript(mmap_object *self, PyObject *item) "mmap index out of range"); return NULL; } + CHECK_VALID(NULL); return PyLong_FromLong(Py_CHARMASK(self->data[i])); } else if (PySlice_Check(item)) { From e0de742bb7b4d81bc0291843d4fea06fc298c562 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sun, 30 Apr 2023 12:06:01 +0530 Subject: [PATCH 22/34] fix test --- Lib/test/test_mmap.py | 40 ++++++++++++++++++++++++++++++---------- Modules/mmapmodule.c | 3 +++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index cad097bca221ab..e4b6ac7addbd01 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -283,23 +283,30 @@ def __index__(self): with self.assertRaises(ValueError): m[X()] # should not crash - with self.assertRaises(ValueError): - m[X():5] # should not crash - with self.assertRaises(ValueError): m[X()] = 1 m[X()] # should through ValueError - with self.assertRaises(ValueError): - m[X():5] = b'abcde' - m[X():5] # should through ValueError + def test_unexpected_mmap_close_scenario_2(self): + # See gh-103987 + with open(TESTFN, "wb+") as f: + data = b'aabaac\x00deef\x00\x00aa\x00' + n = len(data) + f.write(data) + f.flush() - m.close() # first CHECK_VALID will pick and raise ValueError - with self.assertRaises(ValueError): - m[0:5] # should not crash + class X: + def __index__(self): + m.close() + return 1 + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) - def test_unexpected_mmap_close_scenario_2(self): + self.assertEqual(m[1], 97) + with self.assertRaises(ValueError): + m[X():5] # should not crash + + def test_unexpected_mmap_close_scenario_3(self): with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' @@ -313,6 +320,19 @@ def test_unexpected_mmap_close_scenario_2(self): with self.assertRaises(ValueError): m["abc"] # first CHECK_VALID will pick and raise ValueError + def test_unexpected_mmap_close_scenario_4(self): + + with open(TESTFN, "wb+") as f: + data = b'aabaac\x00deef\x00\x00aa\x00' + n = len(data) + f.write(data) + f.flush() + + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + + m.close() # first CHECK_VALID will pick and raise ValueError + with self.assertRaises(ValueError): + m[0:5] # should not crash def test_tougher_find(self): diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 0cb620c0818154..f3c512d374bcaf 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -954,6 +954,7 @@ mmap_subscript(mmap_object *self, PyObject *item) if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + CHECK_VALID(NULL); slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (slicelen <= 0) @@ -969,6 +970,8 @@ mmap_subscript(mmap_object *self, PyObject *item) if (result_buf == NULL) return PyErr_NoMemory(); + CHECK_VALID(NULL); + for (cur = start, i = 0; i < slicelen; cur += step, i++) { result_buf[i] = self->data[cur]; From 5389a41d777cfa85d0ec2f65a8738c490b0df7a4 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sun, 30 Apr 2023 13:44:00 +0530 Subject: [PATCH 23/34] add checks in mmap_ass_subscript --- Lib/test/test_mmap.py | 41 +++++++++++++++++++++++++++++++++++++++-- Modules/mmapmodule.c | 2 ++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index e4b6ac7addbd01..329d88133d44e2 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -307,7 +307,7 @@ def __index__(self): m[X():5] # should not crash def test_unexpected_mmap_close_scenario_3(self): - + # See gh-103987 with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' n = len(data) @@ -321,7 +321,7 @@ def test_unexpected_mmap_close_scenario_3(self): m["abc"] # first CHECK_VALID will pick and raise ValueError def test_unexpected_mmap_close_scenario_4(self): - + # See gh-103987 with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' n = len(data) @@ -334,6 +334,43 @@ def test_unexpected_mmap_close_scenario_4(self): with self.assertRaises(ValueError): m[0:5] # should not crash + def test_unexpected_mmap_close_scenario_5(self): + # See gh-103987 + with open(TESTFN, "wb+") as f: + data = b'aabaac\x00deef\x00\x00aa\x00' + n = len(data) + f.write(data) + f.flush() + + class X: + def __index__(self): + m.close() + return 1 + + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + + m.close() # first CHECK_VALID will pick and raise ValueError + with self.assertRaises(ValueError): + m[X()] = 1 # should not crash + + def test_unexpected_mmap_close_scenario_6(self): + # See gh-103987 + with open(TESTFN, "wb+") as f: + data = b'aabaac\x00deef\x00\x00aa\x00' + n = len(data) + f.write(data) + f.flush() + + class X: + def __index__(self): + m.close() + return 1 + + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + + m.close() # first CHECK_VALID will pick and raise ValueError + with self.assertRaises(ValueError): + m[X():5] = b'abcde' # should not crash def test_tougher_find(self): # Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2, diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index f3c512d374bcaf..3418daf23911b9 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1056,6 +1056,7 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) "in range(0, 256)"); return -1; } + CHECK_VALID(-1); self->data[i] = (char) v; return 0; } @@ -1081,6 +1082,7 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) return -1; } + CHECK_VALID(-1); if (slicelen == 0) { } else if (step == 1) { From b5e37d7471e6cec647bb865c142b90d619327d63 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sun, 30 Apr 2023 14:06:34 +0530 Subject: [PATCH 24/34] fix test --- Lib/test/test_mmap.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 329d88133d44e2..a98187d2c37191 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -283,9 +283,6 @@ def __index__(self): with self.assertRaises(ValueError): m[X()] # should not crash - with self.assertRaises(ValueError): - m[X()] = 1 - m[X()] # should through ValueError def test_unexpected_mmap_close_scenario_2(self): # See gh-103987 From ed1715ecea2602417479ff41a2782cf18eb1ff0b Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sun, 30 Apr 2023 17:45:23 +0530 Subject: [PATCH 25/34] fix test --- Lib/test/test_mmap.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index a98187d2c37191..eb467a4a4b664b 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -315,7 +315,7 @@ def test_unexpected_mmap_close_scenario_3(self): m.close() with self.assertRaises(ValueError): - m["abc"] # first CHECK_VALID will pick and raise ValueError + m["abc"] def test_unexpected_mmap_close_scenario_4(self): # See gh-103987 @@ -327,7 +327,7 @@ def test_unexpected_mmap_close_scenario_4(self): m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) - m.close() # first CHECK_VALID will pick and raise ValueError + m.close() with self.assertRaises(ValueError): m[0:5] # should not crash @@ -344,9 +344,9 @@ def __index__(self): m.close() return 1 - m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_WRITE) - m.close() # first CHECK_VALID will pick and raise ValueError + m.close() with self.assertRaises(ValueError): m[X()] = 1 # should not crash @@ -363,9 +363,9 @@ def __index__(self): m.close() return 1 - m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_WRITE) - m.close() # first CHECK_VALID will pick and raise ValueError + m.close() with self.assertRaises(ValueError): m[X():5] = b'abcde' # should not crash From 49d109477bde0940a69f555f42bcddae4bb2aa2f Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Fri, 12 May 2023 19:07:17 +0800 Subject: [PATCH 26/34] Update Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst --- .../next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst index 4e243cf0c1675f..060c218da99ffb 100644 --- a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst +++ b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst @@ -1 +1 @@ -Fix ``mmap_subscript`` in mmapmodule.c to avoid segmentation fault +add more `CHECK_VALID`s in `mmapmodule.c` to avoid the file being invalidated by Python code. From 61cdd6e15faef743c0948819d79ca5aa9f3c24ff Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Fri, 12 May 2023 18:57:05 +0530 Subject: [PATCH 27/34] Add new test --- Lib/test/test_mmap.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index eb467a4a4b664b..aa75da1595e616 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -311,27 +311,18 @@ def test_unexpected_mmap_close_scenario_3(self): f.write(data) f.flush() - m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) - - m.close() - with self.assertRaises(ValueError): - m["abc"] - - def test_unexpected_mmap_close_scenario_4(self): - # See gh-103987 - with open(TESTFN, "wb+") as f: - data = b'aabaac\x00deef\x00\x00aa\x00' - n = len(data) - f.write(data) - f.flush() + class X: + def __index__(self): + m.close() + return 1 - m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) + m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_WRITE) m.close() with self.assertRaises(ValueError): - m[0:5] # should not crash + m[X()] = 1 # should not crash - def test_unexpected_mmap_close_scenario_5(self): + def test_unexpected_mmap_close_scenario_4(self): # See gh-103987 with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' @@ -348,9 +339,9 @@ def __index__(self): m.close() with self.assertRaises(ValueError): - m[X()] = 1 # should not crash + m[X():5] = b'abcde' # should not crash - def test_unexpected_mmap_close_scenario_6(self): + def test_unexpected_mmap_close_scenario_5(self): # See gh-103987 with open(TESTFN, "wb+") as f: data = b'aabaac\x00deef\x00\x00aa\x00' @@ -367,7 +358,7 @@ def __index__(self): m.close() with self.assertRaises(ValueError): - m[X():5] = b'abcde' # should not crash + m[X():5:1] = b'abcde' # should not crash def test_tougher_find(self): # Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2, From 341b90a1c4bd741f8f30eaf0e96cfd7c5513f9d9 Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Fri, 12 May 2023 21:46:10 +0800 Subject: [PATCH 28/34] Update Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst --- .../next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst index 060c218da99ffb..230edaa70c1946 100644 --- a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst +++ b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst @@ -1 +1 @@ -add more `CHECK_VALID`s in `mmapmodule.c` to avoid the file being invalidated by Python code. +add more ``CHECK_VALID``s in ``mmapmodule.c`` to avoid the file being invalidated by Python code. From 6dece9fcd69592621a3556e7d7aae96020c8a28e Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Fri, 12 May 2023 21:50:39 +0800 Subject: [PATCH 29/34] Update Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst --- .../next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst index 230edaa70c1946..f3b47c9fcc1bf6 100644 --- a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst +++ b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst @@ -1 +1 @@ -add more ``CHECK_VALID``s in ``mmapmodule.c`` to avoid the file being invalidated by Python code. +add more ``CHECK_VALID`` s in ``mmapmodule.c`` to avoid the file being invalidated by Python code. From 5cf8f90389f4171e285fb26b476ded51d74bb41c Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Fri, 12 May 2023 22:00:05 +0530 Subject: [PATCH 30/34] fix test --- Lib/test/test_mmap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index bab9b1d09e87fa..2abb1dbf6cfc63 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -339,7 +339,7 @@ def __index__(self): with self.assertRaises(ValueError): - m[X():5] = b'abcde' # should not crash + m[X():5] = b'abcd' # should not crash def test_unexpected_mmap_close_scenario_5(self): # See gh-103987 @@ -358,7 +358,7 @@ def __index__(self): with self.assertRaises(ValueError): - m[X():5:1] = b'abcde' # should not crash + m[X():5:1] = b'abcd' # should not crash def test_tougher_find(self): # Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2, From 843c2eb4d9613587173f3d8424b9968dc155881a Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Sat, 20 May 2023 04:35:22 +0800 Subject: [PATCH 31/34] add more checks, fix test cases --- Lib/test/test_mmap.py | 183 ++++++++++++++++++++---------------------- Modules/mmapmodule.c | 15 ++-- 2 files changed, 96 insertions(+), 102 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 2abb1dbf6cfc63..ba59cd4f46a223 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -264,102 +264,6 @@ def test_bad_file_desc(self): # Try opening a bad file descriptor... self.assertRaises(OSError, mmap.mmap, -2, 4096) - def test_unexpected_mmap_close_scenario_1(self): - # See gh-103987 - with open(TESTFN, "wb+") as f: - data = b'aabaac\x00deef\x00\x00aa\x00' - n = len(data) - f.write(data) - f.flush() - - class X: - def __index__(self): - m.close() - return 1 - - m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) - - self.assertEqual(m[1], 97) - with self.assertRaises(ValueError): - m[X()] # should not crash - - - def test_unexpected_mmap_close_scenario_2(self): - # See gh-103987 - with open(TESTFN, "wb+") as f: - data = b'aabaac\x00deef\x00\x00aa\x00' - n = len(data) - f.write(data) - f.flush() - - class X: - def __index__(self): - m.close() - return 1 - - m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_READ) - - self.assertEqual(m[1], 97) - with self.assertRaises(ValueError): - m[X():5] # should not crash - - def test_unexpected_mmap_close_scenario_3(self): - # See gh-103987 - with open(TESTFN, "wb+") as f: - data = b'aabaac\x00deef\x00\x00aa\x00' - n = len(data) - f.write(data) - f.flush() - - class X: - def __index__(self): - m.close() - return 1 - - m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_WRITE) - - - with self.assertRaises(ValueError): - m[X()] = 1 # should not crash - - def test_unexpected_mmap_close_scenario_4(self): - # See gh-103987 - with open(TESTFN, "wb+") as f: - data = b'aabaac\x00deef\x00\x00aa\x00' - n = len(data) - f.write(data) - f.flush() - - class X: - def __index__(self): - m.close() - return 1 - - m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_WRITE) - - - with self.assertRaises(ValueError): - m[X():5] = b'abcd' # should not crash - - def test_unexpected_mmap_close_scenario_5(self): - # See gh-103987 - with open(TESTFN, "wb+") as f: - data = b'aabaac\x00deef\x00\x00aa\x00' - n = len(data) - f.write(data) - f.flush() - - class X: - def __index__(self): - m.close() - return 1 - - m = mmap.mmap(f.fileno(), n, access=mmap.ACCESS_WRITE) - - - with self.assertRaises(ValueError): - m[X():5:1] = b'abcd' # should not crash - def test_tougher_find(self): # Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2, # searching for data with embedded \0 bytes didn't work. @@ -503,7 +407,6 @@ def test_move(self): m.move(0, 0, 1) m.move(0, 0, 0) - def test_anonymous(self): # anonymous mmap.mmap(-1, PAGE) m = mmap.mmap(-1, PAGESIZE) @@ -983,6 +886,92 @@ def test_resize_succeeds_with_error_for_second_named_mapping(self): self.assertEqual(m1[:data_length], data) self.assertEqual(m2[:data_length], data) + def test_mmap_closed_by_int_scenarios(self): + """ + gh-103987: Test that mmap objects raise ValueError + for accidentally closed mmap files + """ + + class MmapClosedByIntContext: + def __init__(self, access) -> None: + self.access = access + + def __enter__(self): + self.f = open(TESTFN, "w+b") + self.f.write(random.randbytes(100)) + self.f.flush() + + m = mmap.mmap(self.f.fileno(), 100, access=self.access) + + class X: + def __index__(self): + m.close() + return 10 + + return (m, X) + + def __exit__(self, exc_type, exc_value, traceback): + self.f.close() + + read_access_modes = [ + mmap.ACCESS_READ, + mmap.ACCESS_WRITE, + mmap.ACCESS_COPY, + mmap.ACCESS_DEFAULT, + ] + + write_access_modes = [ + mmap.ACCESS_WRITE, + mmap.ACCESS_COPY, + mmap.ACCESS_DEFAULT, + ] + + for access in read_access_modes: + with MmapClosedByIntContext(access) as (m, X): + with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): + m[X()] + + with MmapClosedByIntContext(access) as (m, X): + with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): + m[X() : 20] + + with MmapClosedByIntContext(access) as (m, X): + with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): + m[X() : 20 : 2] + + with MmapClosedByIntContext(access) as (m, X): + with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): + m[20 : X() : -2] + + with MmapClosedByIntContext(access) as (m, X): + with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): + m.read(X()) + + with MmapClosedByIntContext(access) as (m, X): + with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): + m.find(b"1", 1, X()) + + for access in write_access_modes: + with MmapClosedByIntContext(access) as (m, X): + with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): + m[X() : 20] = b"1" * 10 + + with MmapClosedByIntContext(access) as (m, X): + with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): + m[X() : 20 : 2] = b"1" * 5 + + with MmapClosedByIntContext(access) as (m, X): + with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): + m[20 : X() : -2] = b"1" * 5 + + with MmapClosedByIntContext(access) as (m, X): + with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): + m.move(1, 2, X()) + + with MmapClosedByIntContext(access) as (m, X): + with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): + m.write_byte(X()) + class LargeMmapTests(unittest.TestCase): def setUp(self): diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 3418daf23911b9..6dfcf729759bf8 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -284,7 +284,8 @@ mmap_read_method(mmap_object *self, CHECK_VALID(NULL); if (!PyArg_ParseTuple(args, "|O&:read", _Py_convert_optional_to_ssize_t, &num_bytes)) - return(NULL); + return NULL; + CHECK_VALID(NULL); /* silently 'adjust' out-of-range requests */ remaining = (self->pos < self->size) ? self->size - self->pos : 0; @@ -325,6 +326,7 @@ mmap_gfind(mmap_object *self, end = self->size; Py_ssize_t res; + CHECK_VALID(NULL); if (reverse) { res = _PyBytes_ReverseFind( self->data + start, end - start, @@ -388,7 +390,7 @@ mmap_write_method(mmap_object *self, CHECK_VALID(NULL); if (!PyArg_ParseTuple(args, "y*:write", &data)) - return(NULL); + return NULL; if (!is_writable(self)) { PyBuffer_Release(&data); @@ -401,6 +403,7 @@ mmap_write_method(mmap_object *self, return NULL; } + CHECK_VALID(NULL); memcpy(&self->data[self->pos], data.buf, data.len); self->pos += data.len; PyBuffer_Release(&data); @@ -420,6 +423,7 @@ mmap_write_byte_method(mmap_object *self, if (!is_writable(self)) return NULL; + CHECK_VALID(NULL); if (self->pos < self->size) { self->data[self->pos++] = value; Py_RETURN_NONE; @@ -724,6 +728,7 @@ mmap_move_method(mmap_object *self, PyObject *args) if (self->size - dest < cnt || self->size - src < cnt) goto bounds; + CHECK_VALID(NULL); memmove(&self->data[dest], &self->data[src], cnt); Py_RETURN_NONE; @@ -847,6 +852,7 @@ mmap_madvise_method(mmap_object *self, PyObject *args) length = self->size - start; } + CHECK_VALID(NULL); if (madvise(self->data + start, length, option) != 0) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -954,9 +960,9 @@ mmap_subscript(mmap_object *self, PyObject *item) if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } - CHECK_VALID(NULL); slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); - + + CHECK_VALID(NULL); if (slicelen <= 0) return PyBytes_FromStringAndSize("", 0); else if (step == 1) @@ -970,7 +976,6 @@ mmap_subscript(mmap_object *self, PyObject *item) if (result_buf == NULL) return PyErr_NoMemory(); - CHECK_VALID(NULL); for (cur = start, i = 0; i < slicelen; cur += step, i++) { From 168f5b107463e412a937b61075b2a40b2ad1410b Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Sat, 20 May 2023 04:36:29 +0800 Subject: [PATCH 32/34] Update Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst Co-authored-by: Jelle Zijlstra --- .../Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst index f3b47c9fcc1bf6..48c6d149a8ed42 100644 --- a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst +++ b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst @@ -1 +1,2 @@ -add more ``CHECK_VALID`` s in ``mmapmodule.c`` to avoid the file being invalidated by Python code. +In :mod:`mmap`, fix several bugs that could lead to access to memory-mapped files after +they have been invalidated. From 435ed41715d1ddf9d4090e3f530dbef27150c3de Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Sat, 20 May 2023 04:38:08 +0800 Subject: [PATCH 33/34] remove trailing spaces --- Modules/mmapmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6dfcf729759bf8..8e5a0bde889901 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -961,7 +961,7 @@ mmap_subscript(mmap_object *self, PyObject *item) return NULL; } slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); - + CHECK_VALID(NULL); if (slicelen <= 0) return PyBytes_FromStringAndSize("", 0); From 320feacc14b74e8815fd34eb7ae4a5e8da456ad0 Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Sat, 20 May 2023 09:56:59 +0800 Subject: [PATCH 34/34] Update Lib/test/test_mmap.py Co-authored-by: Jelle Zijlstra --- Lib/test/test_mmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index ba59cd4f46a223..517cbe0cb115ab 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -889,7 +889,7 @@ def test_resize_succeeds_with_error_for_second_named_mapping(self): def test_mmap_closed_by_int_scenarios(self): """ gh-103987: Test that mmap objects raise ValueError - for accidentally closed mmap files + for closed mmap files """ class MmapClosedByIntContext: