diff --git a/Lib/test/test_capi/test_marshal.py b/Lib/test/test_capi/test_marshal.py index 82a20c44fac424..972ff4ed53d687 100644 --- a/Lib/test/test_capi/test_marshal.py +++ b/Lib/test/test_capi/test_marshal.py @@ -1,19 +1,54 @@ +# Test PyMarshal C API + import marshal import os.path +import struct import unittest from test import support from test.support import import_helper from test.support import os_helper -from test.test_marshal import HelperMixin, omit_last_byte # Skip this test if _testcapi is are not available. _testcapi = import_helper.import_module('_testcapi') +def noop_func(): + pass + +NULL = None +SIMPLE_OBJECT = 123 +# Only test a few objects: see test_marshal for more exhaustive tests +TEST_OBJECTS = ( + '\u20ac', + b'abc', + True, + 123, + 45.6, + 7+8j, + 'long line '*1000, + # Check that serializing code object is allowed (allow_code = 1) + noop_func.__code__, +) +UNMARSHALLABLE = object() + +# Invalid marshal data +JUNK_BYTES = b'\xff' * 32 + + +def read_file(filename): + with open(filename, 'rb') as fp: + return fp.read() + + +def write_file(filename, data): + with open(filename, 'wb') as fp: + fp.write(data) + + @support.cpython_only -class CAPI_TestCase(unittest.TestCase, HelperMixin): +class CAPI_TestCase(unittest.TestCase): def test_read_from_file_error(self): # A read error is reported as OSError, not EOFError. @@ -38,89 +73,163 @@ def test_write_to_file_error(self): _testcapi.pymarshal_write_object_to_file(obj, '/dev/full', marshal.version) - def test_write_unmarshallable_to_file(self): - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with self.assertRaisesRegex(ValueError, 'unmarshallable object'): - _testcapi.pymarshal_write_object_to_file(object(), os_helper.TESTFN, - marshal.version) + def check_object(self, obj2, obj): + self.assertEqual(obj2, obj) + self.assertEqual(type(obj2), type(obj)) def test_write_long_to_file(self): - for v in range(marshal.version + 1): - _testcapi.pymarshal_write_long_to_file(0x12345678, os_helper.TESTFN, v) - with open(os_helper.TESTFN, 'rb') as f: - data = f.read() - os_helper.unlink(os_helper.TESTFN) - self.assertEqual(data, b'\x78\x56\x34\x12') + # Test PyMarshal_WriteLongToFile() + write_long_to_file = _testcapi.pymarshal_write_long_to_file + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + + def mask32(value): + res = value & (2 ** 32 - 1) + if res >= 2147483648: + return res - 4294967296 + else: + return res + + limit = 2 ** 31 + values = [ + _testcapi.LONG_MIN, _testcapi.LONG_MAX, + -limit, -limit + 2, limit - 2, limit - 1, + 0, 123, -123, + ] + # Test values larger than 32-bit on platforms with 64-bit C long + if _testcapi.LONG_MAX > (2**31-1): + values.extend((-limit - 2, limit, limit + 2)) + + for version in range(marshal.version + 1): + for value in values: + with self.subTest(value=value, version=version): + write_long_to_file(value, filename, version) + data = read_file(filename) + self.assertEqual(len(data), 4) + value2 = struct.unpack('tok, new_token, &length); parser_token->type = token_type == NAME ? _get_keyword_or_name_type(p, text, length) : token_type; - parser_token->bytes = PyBytes_FromStringAndSize(text, length); - if (parser_token->bytes == NULL) { - return -1; + if (token_type == NAME || token_needs_text(parser_token->type)) { + parser_token->bytes = PyBytes_FromStringAndSize(text, length); + if (parser_token->bytes == NULL) { + return -1; + } + if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) { + Py_DECREF(parser_token->bytes); + return -1; + } } - if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) { - Py_DECREF(parser_token->bytes); - return -1; + else { + parser_token->bytes = NULL; } parser_token->metadata = NULL; diff --git a/Python/assemble.c b/Python/assemble.c index 4bbebe30299906..8b92042345f150 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -48,8 +48,10 @@ instr_size(instruction *instr) } struct assembler { - PyObject *a_bytecode; /* bytes containing bytecode */ + PyBytesWriter *a_bytecode_writer; /* writer containing bytecode */ + PyObject *a_bytecode; /* bytes containing bytecode */ int a_offset; /* offset into bytecode */ + PyBytesWriter *a_except_table_writer; /* writer containing exception table */ PyObject *a_except_table; /* bytes containing exception table */ int a_except_table_off; /* offset into exception table */ /* Location Info */ @@ -64,38 +66,40 @@ assemble_init(struct assembler *a, int firstlineno) { memset(a, 0, sizeof(struct assembler)); a->a_lineno = firstlineno; - a->a_bytecode = PyBytes_FromStringAndSize(NULL, DEFAULT_CODE_SIZE); - if (a->a_bytecode == NULL) { + a->a_bytecode_writer = PyBytesWriter_Create(DEFAULT_CODE_SIZE); + if (a->a_bytecode_writer == NULL) { goto error; } a->a_linetable_writer = PyBytesWriter_Create(DEFAULT_CNOTAB_SIZE); if (a->a_linetable_writer == NULL) { goto error; } - a->a_except_table = PyBytes_FromStringAndSize(NULL, DEFAULT_LNOTAB_SIZE); - if (a->a_except_table == NULL) { + a->a_except_table_writer = PyBytesWriter_Create(DEFAULT_LNOTAB_SIZE); + if (a->a_except_table_writer == NULL) { goto error; } return SUCCESS; error: - Py_CLEAR(a->a_bytecode); + PyBytesWriter_Discard(a->a_bytecode_writer); PyBytesWriter_Discard(a->a_linetable_writer); - Py_CLEAR(a->a_except_table); + PyBytesWriter_Discard(a->a_except_table_writer); return ERROR; } static void assemble_free(struct assembler *a) { - Py_XDECREF(a->a_bytecode); + PyBytesWriter_Discard(a->a_bytecode_writer); PyBytesWriter_Discard(a->a_linetable_writer); + PyBytesWriter_Discard(a->a_except_table_writer); + Py_XDECREF(a->a_bytecode); Py_XDECREF(a->a_linetable); Py_XDECREF(a->a_except_table); } static inline void write_except_byte(struct assembler *a, int byte) { - unsigned char *p = (unsigned char *) PyBytes_AS_STRING(a->a_except_table); + unsigned char *p = (unsigned char *) PyBytesWriter_GetData(a->a_except_table_writer); p[a->a_except_table_off++] = byte; } @@ -133,9 +137,9 @@ assemble_emit_exception_table_entry(struct assembler *a, int start, int end, int handler_offset, _PyExceptHandlerInfo *handler) { - Py_ssize_t len = PyBytes_GET_SIZE(a->a_except_table); + Py_ssize_t len = PyBytesWriter_GetSize(a->a_except_table_writer); if (a->a_except_table_off + MAX_SIZE_OF_ENTRY >= len) { - RETURN_IF_ERROR(_PyBytes_Resize(&a->a_except_table, len * 2)); + RETURN_IF_ERROR(PyBytesWriter_Resize(a->a_except_table_writer, len * 2)); } int size = end-start; assert(end > start); @@ -412,7 +416,7 @@ write_instr(_Py_CODEUNIT *codestr, instruction *instr, int ilen) static int assemble_emit_instr(struct assembler *a, instruction *instr) { - Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode); + Py_ssize_t len = PyBytesWriter_GetSize(a->a_bytecode_writer); _Py_CODEUNIT *code; int size = instr_size(instr); @@ -421,9 +425,9 @@ assemble_emit_instr(struct assembler *a, instruction *instr) PyErr_NoMemory(); return ERROR; } - RETURN_IF_ERROR(_PyBytes_Resize(&a->a_bytecode, len * 2)); + RETURN_IF_ERROR(PyBytesWriter_Resize(a->a_bytecode_writer, len * 2)); } - code = (_Py_CODEUNIT *)PyBytes_AS_STRING(a->a_bytecode) + a->a_offset; + code = (_Py_CODEUNIT *)PyBytesWriter_GetData(a->a_bytecode_writer) + a->a_offset; a->a_offset += size; write_instr(code, instr, size); return SUCCESS; @@ -444,7 +448,12 @@ assemble_emit(struct assembler *a, instr_sequence *instrs, RETURN_IF_ERROR(assemble_exception_table(a, instrs)); - RETURN_IF_ERROR(_PyBytes_Resize(&a->a_except_table, a->a_except_table_off)); + a->a_except_table = PyBytesWriter_FinishWithSize(a->a_except_table_writer, + a->a_except_table_off); + a->a_except_table_writer = NULL; + if (a->a_except_table == NULL) { + return ERROR; + } RETURN_IF_ERROR(_PyCompile_ConstCacheMergeOne(const_cache, &a->a_except_table)); a->a_linetable = PyBytesWriter_FinishWithSize(a->a_linetable_writer, @@ -455,7 +464,12 @@ assemble_emit(struct assembler *a, instr_sequence *instrs, } RETURN_IF_ERROR(_PyCompile_ConstCacheMergeOne(const_cache, &a->a_linetable)); - RETURN_IF_ERROR(_PyBytes_Resize(&a->a_bytecode, a->a_offset * sizeof(_Py_CODEUNIT))); + a->a_bytecode = PyBytesWriter_FinishWithSize(a->a_bytecode_writer, + a->a_offset * sizeof(_Py_CODEUNIT)); + a->a_bytecode_writer = NULL; + if (a->a_bytecode == NULL) { + return ERROR; + } RETURN_IF_ERROR(_PyCompile_ConstCacheMergeOne(const_cache, &a->a_bytecode)); return SUCCESS; }