python · tonybaloney · Apr 8, 2019 · Apr 8, 2019 · Apr 8, 2019 · Apr 8, 2019
diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst
@@ -874,6 +874,13 @@ All of the following opcodes use their arguments.
    Works as :opcode:`BUILD_TUPLE`, but creates a list.
 
 
+.. opcode:: BUILD_LIST_PREALLOC (var_num)
+
+   Creates a list with a pre-allocated size and pushes the resulting list onto the stack.
+   The size is the length-hint of the local ``co_varnames[var_num]``, if the length cannot
+   be established, it will default a length of 0.
+
+
 .. opcode:: BUILD_SET (count)
 
    Works as :opcode:`BUILD_TUPLE`, but creates a set.

diff --git a/Include/listobject.h b/Include/listobject.h
@@ -62,7 +62,7 @@ PyAPI_FUNC(int) PyList_Reverse(PyObject *);
 PyAPI_FUNC(PyObject *) PyList_AsTuple(PyObject *);
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *);
-
+PyAPI_FUNC(PyObject *) _PyList_NewPrealloc(Py_ssize_t size);
 PyAPI_FUNC(int) PyList_ClearFreeList(void);
 PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out);
 #endif

diff --git a/Include/opcode.h b/Include/opcode.h
diff --git a/Lib/opcode.py b/Lib/opcode.py
@@ -207,6 +207,7 @@ def jabs_op(name, op):
 def_op('BUILD_CONST_KEY_MAP', 156)
 def_op('BUILD_STRING', 157)
 def_op('BUILD_TUPLE_UNPACK_WITH_CALL', 158)
+def_op('BUILD_LIST_PREALLOC', 159)
 
 name_op('LOAD_METHOD', 160)
 def_op('CALL_METHOD', 161)

diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
@@ -384,7 +384,7 @@ def foo(x):
 
 dis_nested_2 = """%s
 Disassembly of <code object <listcomp> at 0x..., file "%s", line %d>:
-%3d           0 BUILD_LIST               0
+%3d           0 BUILD_LIST_PREALLOC      0
               2 LOAD_FAST                0 (.0)
         >>    4 FOR_ITER                12 (to 18)
               6 STORE_FAST               1 (z)

diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py
@@ -123,6 +123,13 @@
     >>> test_func()
     [2, 2, 2, 2, 2]
 
+Verify that an overflow error is raised for listcomps with very-large iterators
+
+    >>> [y for y in range(2**256)]          # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+       ...
+    OverflowError: ...
+
 """
 
 

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-04-08-03-42-24.bpo-36551.5Ocm2j.rst b/Misc/NEWS.d/next/Core and Builtins/2019-04-08-03-42-24.bpo-36551.5Ocm2j.rst
@@ -0,0 +1,2 @@
+Change the compiled bytecode for list comprehensions where there is no if statement. Allocates the size of the list to the length of the iterator in order to avoid having to continuously resize the list. 
+Also fixes a bug for a list comprehension containing an iterator with a potential size > PY_SSIZE_MAX, e.g. range(2**256).
diff --git a/Objects/listobject.c b/Objects/listobject.c
@@ -215,6 +215,12 @@ list_new_prealloc(Py_ssize_t size)
     return (PyObject *) op;
 }
 
+PyObject *
+_PyList_NewPrealloc(Py_ssize_t size)
+{
+    return list_new_prealloc(size);
+}
+
 Py_ssize_t
 PyList_Size(PyObject *op)
 {

diff --git a/Python/ceval.c b/Python/ceval.c
@@ -1168,6 +1168,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
         }
 
         case TARGET(LOAD_FAST): {
+            PREDICTED(LOAD_FAST);
             PyObject *value = GETLOCAL(oparg);
             if (value == NULL) {
                 format_exc_check_arg(PyExc_UnboundLocalError,
@@ -2484,6 +2485,20 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
+        case TARGET(BUILD_LIST_PREALLOC): {
+            PyObject *list, *target = GETLOCAL(oparg);
+            Py_INCREF(target);
+            Py_ssize_t size = PyObject_LengthHint(target, 0);
+            Py_DECREF(target);
+            if (size < 0)
+                goto error;
+
+            list = _PyList_NewPrealloc(size);
+            PUSH(list);
+            PREDICT(LOAD_FAST);
+            DISPATCH();
+        }
+
         case TARGET(BUILD_LIST): {
             PyObject *list =  PyList_New(oparg);
             if (list == NULL)

diff --git a/Python/compile.c b/Python/compile.c
@@ -1006,6 +1006,8 @@ stack_effect(int opcode, int oparg, int jump)
         case BUILD_SET:
         case BUILD_STRING:
             return 1-oparg;
+        case BUILD_LIST_PREALLOC:
+            return 1;
         case BUILD_LIST_UNPACK:
         case BUILD_TUPLE_UNPACK:
         case BUILD_TUPLE_UNPACK_WITH_CALL:
@@ -4368,7 +4370,10 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,
         int op;
         switch (type) {
         case COMP_LISTCOMP:
-            op = BUILD_LIST;
+            if (asdl_seq_LEN(outermost->ifs) > 0)
+                op = BUILD_LIST;
+            else
+                op = BUILD_LIST_PREALLOC;
             break;
         case COMP_SETCOMP:
             op = BUILD_SET;

diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Change the compiled bytecode for list comprehensions where there is no if statement. Allocates the size of the list to the length of the iterator in order to avoid having to continuously resize the list.
		Also fixes a bug for a list comprehension containing an iterator with a potential size > PY_SSIZE_MAX, e.g. range(2**256).