python · ericsnowcurrently · Nov 21, 2017 · Nov 20, 2017 · Nov 21, 2017 · Nov 21, 2017
diff --git a/Include/internal/mem.h b/Include/internal/mem.h
@@ -44,6 +44,8 @@ struct _pymem_runtime_state {
     size_t serialno;     /* incremented on each debug {m,re}alloc */
 };
 
+PyAPI_FUNC(void *) _PyMem_RawMalloc(void *ctx, size_t size);
+PyAPI_FUNC(void) _PyMem_RawFree(void *ctx, void *ptr);
 PyAPI_FUNC(void) _PyMem_Initialize(struct _pymem_runtime_state *);
 
 

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
@@ -593,6 +593,16 @@ def test_forced_io_encoding(self):
         self.maxDiff = None
         self.assertEqual(out.strip(), expected_output)
 
+    def test_pre_initialization_api(self):
+        """
+        Checks the few parts of the C-API that work before the runtine
+        is initialized (via Py_Initialize()).
+        """
+        env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path))
+        out, err = self.run_embedded_interpreter("pre_initialization_api", env=env)
+        self.assertEqual(out, '')
+        self.assertEqual(err, '')
+
 
 class SkipitemTest(unittest.TestCase):
 

diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-21-12-44-51.bpo-32096.tXX68e.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-21-12-44-51.bpo-32096.tXX68e.rst
@@ -0,0 +1,6 @@
+Since the recent global runtime state consolidation, calls to
+PyMem_RawMalloc() and PyMem_RawFree() made before runtime initialization
+have been crashing.  This is because they rely on the raw memory allocator
+having been initialized already.  Before the runtime state change the
+default raw allocator was initialized statically.  This has now been fixed
+by falling back to the defaults in PyMem_RawMalloc() and PyMem_RawFree().
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
@@ -63,7 +63,7 @@ static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
 #endif
 
 
-static void *
+void *
 _PyMem_RawMalloc(void *ctx, size_t size)
 {
     /* PyMem_RawMalloc(0) means malloc(1). Some systems would return NULL
@@ -97,7 +97,7 @@ _PyMem_RawRealloc(void *ctx, void *ptr, size_t size)
     return realloc(ptr, size);
 }
 
-static void
+void
 _PyMem_RawFree(void *ctx, void *ptr)
 {
     free(ptr);

diff --git a/Programs/_testembed.c b/Programs/_testembed.c
@@ -72,6 +72,7 @@ static int test_repeated_init_and_subinterpreters(void)
     return 0;
 }
 
+
 /*****************************************************
  * Test forcing a particular IO encoding
  *****************************************************/
@@ -125,6 +126,27 @@ static int test_forced_io_encoding(void)
     return 0;
 }
 
+/*********************************************************
+ * Test parts of the C-API that work before initialization
+ *********************************************************/
+
+static int test_pre_initialization_api(void)
+{
+    wchar_t *program = Py_DecodeLocale("spam", NULL);
+    if (program == NULL) {
+        fprintf(stderr, "Fatal error: cannot decode program name\n");
+        return 1;
+    }
+    Py_SetProgramName(program);
+
+    Py_Initialize();
+    Py_Finalize();
+
+    PyMem_RawFree(program);
+    return 0;
+}
+
+
 /* *********************************************************
  * List of test cases and the function that implements it.
  *
@@ -146,6 +168,7 @@ struct TestCase
 static struct TestCase TestCases[] = {
     { "forced_io_encoding", test_forced_io_encoding },
     { "repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters },
+    { "pre_initialization_api", test_pre_initialization_api },
     { NULL, NULL }
 };
 

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
@@ -5,6 +5,7 @@
 #include "Python-ast.h"
 #undef Yield /* undefine macro conflicting with winbase.h */
 #include "internal/pystate.h"
+#include "internal/mem.h"
 #include "grammar.h"
 #include "node.h"
 #include "token.h"
@@ -76,7 +77,22 @@ extern void _Py_ReadyTypes(void);
 extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *);
 extern void _PyGILState_Fini(void);
 
-_PyRuntimeState _PyRuntime = _PyRuntimeState_INIT;
+//_PyRuntimeState _PyRuntime = _PyRuntimeState_INIT;
+_PyRuntimeState _PyRuntime = {
+    .initialized = 0,
+    .core_initialized = 0,
+    // Py_DecodeLocale() relies on PyMem_RawMalloc() and PyMem_RawFree()
+    // and may be needed before runtime initialization.  To allow for
+    // this we pre-initialize the raw allocator statically here.
+    .mem = {
+        .allocators = {
+            .raw = {
+                .malloc = _PyMem_RawMalloc,
+                .free = _PyMem_RawFree
+            }
+        }
+    }
+};
 
 _PyInitError
 _PyRuntime_Initialize(void)