Skip to content

bpo-32030: Fix usage of memory allocators #4953

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 2 commits into from
Dec 20, 2017
Merged

bpo-32030: Fix usage of memory allocators #4953

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f5bc239
bpo-32030: Fix usage of memory allocators
vstinner Dec 20, 2017
3330941
Add NEWS entry; add debug check
vstinner Dec 20, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/C API/2017-12-20-23-22-32.bpo-32030.d1dcwh.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Py_Initialize() doesn't reset the memory allocators to default if the
``PYTHONMALLOC`` environment variable is not set.
26 changes: 19 additions & 7 deletions Modules/main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -649,11 +649,12 @@ pymain_free_raw(_PyMain *pymain)
Py_Initialize()-Py_Finalize() can be called multiple times. */
_PyPathConfig_Clear(&_Py_path_config);

pymain_clear_config(pymain);

/* Force the allocator used by pymain_read_conf() */
PyMemAllocatorEx old_alloc;
_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

_PyCoreConfig_Clear(&pymain->config);
pymain_clear_pymain(pymain);

clear_wstrlist(orig_argc, orig_argv);
Expand Down Expand Up @@ -1963,11 +1964,6 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
{
int res = -1;

/* Force default allocator, since pymain_free() must use the same allocator
than this function. */
PyMemAllocatorEx old_alloc;
_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

char *oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL));
if (oldloc == NULL) {
pymain->err = _Py_INIT_NO_MEMORY();
Expand Down Expand Up @@ -2055,7 +2051,6 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
PyMem_RawFree(oldloc);
}

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
return res;
}

Expand Down Expand Up @@ -2578,6 +2573,15 @@ pymain_cmdline_impl(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
static int
pymain_cmdline(_PyMain *pymain)
{
/* Force default allocator, since pymain_free() and pymain_clear_config()
must use the same allocator than this function. */
PyMemAllocatorEx old_alloc;
_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
#ifdef Py_DEBUG
PyMemAllocatorEx default_alloc;
PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &default_alloc);
#endif

_Py_CommandLineDetails cmdline;
memset(&cmdline, 0, sizeof(cmdline));

Expand All @@ -2588,6 +2592,14 @@ pymain_cmdline(_PyMain *pymain)
pymain_set_global_config(pymain, &cmdline);

pymain_clear_cmdline(pymain, &cmdline);

#ifdef Py_DEBUG
/* Make sure that PYMEM_DOMAIN_RAW has not been modified */
PyMemAllocatorEx cur_alloc;
PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &cur_alloc);
assert(memcmp(&cur_alloc, &default_alloc, sizeof(cur_alloc)) == 0);
#endif
PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
return res;
}

Expand Down
34 changes: 29 additions & 5 deletions Python/pylifecycle.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,15 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
/* This is too late to have any effect */
return -1;
}

int res = 0;

/* Py_SetStandardStreamEncoding() can be called before Py_Initialize(),
but Py_Initialize() can change the allocator. Use a known allocator
to be able to release the memory later. */
PyMemAllocatorEx old_alloc;
_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

/* Can't call PyErr_NoMemory() on errors, as Python hasn't been
* initialised yet.
*
Expand All @@ -182,7 +191,8 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
if (encoding) {
_Py_StandardStreamEncoding = _PyMem_RawStrdup(encoding);
if (!_Py_StandardStreamEncoding) {
return -2;
res = -2;
goto done;
}
}
if (errors) {
Expand All @@ -191,7 +201,8 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
if (_Py_StandardStreamEncoding) {
PyMem_RawFree(_Py_StandardStreamEncoding);
}
return -3;
res = -3;
goto done;
}
}
#ifdef MS_WINDOWS
Expand All @@ -200,7 +211,11 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
Py_LegacyWindowsStdioFlag = 1;
}
#endif
return 0;

done:
PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

return res;
}


Expand Down Expand Up @@ -597,8 +612,10 @@ _Py_InitializeCore(const _PyCoreConfig *core_config)
return err;
}

if (_PyMem_SetupAllocators(core_config->allocator) < 0) {
return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator");
if (core_config->allocator != NULL) {
if (_PyMem_SetupAllocators(core_config->allocator) < 0) {
return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator");
}
}

if (_PyRuntime.initialized) {
Expand Down Expand Up @@ -1818,7 +1835,11 @@ init_sys_streams(PyInterpreterState *interp)
error:
res = _Py_INIT_ERR("can't initialize sys standard streams");

/* Use the same allocator than Py_SetStandardStreamEncoding() */
PyMemAllocatorEx old_alloc;
done:
_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

/* We won't need them anymore. */
if (_Py_StandardStreamEncoding) {
PyMem_RawFree(_Py_StandardStreamEncoding);
Expand All @@ -1828,6 +1849,9 @@ init_sys_streams(PyInterpreterState *interp)
PyMem_RawFree(_Py_StandardStreamErrors);
_Py_StandardStreamErrors = NULL;
}

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

PyMem_Free(pythonioencoding);
Py_XDECREF(bimod);
Py_XDECREF(iomod);
Expand Down