Skip to content

bpo-36142: Add _PyPreConfig.allocator #12181

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 1 commit into from
Mar 5, 2019
Merged

bpo-36142: Add _PyPreConfig.allocator #12181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Include/cpython/coreconfig.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ typedef struct {
Set by -X utf8 command line option and PYTHONUTF8 environment variable.
If set to -1 (default), inherit Py_UTF8Mode value. */
int utf8_mode;

int dev_mode; /* Development mode. PYTHONDEVMODE, -X dev */
char *allocator; /* Memory allocator: PYTHONMALLOC */
} _PyPreConfig;

#ifdef MS_WINDOWS
Expand Down Expand Up @@ -109,9 +112,6 @@ typedef struct {
int use_hash_seed; /* PYTHONHASHSEED=x */
unsigned long hash_seed;

const char *allocator; /* Memory allocator: PYTHONMALLOC */
int dev_mode; /* PYTHONDEVMODE, -X dev */

/* Enable faulthandler?
Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */
int faulthandler;
Expand Down
52 changes: 29 additions & 23 deletions Lib/test/test_embed.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,30 +561,36 @@ def test_init_from_config(self):
}
self.check_config("init_from_config", config)

INIT_ENV_CONFIG = {
'use_hash_seed': 1,
'hash_seed': 42,
'allocator': 'malloc_debug',
'tracemalloc': 2,
'import_time': 1,
'malloc_stats': 1,
'utf8_mode': 1,
'filesystem_encoding': 'utf-8',
'filesystem_errors': UTF8_MODE_ERRORS,
'inspect': 1,
'optimization_level': 2,
'pycache_prefix': 'env_pycache_prefix',
'write_bytecode': 0,
'verbose': 1,
'buffered_stdio': 0,
'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace',
'user_site_directory': 0,
'faulthandler': 1,
}

def test_init_env(self):
config = {
'use_hash_seed': 1,
'hash_seed': 42,
'allocator': 'malloc_debug',
'tracemalloc': 2,
'import_time': 1,
'malloc_stats': 1,
'utf8_mode': 1,
'filesystem_encoding': 'utf-8',
'filesystem_errors': self.UTF8_MODE_ERRORS,
'inspect': 1,
'optimization_level': 2,
'pycache_prefix': 'env_pycache_prefix',
'write_bytecode': 0,
'verbose': 1,
'buffered_stdio': 0,
'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace',
'user_site_directory': 0,
'faulthandler': 1,
'dev_mode': 1,
}
self.check_config("init_env", config)
self.check_config("init_env", self.INIT_ENV_CONFIG)

def test_init_env_dev_mode(self):
config = dict(self.INIT_ENV_CONFIG,
allocator='debug',
dev_mode=1)
self.check_config("init_env_dev_mode", config)

def test_init_dev_mode(self):
config = {
Expand Down
29 changes: 25 additions & 4 deletions Programs/_testembed.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,7 @@ static int test_init_from_config(void)
config.hash_seed = 123;

putenv("PYTHONMALLOC=malloc");
config.allocator = "malloc_debug";
config.preconfig.allocator = "malloc_debug";

/* dev_mode=1 is tested in test_init_dev_mode() */

Expand Down Expand Up @@ -577,7 +577,6 @@ static void test_init_env_putenvs(void)
putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix");
putenv("PYTHONNOUSERSITE=1");
putenv("PYTHONFAULTHANDLER=1");
putenv("PYTHONDEVMODE=1");
putenv("PYTHONIOENCODING=iso8859-1:replace");
/* FIXME: test PYTHONWARNINGS */
/* FIXME: test PYTHONEXECUTABLE */
Expand All @@ -589,6 +588,15 @@ static void test_init_env_putenvs(void)
}


static void test_init_env_dev_mode_putenvs(void)
{
test_init_env_putenvs();
putenv("PYTHONMALLOC=malloc");
putenv("PYTHONFAULTHANDLER=");
putenv("PYTHONDEVMODE=1");
}


static int test_init_env(void)
{
/* Test initialization from environment variables */
Expand All @@ -601,6 +609,18 @@ static int test_init_env(void)
}


static int test_init_env_dev_mode(void)
{
/* Test initialization from environment variables */
Py_IgnoreEnvironmentFlag = 0;
test_init_env_dev_mode_putenvs();
_testembed_Py_Initialize();
dump_config();
Py_Finalize();
return 0;
}


static int test_init_isolated(void)
{
/* Test _PyCoreConfig.isolated=1 */
Expand All @@ -615,7 +635,7 @@ static int test_init_isolated(void)
/* Use path starting with "./" avoids a search along the PATH */
config.program_name = L"./_testembed";

test_init_env_putenvs();
test_init_env_dev_mode_putenvs();
_PyInitError err = _Py_InitializeFromConfig(&config);
if (_Py_INIT_FAILED(err)) {
_Py_ExitInitError(err);
Expand All @@ -631,7 +651,7 @@ static int test_init_dev_mode(void)
_PyCoreConfig config = _PyCoreConfig_INIT;
putenv("PYTHONFAULTHANDLER=");
putenv("PYTHONMALLOC=");
config.dev_mode = 1;
config.preconfig.dev_mode = 1;
config.program_name = L"./_testembed";
_PyInitError err = _Py_InitializeFromConfig(&config);
if (_Py_INIT_FAILED(err)) {
Expand Down Expand Up @@ -673,6 +693,7 @@ static struct TestCase TestCases[] = {
{ "init_global_config", test_init_global_config },
{ "init_from_config", test_init_from_config },
{ "init_env", test_init_env },
{ "init_env_dev_mode", test_init_env_dev_mode },
{ "init_dev_mode", test_init_dev_mode },
{ "init_isolated", test_init_isolated },
{ NULL, NULL }
Expand Down
20 changes: 2 additions & 18 deletions Python/coreconfig.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -521,8 +521,6 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
COPY_ATTR(use_hash_seed);
COPY_ATTR(hash_seed);
COPY_ATTR(_install_importlib);
COPY_ATTR(allocator);
COPY_ATTR(dev_mode);
COPY_ATTR(faulthandler);
COPY_ATTR(tracemalloc);
COPY_ATTR(import_time);
Expand Down Expand Up @@ -931,10 +929,6 @@ config_read_env_vars(_PyCoreConfig *config)
"PYTHONLEGACYWINDOWSSTDIO");
#endif

if (config->allocator == NULL) {
config->allocator = _PyCoreConfig_GetEnv(config, "PYTHONMALLOC");
}

if (_PyCoreConfig_GetEnv(config, "PYTHONDUMPREFS")) {
config->dump_refs = 1;
}
Expand Down Expand Up @@ -1059,11 +1053,6 @@ config_read_complex_options(_PyCoreConfig *config)
|| config_get_xoption(config, L"importtime")) {
config->import_time = 1;
}
if (config_get_xoption(config, L"dev" ) ||
_PyCoreConfig_GetEnv(config, "PYTHONDEVMODE"))
{
config->dev_mode = 1;
}

_PyInitError err;
if (config->tracemalloc < 0) {
Expand Down Expand Up @@ -1427,13 +1416,10 @@ _PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig)
}

/* default values */
if (config->dev_mode) {
if (config->preconfig.dev_mode) {
if (config->faulthandler < 0) {
config->faulthandler = 1;
}
if (config->allocator == NULL) {
config->allocator = "debug";
}
}
if (config->use_hash_seed < 0) {
config->use_hash_seed = 0;
Expand Down Expand Up @@ -1572,8 +1558,6 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config)
SET_ITEM_INT(install_signal_handlers);
SET_ITEM_INT(use_hash_seed);
SET_ITEM_UINT(hash_seed);
SET_ITEM_STR(allocator);
SET_ITEM_INT(dev_mode);
SET_ITEM_INT(faulthandler);
SET_ITEM_INT(tracemalloc);
SET_ITEM_INT(import_time);
Expand Down Expand Up @@ -1950,7 +1934,7 @@ config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline)
* the lowest precedence entries first so that later entries override them.
*/

if (config->dev_mode) {
if (config->preconfig.dev_mode) {
err = _Py_wstrlist_append(&config->nwarnoption,
&config->warnoptions,
L"default");
Expand Down
65 changes: 65 additions & 0 deletions Python/preconfig.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,15 @@ precmdline_clear(_PyPreCmdline *cmdline)
void
_PyPreConfig_Clear(_PyPreConfig *config)
{
#define CLEAR(ATTR) \
do { \
PyMem_RawFree(ATTR); \
ATTR = NULL; \
} while (0)

CLEAR(config->allocator);

#undef CLEAR
}


Expand All @@ -134,6 +143,15 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2)
_PyPreConfig_Clear(config);

#define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
#define COPY_STR_ATTR(ATTR) \
do { \
if (config2->ATTR != NULL) { \
config->ATTR = _PyMem_RawStrdup(config2->ATTR); \
if (config->ATTR == NULL) { \
return -1; \
} \
} \
} while (0)

COPY_ATTR(isolated);
COPY_ATTR(use_environment);
Expand All @@ -143,8 +161,11 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2)
COPY_ATTR(legacy_windows_fs_encoding);
#endif
COPY_ATTR(utf8_mode);
COPY_ATTR(dev_mode);
COPY_STR_ATTR(allocator);

#undef COPY_ATTR
#undef COPY_STR_ATTR
return 0;
}

Expand Down Expand Up @@ -345,6 +366,7 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline)
{
_PyPreConfig_GetGlobalConfig(config);

/* isolated and use_environment */
if (config->isolated > 0) {
config->use_environment = 0;
}
Expand All @@ -354,6 +376,7 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline)
config->use_environment = 0;
}

/* legacy_windows_fs_encoding, utf8_mode, coerce_c_locale */
if (config->use_environment) {
#ifdef MS_WINDOWS
_Py_get_env_flag(config, &config->legacy_windows_fs_encoding,
Expand Down Expand Up @@ -414,11 +437,43 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline)
if (config->utf8_mode < 0) {
config->utf8_mode = 0;
}
if (config->coerce_c_locale < 0) {
config->coerce_c_locale = 0;
}

/* dev_mode */
if ((cmdline && _Py_get_xoption(cmdline->nxoption, cmdline->xoptions, L"dev"))
|| _PyPreConfig_GetEnv(config, "PYTHONDEVMODE"))
{
config->dev_mode = 1;
}
if (config->dev_mode < 0) {
config->dev_mode = 0;
}

/* allocator */
if (config->dev_mode && config->allocator == NULL) {
config->allocator = _PyMem_RawStrdup("debug");
if (config->allocator == NULL) {
return _Py_INIT_NO_MEMORY();
}
}

if (config->allocator == NULL) {
const char *allocator = _PyPreConfig_GetEnv(config, "PYTHONMALLOC");
if (allocator) {
config->allocator = _PyMem_RawStrdup(allocator);
if (config->allocator == NULL) {
return _Py_INIT_NO_MEMORY();
}
}
}

assert(config->coerce_c_locale >= 0);
assert(config->utf8_mode >= 0);
assert(config->isolated >= 0);
assert(config->use_environment >= 0);
assert(config->dev_mode >= 0);

return _Py_INIT_OK();
}
Expand Down Expand Up @@ -448,6 +503,12 @@ _PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict)
} while (0)
#define SET_ITEM_INT(ATTR) \
SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR))
#define FROM_STRING(STR) \
((STR != NULL) ? \
PyUnicode_FromString(STR) \
: (Py_INCREF(Py_None), Py_None))
#define SET_ITEM_STR(ATTR) \
SET_ITEM(#ATTR, FROM_STRING(config->ATTR))

SET_ITEM_INT(isolated);
SET_ITEM_INT(use_environment);
Expand All @@ -457,13 +518,17 @@ _PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict)
#ifdef MS_WINDOWS
SET_ITEM_INT(legacy_windows_fs_encoding);
#endif
SET_ITEM_INT(dev_mode);
SET_ITEM_STR(allocator);
return 0;

fail:
return -1;

#undef FROM_STRING
#undef SET_ITEM
#undef SET_ITEM_INT
#undef SET_ITEM_STR
}


Expand Down
8 changes: 4 additions & 4 deletions Python/pylifecycle.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,9 +482,9 @@ _Py_Initialize_ReconfigureCore(PyInterpreterState **interp_p,

/* bpo-34008: For backward compatibility reasons, calling Py_Main() after
Py_Initialize() ignores the new configuration. */
if (core_config->allocator != NULL) {
if (core_config->preconfig.allocator != NULL) {
const char *allocator = _PyMem_GetAllocatorsName();
if (allocator == NULL || strcmp(core_config->allocator, allocator) != 0) {
if (allocator == NULL || strcmp(core_config->preconfig.allocator, allocator) != 0) {
return _Py_INIT_USER_ERR("cannot modify memory allocator "
"after first Py_Initialize()");
}
Expand Down Expand Up @@ -521,8 +521,8 @@ pycore_init_runtime(const _PyCoreConfig *core_config)
return err;
}

if (core_config->allocator != NULL) {
if (_PyMem_SetupAllocators(core_config->allocator) < 0) {
if (core_config->preconfig.allocator != NULL) {
if (_PyMem_SetupAllocators(core_config->preconfig.allocator) < 0) {
return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator");
}
}
Expand Down
2 changes: 1 addition & 1 deletion Python/sysmodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2180,7 +2180,7 @@ make_flags(void)
SetFlag(config->quiet);
SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0);
SetFlag(config->preconfig.isolated);
PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode));
PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->preconfig.dev_mode));
SetFlag(config->preconfig.utf8_mode);
#undef SetFlag

Expand Down