GeezerVison is showing github.com — bpo-29585: optimize site.py startup time by methane · Pull Request #136 · python/cpython
bpo-29585: optimize site.py startup time
#136
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement . We’ll occasionally send you account related emails.
Already on GitHub?
Sign in
to your account
Uh oh!
There was an error while loading. Please reload this page.
Show file tree
Hide file tree
Changes from all commits
Commits
File filter
Filter by extension
.ac
(1)
.c
(1)
.in
(1)
.py
(3)
.rst
(1)
No extension
(1)
All 6 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Hide whitespace
Diff view
Hide whitespace
There are no files selected for viewing
Original file line number
Diff line number
Diff line change
Expand Up
@@ -124,7 +124,7 @@ def removeduppaths():
# if they only differ in case); turn relative paths into absolute
# paths.
dir, dircase = makepath(dir)
if not dircase in known_paths:
if dircase not in known_paths:
L.append(dir)
known_paths.add(dircase)
sys.path[:] = L
Expand Down
Expand Up
@@ -234,6 +234,46 @@ def check_enableusersite():
return True
# NOTE: sysconfig and it's dependencies are relatively large but site module
# needs very limited part of them.
# To speedup startup time, we have copy of them.
#
# See https://bugs.python.org/issue29585
# Copy of sysconfig._getuserbase()
def _getuserbase():
env_base = os.environ.get("PYTHONUSERBASE", None)
if env_base:
return env_base
def joinuser(*args):
return os.path.expanduser(os.path.join(*args))
if os.name == "nt":
base = os.environ.get("APPDATA") or "~"
return joinuser(base, "Python")
if sys.platform == "darwin" and sys._framework:
return joinuser("~", "Library", sys._framework,
"%d.%d" % sys.version_info[:2])
return joinuser("~", ".local")
# Same to sysconfig.get_path('purelib', os.name+'_user')
def _get_path(userbase):
version = sys.version_info
if os.name == 'nt':
return f'{userbase}/Python{version[0]}{version[1]}/site-packages'
if sys.platform == 'darwin' and sys._framework:
return f'{userbase}/lib/python/site-packages'
return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages'
def getuserbase():
"""Returns the `user base` directory path.
Expand All
@@ -242,33 +282,23 @@ def getuserbase():
it.
"""
global USER_BASE
if USER_BASE is not None:
return USER_BASE
from sysconfig import get_config_var
USER_BASE = get_config_var('userbase')
if USER_BASE is None:
USER_BASE = _getuserbase()
return USER_BASE
def getusersitepackages():
"""Returns the user-specific site-packages directory path.
If the global variable ``USER_SITE`` is not initialized yet, this
function will also set it.
"""
global USER_SITE
user_base = getuserbase() # this will also set USER_BASE
if USER_SITE is not None:
return USER_SITE
from sysconfig import get_path
userbase = getuserbase() # this will also set USER_BASE
if sys.platform == 'darwin':
from sysconfig import get_config_var
if get_config_var('PYTHONFRAMEWORK'):
USER_SITE = get_path('purelib', 'osx_framework_user')
return USER_SITE
if USER_SITE is None:
USER_SITE = _get_path(userbase)
USER_SITE = get_path('purelib', '%s_user' % os.name)
return USER_SITE
def addusersitepackages(known_paths):
Expand Down
Expand Up
@@ -310,15 +340,11 @@ def getsitepackages(prefixes=None):
else:
sitepackages.append(prefix)
sitepackages.append(os.path.join(prefix, "lib", "site-packages"))
if sys.platform == "darwin":
# for framework builds *only* we add the standard Apple
# locations.
from sysconfig import get_config_var
framework = get_config_var("PYTHONFRAMEWORK")
if framework:
sitepackages.append(
os.path.join("/Library", framework,
'%d.%d' % sys.version_info[:2], "site-packages"))
# for framework builds *only* we add the standard Apple locations.
if sys.platform == "darwin" and sys._framework:
sitepackages.append(
os.path.join("/Library", framework,
'%d.%d' % sys.version_info[:2], "site-packages"))
return sitepackages
def addsitepackages(known_paths, prefixes=None):
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -51,6 +51,7 @@
'scripts': '{base}/Scripts',
'data': '{base}',
},
# NOTE: When modifying "purelib" scheme, update site._get_path() too.
'nt_user': {
'stdlib': '{userbase}/Python{py_version_nodot}',
'platstdlib': '{userbase}/Python{py_version_nodot}',
Expand Down
Expand Up
@@ -177,32 +178,25 @@ def _get_default_scheme():
return os.name
# NOTE: site.py has copy of this function.
# Sync it when modify this function.
def _getuserbase():
env_base = os.environ.get("PYTHONUSERBASE", None)
if env_base:
return env_base
def joinuser(*args):
return os.path.expanduser(os.path.join(*args))
if os.name == "nt":
base = os.environ.get("APPDATA") or "~"
if env_base:
return env_base
else:
return joinuser(base, "Python")
return joinuser(base, "Python")
if sys.platform == "darwin":
framework = get_config_var("PYTHONFRAMEWORK")
if framework:
if env_base:
return env_base
else:
return joinuser("~", "Library", framework, "%d.%d" %
sys.version_info[:2])
if sys.platform == "darwin" and sys._framework:
return joinuser("~", "Library", sys._framework,
"%d.%d" % sys.version_info[:2])
if env_base:
return env_base
else:
return joinuser("~", ".local")
return joinuser("~", ".local")
def _parse_makefile(filename, vars=None):
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -180,6 +180,13 @@ def test_addsitedir(self):
finally:
pth_file.cleanup()
def test_getuserbase(self):
self.assertEqual(site._getuserbase(), sysconfig._getuserbase())
def test_get_path(self):
self.assertEqual(site._get_path(site._getuserbase()),
sysconfig.get_path('purelib', os.name + '_user'))
@unittest.skipUnless(site.ENABLE_USER_SITE, "requires access to PEP 370 "
"user-site (site.ENABLE_USER_SITE)")
def test_s_option(self):
Expand Down
Original file line number
Diff line number
Diff line change
@@ -0,0 +1,2 @@
Avoid importing ``sysconfig`` from ``site`` to improve startup speed. Python
startup is about 5% faster on Linux and 30% faster on macOS.
Original file line number
Diff line number
Diff line change
Expand Up
@@ -1965,6 +1965,7 @@ _PySys_BeginInit(void)
SET_SYS_FROM_STRING("_git",
Py_BuildValue("(szz)", "CPython", _Py_gitidentifier(),
_Py_gitversion()));
SET_SYS_FROM_STRING("_framework", PyUnicode_FromString(PYTHONFRAMEWORK));
SET_SYS_FROM_STRING("api_version",
PyLong_FromLong(PYTHON_API_VERSION));
SET_SYS_FROM_STRING("copyright",
Expand Down
20 changes: 19 additions & 1 deletion
20
configure
Original file line number
Diff line number
Diff line change
Expand Up
@@ -783,6 +783,7 @@ infodir
docdir
oldincludedir
includedir
runstatedir
localstatedir
sharedstatedir
sysconfdir
Expand Down
Expand Up
@@ -896,6 +897,7 @@ datadir='${datarootdir}'
sysconfdir='${prefix}/etc'
sharedstatedir='${prefix}/com'
localstatedir='${prefix}/var'
runstatedir='${localstatedir}/run'
includedir='${prefix}/include'
oldincludedir='/usr/include'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
Expand Down
Expand Up
@@ -1148,6 +1150,15 @@ do
| -silent | --silent | --silen | --sile | --sil)
silent=yes ;;
-runstatedir | --runstatedir | --runstatedi | --runstated \
| --runstate | --runstat | --runsta | --runst | --runs \
| --run | --ru | --r)
ac_prev=runstatedir ;;
-runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
| --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
| --run=* | --ru=* | --r=*)
runstatedir=$ac_optarg ;;
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
ac_prev=sbindir ;;
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
Expand Down
Expand Up
@@ -1285,7 +1296,7 @@ fi
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
datadir sysconfdir sharedstatedir localstatedir includedir \
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
libdir localedir mandir
libdir localedir mandir runstatedir
do
eval ac_val=\$$ac_var
# Remove trailing slashes.
Expand Down
Expand Up
@@ -1438,6 +1449,7 @@ Fine tuning of the installation directories:
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
--runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
--libdir=DIR object code libraries [EPREFIX/lib]
--includedir=DIR C header files [PREFIX/include]
--oldincludedir=DIR C header files for non-gcc [/usr/include]
Expand Down
Expand Up
@@ -3231,6 +3243,12 @@ fi
cat >>confdefs.h <<_ACEOF
#define PYTHONFRAMEWORK "${PYTHONFRAMEWORK}"
_ACEOF
##AC_ARG_WITH(dyld,
## AS_HELP_STRING([--with-dyld],
## [Use (OpenStep|Rhapsody) dynamic linker]))
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -355,6 +355,8 @@ AC_SUBST(FRAMEWORKPYTHONW)
AC_SUBST(FRAMEWORKUNIXTOOLSPREFIX)
AC_SUBST(FRAMEWORKINSTALLAPPSPREFIX)
AC_DEFINE_UNQUOTED(PYTHONFRAMEWORK, "${PYTHONFRAMEWORK}", [framework name])
##AC_ARG_WITH(dyld,
## AS_HELP_STRING([--with-dyld],
## [Use (OpenStep|Rhapsody) dynamic linker]))
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -1247,6 +1247,9 @@
/* Define as the preferred size in bits of long digits */
#undef PYLONG_BITS_IN_DIGIT
/* framework name */
#undef PYTHONFRAMEWORK
/* Define if you want to coerce the C locale to a UTF-8 based locale */
#undef PY_COERCE_C_LOCALE
Expand Down
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.