Skip to content

Commit ca894ea

Browse files
PhilipVincPhilipVinc
committed
refactor PyPreferences
1 parent bfb52f8 commit ca894ea

File tree

4 files changed

+183
-169
lines changed

4 files changed

+183
-169
lines changed

‎PyPreferences.jl/src/PyPreferences.jl‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,13 @@ function instruction_message end
1616
function status end
1717

1818
module Implementations
19-
include("core.jl")
20-
include("api.jl")
19+
20+
module PythonUtils
21+
include("python_utils.jl")
22+
end
23+
24+
include("core.jl")
25+
include("api.jl")
2126
end
2227

2328
let prefs = Implementations.setup_non_failing()

‎PyPreferences.jl/src/api.jl‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@ function PyPreferences.use_conda()
2828
return Implementations.set(conda = true)
2929
end
3030

31+
#=
32+
function use_jll()
33+
end
34+
=#
35+
3136
function PyPreferences.use_inprocess()
3237
return Implementations.set(inprocess = true)
3338
end

‎PyPreferences.jl/src/core.jl‎

Lines changed: 17 additions & 167 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,8 @@
11
using ..PyPreferences: PyPreferences
2+
using .PythonUtils: find_libpython, python_version_of, pythonhome_of
23

34
using Preferences: @set_preferences!, @load_preference, @delete_preferences!
45

5-
using Logging
6-
7-
import Libdl
8-
using Pkg.Artifacts
9-
using VersionParsing
10-
11-
using Conda
12-
136
struct PythonPreferences
147
python::Union{Nothing,String}
158
inprocess::Bool
@@ -21,169 +14,12 @@ function Base.show(io::IO,x::PythonPreferences)
2114
print(io, "PythonPreferences(python=$(x.python), inprocess=$(x.inprocess), conda=$(x.conda))")
2215
end
2316

24-
# Fix the environment for running `python`, and setts IO encoding to UTF-8.
25-
# If cmd is the Conda python, then additionally removes all PYTHON* and
26-
# CONDA* environment variables.
27-
function pythonenv(cmd::Cmd)
28-
@assert cmd.env === nothing # TODO: handle non-nothing case
29-
env = copy(ENV)
30-
if dirname(cmd.exec[1]) == abspath(Conda.PYTHONDIR)
31-
pythonvars = String[]
32-
for var in keys(env)
33-
if startswith(var, "CONDA") || startswith(var, "PYTHON")
34-
push!(pythonvars, var)
35-
end
36-
end
37-
for var in pythonvars
38-
pop!(env, var)
39-
end
40-
end
41-
42-
# set PYTHONIOENCODING when running python executable, so that
43-
# we get UTF-8 encoded text as output (this is not the default on Windows).
44-
env["PYTHONIOENCODING"] = "UTF-8"
45-
setenv(cmd, env)
46-
end
47-
48-
pyvar(python::AbstractString, mod::AbstractString, var::AbstractString) =
49-
chomp(read(pythonenv(`$python -c "import $mod; print($mod.$(var))"`), String))
50-
51-
pyconfigvar(python::AbstractString, var::AbstractString) =
52-
pyvar(python, "distutils.sysconfig", "get_config_var('$(var)')")
53-
pyconfigvar(python, var, default) =
54-
let v = pyconfigvar(python, var)
55-
v == "None" ? default : v
56-
end
57-
58-
function pythonhome_of(pyprogramname::AbstractString)
59-
if Sys.iswindows()
60-
# PYTHONHOME tells python where to look for both pure python
61-
# and binary modules. When it is set, it replaces both
62-
# `prefix` and `exec_prefix` and we thus need to set it to
63-
# both in case they differ. This is also what the
64-
# documentation recommends. However, they are documented
65-
# to always be the same on Windows, where it causes
66-
# problems if we try to include both.
67-
script = """
68-
import sys
69-
if hasattr(sys, "base_exec_prefix"):
70-
sys.stdout.write(sys.base_exec_prefix)
71-
else:
72-
sys.stdout.write(sys.exec_prefix)
73-
"""
74-
else
75-
script = """
76-
import sys
77-
if hasattr(sys, "base_exec_prefix"):
78-
sys.stdout.write(sys.base_prefix)
79-
sys.stdout.write(":")
80-
sys.stdout.write(sys.base_exec_prefix)
81-
else:
82-
sys.stdout.write(sys.prefix)
83-
sys.stdout.write(":")
84-
sys.stdout.write(sys.exec_prefix)
85-
"""
86-
# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME
87-
end
88-
return read(pythonenv(`$pyprogramname -c $script`), String)
89-
end
90-
# To support `venv` standard library (as well as `virtualenv`), we
91-
# need to use `sys.base_prefix` and `sys.base_exec_prefix` here.
92-
# Otherwise, initializing Python in `__init__` below fails with
93-
# unrecoverable error:
94-
#
95-
# Fatal Python error: initfsencoding: unable to load the file system codec
96-
# ModuleNotFoundError: No module named 'encodings'
97-
#
98-
# This is because `venv` does not symlink standard libraries like
99-
# `virtualenv`. For example, `lib/python3.X/encodings` does not
100-
# exist. Rather, `venv` relies on the behavior of Python runtime:
101-
#
102-
# If a file named "pyvenv.cfg" exists one directory above
103-
# sys.executable, sys.prefix and sys.exec_prefix are set to that
104-
# directory and it is also checked for site-packages
105-
# --- https://docs.python.org/3/library/venv.html
106-
#
107-
# Thus, we need point `PYTHONHOME` to `sys.base_prefix` and
108-
# `sys.base_exec_prefix`. If the virtual environment is created by
109-
# `virtualenv`, those `sys.base_*` paths point to the virtual
110-
# environment. Thus, above code supports both use cases.
111-
#
112-
# See also:
113-
# * https://docs.python.org/3/library/venv.html
114-
# * https://docs.python.org/3/library/site.html
115-
# * https://docs.python.org/3/library/sys.html#sys.base_exec_prefix
116-
# * https://github.com/JuliaPy/PyCall.jl/issues/410
117-
118-
python_version_of(python) = vparse(pyvar(python, "platform", "python_version()"))
119-
120-
function find_libpython_py_path()
121-
122-
return joinpath(@__DIR__, "find_libpython.py")
123-
end
124-
125-
function exec_find_libpython(python::AbstractString, options, verbose::Bool)
126-
# Do not inline `@__DIR__` into the backticks to expand correctly.
127-
# See: https://github.com/JuliaLang/julia/issues/26323
128-
script = find_libpython_py_path()
129-
cmd = `$python $script $options`
130-
if verbose
131-
cmd = `$cmd --verbose`
132-
end
133-
return readlines(pythonenv(cmd))
134-
end
135-
136-
# return libpython path, libpython pointer
137-
function find_libpython(
138-
python::AbstractString;
139-
_dlopen = Libdl.dlopen,
140-
verbose::Bool = false,
141-
)
142-
dlopen_flags = Libdl.RTLD_LAZY | Libdl.RTLD_DEEPBIND | Libdl.RTLD_GLOBAL
143-
144-
libpaths = exec_find_libpython(python, `--list-all`, verbose)
145-
for lib in libpaths
146-
try
147-
return (lib, _dlopen(lib, dlopen_flags))
148-
catch e
149-
@warn "Failed to `dlopen` $lib" exception = (e, catch_backtrace())
150-
end
151-
end
152-
@warn """
153-
Python (`find_libpython.py`) failed to find `libpython`.
154-
Falling back to `Libdl`-based discovery.
155-
"""
156-
157-
# Try all candidate libpython names and let Libdl find the path.
158-
# We do this *last* because the libpython in the system
159-
# library path might be the wrong one if multiple python
160-
# versions are installed (we prefer the one in LIBDIR):
161-
libs = exec_find_libpython(python, `--candidate-names`, verbose)
162-
for lib in libs
163-
lib = splitext(lib)[1]
164-
try
165-
libpython = _dlopen(lib, dlopen_flags)
166-
return (Libdl.dlpath(libpython), libpython)
167-
catch e
168-
@debug "Failed to `dlopen` $lib" exception = (e, catch_backtrace())
169-
end
170-
end
171-
172-
return nothing, nothing
173-
end
174-
175-
conda_python_fullpath() =
176-
abspath(Conda.PYTHONDIR, "python" * (Sys.iswindows() ? ".exe" : ""))
177-
178-
#=
179-
function use_jll()
180-
end
181-
=#
18217

18318
set(; python = nothing, inprocess = false, conda = false) =
184-
set(PythonPreferences(python, inprocess, conda))
19+
set(PythonPreferences(get_python_fullpath(python), inprocess, conda))
18520

18621
function set(prefs::PythonPreferences)
22+
@debug "setting new Python Preferences" prefs
18723
if prefs.python === nothing
18824
@delete_preferences!("python")
18925
else
@@ -248,6 +84,18 @@ function PyPreferences.recompile()
24884
return
24985
end
25086

87+
function get_python_fullpath(python)
88+
python_fullpath = nothing
89+
if python !== nothing
90+
python_fullpath = Sys.which(python)
91+
if python_fullpath === nothing
92+
@error "Failed to find a binary named `$(python)` in PATH."
93+
else
94+
@debug "Found path for command $(python)" python_fullpath
95+
end
96+
end
97+
return python_fullpath
98+
end
25199

252100
function setup_non_failing()
253101
python = nothing
@@ -279,6 +127,8 @@ function setup_non_failing()
279127
python_fullpath = Sys.which(python)
280128
if python_fullpath === nothing
281129
@error "Failed to find a binary named `$(python)` in PATH."
130+
else
131+
@debug "Found path for command $(python)" python_fullpath
282132
end
283133
end
284134

‎PyPreferences.jl/src/python_utils.jl‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
using Libdl: Libdl
2+
using VersionParsing: vparse
3+
using Conda: Conda
4+
5+
# Fix the environment for running `python`, and setts IO encoding to UTF-8.
6+
# If cmd is the Conda python, then additionally removes all PYTHON* and
7+
# CONDA* environment variables.
8+
function pythonenv(cmd::Cmd)
9+
@assert cmd.env === nothing # TODO: handle non-nothing case
10+
env = copy(ENV)
11+
if dirname(cmd.exec[1]) == abspath(Conda.PYTHONDIR)
12+
pythonvars = String[]
13+
for var in keys(env)
14+
if startswith(var, "CONDA") || startswith(var, "PYTHON")
15+
push!(pythonvars, var)
16+
end
17+
end
18+
for var in pythonvars
19+
pop!(env, var)
20+
end
21+
end
22+
23+
# set PYTHONIOENCODING when running python executable, so that
24+
# we get UTF-8 encoded text as output (this is not the default on Windows).
25+
env["PYTHONIOENCODING"] = "UTF-8"
26+
setenv(cmd, env)
27+
end
28+
29+
pyvar(python::AbstractString, mod::AbstractString, var::AbstractString) =
30+
chomp(read(pythonenv(`$python -c "import $mod; print($mod.$(var))"`), String))
31+
32+
pyconfigvar(python::AbstractString, var::AbstractString) =
33+
pyvar(python, "distutils.sysconfig", "get_config_var('$(var)')")
34+
pyconfigvar(python, var, default) =
35+
let v = pyconfigvar(python, var)
36+
v == "None" ? default : v
37+
end
38+
39+
function pythonhome_of(pyprogramname::AbstractString)
40+
if Sys.iswindows()
41+
# PYTHONHOME tells python where to look for both pure python
42+
# and binary modules. When it is set, it replaces both
43+
# `prefix` and `exec_prefix` and we thus need to set it to
44+
# both in case they differ. This is also what the
45+
# documentation recommends. However, they are documented
46+
# to always be the same on Windows, where it causes
47+
# problems if we try to include both.
48+
script = """
49+
import sys
50+
if hasattr(sys, "base_exec_prefix"):
51+
sys.stdout.write(sys.base_exec_prefix)
52+
else:
53+
sys.stdout.write(sys.exec_prefix)
54+
"""
55+
else
56+
script = """
57+
import sys
58+
if hasattr(sys, "base_exec_prefix"):
59+
sys.stdout.write(sys.base_prefix)
60+
sys.stdout.write(":")
61+
sys.stdout.write(sys.base_exec_prefix)
62+
else:
63+
sys.stdout.write(sys.prefix)
64+
sys.stdout.write(":")
65+
sys.stdout.write(sys.exec_prefix)
66+
"""
67+
# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME
68+
end
69+
return read(pythonenv(`$pyprogramname -c $script`), String)
70+
end
71+
# To support `venv` standard library (as well as `virtualenv`), we
72+
# need to use `sys.base_prefix` and `sys.base_exec_prefix` here.
73+
# Otherwise, initializing Python in `__init__` below fails with
74+
# unrecoverable error:
75+
#
76+
# Fatal Python error: initfsencoding: unable to load the file system codec
77+
# ModuleNotFoundError: No module named 'encodings'
78+
#
79+
# This is because `venv` does not symlink standard libraries like
80+
# `virtualenv`. For example, `lib/python3.X/encodings` does not
81+
# exist. Rather, `venv` relies on the behavior of Python runtime:
82+
#
83+
# If a file named "pyvenv.cfg" exists one directory above
84+
# sys.executable, sys.prefix and sys.exec_prefix are set to that
85+
# directory and it is also checked for site-packages
86+
# --- https://docs.python.org/3/library/venv.html
87+
#
88+
# Thus, we need point `PYTHONHOME` to `sys.base_prefix` and
89+
# `sys.base_exec_prefix`. If the virtual environment is created by
90+
# `virtualenv`, those `sys.base_*` paths point to the virtual
91+
# environment. Thus, above code supports both use cases.
92+
#
93+
# See also:
94+
# * https://docs.python.org/3/library/venv.html
95+
# * https://docs.python.org/3/library/site.html
96+
# * https://docs.python.org/3/library/sys.html#sys.base_exec_prefix
97+
# * https://github.com/JuliaPy/PyCall.jl/issues/410
98+
99+
python_version_of(python) = vparse(pyvar(python, "platform", "python_version()"))
100+
101+
function find_libpython_py_path()
102+
103+
return joinpath(@__DIR__, "find_libpython.py")
104+
end
105+
106+
function exec_find_libpython(python::AbstractString, options, verbose::Bool)
107+
# Do not inline `@__DIR__` into the backticks to expand correctly.
108+
# See: https://github.com/JuliaLang/julia/issues/26323
109+
script = find_libpython_py_path()
110+
cmd = `$python $script $options`
111+
if verbose
112+
cmd = `$cmd --verbose`
113+
end
114+
return readlines(pythonenv(cmd))
115+
end
116+
117+
# return libpython path, libpython pointer
118+
function find_libpython(
119+
python::AbstractString;
120+
_dlopen = Libdl.dlopen,
121+
verbose::Bool = false,
122+
)
123+
dlopen_flags = Libdl.RTLD_LAZY | Libdl.RTLD_DEEPBIND | Libdl.RTLD_GLOBAL
124+
125+
libpaths = exec_find_libpython(python, `--list-all`, verbose)
126+
for lib in libpaths
127+
try
128+
return (lib, _dlopen(lib, dlopen_flags))
129+
catch e
130+
@warn "Failed to `dlopen` $lib" exception = (e, catch_backtrace())
131+
end
132+
end
133+
@warn """
134+
Python (`find_libpython.py`) failed to find `libpython`.
135+
Falling back to `Libdl`-based discovery.
136+
"""
137+
138+
# Try all candidate libpython names and let Libdl find the path.
139+
# We do this *last* because the libpython in the system
140+
# library path might be the wrong one if multiple python
141+
# versions are installed (we prefer the one in LIBDIR):
142+
libs = exec_find_libpython(python, `--candidate-names`, verbose)
143+
for lib in libs
144+
lib = splitext(lib)[1]
145+
try
146+
libpython = _dlopen(lib, dlopen_flags)
147+
return (Libdl.dlpath(libpython), libpython)
148+
catch e
149+
@debug "Failed to `dlopen` $lib" exception = (e, catch_backtrace())
150+
end
151+
end
152+
153+
return nothing, nothing
154+
end

0 commit comments

Comments
 (0)