Skip to content

Commit ba5633c

Browse files
benoitcbenoitc
authored
Merge pull request #49 from benoitc/feature/per-interpreter-preload-code
Add per-interpreter preload code with inherited globals
2 parents cf75085 + 770f894 commit ba5633c

5 files changed

Lines changed: 537 additions & 34 deletions

File tree

‎c_src/py_nif.c‎

Lines changed: 46 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -3147,34 +3147,40 @@ static void owngil_execute_create_local_env(py_context_t *ctx) {
31473147
res->interp_id = PyInterpreterState_GetID(interp);
31483148
}
31493149

3150-
/* Create globals dict with builtins and erlang module */
3151-
res->globals = PyDict_New();
3150+
/* Copy globals from context to inherit preloaded code */
3151+
res->globals = PyDict_Copy(ctx->globals);
31523152
if (res->globals == NULL) {
31533153
ctx->response_term = enif_make_tuple2(ctx->shared_env,
31543154
enif_make_atom(ctx->shared_env, "error"),
3155-
enif_make_atom(ctx->shared_env, "globals_failed"));
3155+
enif_make_atom(ctx->shared_env, "globals_copy_failed"));
31563156
ctx->response_ok = false;
31573157
return;
31583158
}
31593159

3160-
/* Add __builtins__ */
3161-
PyObject *builtins = PyEval_GetBuiltins();
3162-
if (builtins != NULL) {
3163-
PyDict_SetItemString(res->globals, "__builtins__", builtins);
3160+
/* Ensure __builtins__ is present */
3161+
if (PyDict_GetItemString(res->globals, "__builtins__") == NULL) {
3162+
PyObject *builtins = PyEval_GetBuiltins();
3163+
if (builtins != NULL) {
3164+
PyDict_SetItemString(res->globals, "__builtins__", builtins);
3165+
}
31643166
}
31653167

3166-
/* Add __name__ = '__main__' */
3167-
PyObject *main_name = PyUnicode_FromString("__main__");
3168-
if (main_name != NULL) {
3169-
PyDict_SetItemString(res->globals, "__name__", main_name);
3170-
Py_DECREF(main_name);
3168+
/* Ensure __name__ = '__main__' is set */
3169+
if (PyDict_GetItemString(res->globals, "__name__") == NULL) {
3170+
PyObject *main_name = PyUnicode_FromString("__main__");
3171+
if (main_name != NULL) {
3172+
PyDict_SetItemString(res->globals, "__name__", main_name);
3173+
Py_DECREF(main_name);
3174+
}
31713175
}
31723176

3173-
/* Add erlang module */
3174-
PyObject *erlang = PyImport_ImportModule("erlang");
3175-
if (erlang != NULL) {
3176-
PyDict_SetItemString(res->globals, "erlang", erlang);
3177-
Py_DECREF(erlang);
3177+
/* Ensure erlang module is available */
3178+
if (PyDict_GetItemString(res->globals, "erlang") == NULL) {
3179+
PyObject *erlang = PyImport_ImportModule("erlang");
3180+
if (erlang != NULL) {
3181+
PyDict_SetItemString(res->globals, "erlang", erlang);
3182+
Py_DECREF(erlang);
3183+
}
31783184
}
31793185

31803186
/* Use the same dict for locals (module-level execution) */
@@ -4933,32 +4939,38 @@ static ERL_NIF_TERM nif_create_local_env(ErlNifEnv *env, int argc, const ERL_NIF
49334939
}
49344940
#endif
49354941

4936-
/* Create globals dict with builtins and erlang module */
4937-
res->globals = PyDict_New();
4942+
/* Copy globals from context to inherit preloaded code */
4943+
res->globals = PyDict_Copy(ctx->globals);
49384944
if (res->globals == NULL) {
49394945
py_context_release(&guard);
49404946
enif_release_resource(res);
4941-
return make_error(env, "globals_failed");
4947+
return make_error(env, "globals_copy_failed");
49424948
}
49434949

4944-
/* Add __builtins__ */
4945-
PyObject *builtins = PyEval_GetBuiltins();
4946-
if (builtins != NULL) {
4947-
PyDict_SetItemString(res->globals, "__builtins__", builtins);
4950+
/* Ensure __builtins__ is present (may not be in subinterpreter mode) */
4951+
if (PyDict_GetItemString(res->globals, "__builtins__") == NULL) {
4952+
PyObject *builtins = PyEval_GetBuiltins();
4953+
if (builtins != NULL) {
4954+
PyDict_SetItemString(res->globals, "__builtins__", builtins);
4955+
}
49484956
}
49494957

4950-
/* Add __name__ = '__main__' so defined functions are accessible via __main__ */
4951-
PyObject *main_name = PyUnicode_FromString("__main__");
4952-
if (main_name != NULL) {
4953-
PyDict_SetItemString(res->globals, "__name__", main_name);
4954-
Py_DECREF(main_name);
4958+
/* Ensure __name__ = '__main__' is set */
4959+
if (PyDict_GetItemString(res->globals, "__name__") == NULL) {
4960+
PyObject *main_name = PyUnicode_FromString("__main__");
4961+
if (main_name != NULL) {
4962+
PyDict_SetItemString(res->globals, "__name__", main_name);
4963+
Py_DECREF(main_name);
4964+
}
49554965
}
49564966

4957-
/* Add erlang module */
4958-
PyObject *erlang = PyImport_ImportModule("erlang");
4959-
if (erlang != NULL) {
4960-
PyDict_SetItemString(res->globals, "erlang", erlang);
4961-
Py_DECREF(erlang);
4967+
/* Ensure erlang module is available */
4968+
if (PyDict_GetItemString(res->globals, "erlang") == NULL) {
4969+
PyObject *erlang = PyImport_ImportModule("erlang");
4970+
if (erlang != NULL) {
4971+
PyDict_SetItemString(res->globals, "erlang", erlang);
4972+
Py_DECREF(erlang);
4973+
}
49624974
}
49634975

49644976
/* Use the same dict for locals (module-level execution) */

‎docs/preload.md‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
# Preload Code
2+
3+
This guide covers preloading Python code that executes during interpreter initialization.
4+
5+
## Overview
6+
7+
The `py_preload` module allows you to define Python code that runs once per interpreter at creation time. The resulting globals (imports, functions, variables) become available to all process-local environments.
8+
9+
## Use Cases
10+
11+
- Share common imports across all contexts
12+
- Define utility functions used throughout your application
13+
- Set up configuration or constants
14+
- Avoid repeated initialization overhead
15+
16+
## API
17+
18+
| Function | Description |
19+
|----------|-------------|
20+
| `set_code/1` | Set preload code (binary or iolist) |
21+
| `get_code/0` | Get current preload code or `undefined` |
22+
| `clear_code/0` | Remove preload code |
23+
| `has_preload/0` | Check if preload is configured |
24+
25+
## Basic Usage
26+
27+
```erlang
28+
%% Set preload code at application startup
29+
py_preload:set_code(<<"
30+
import json
31+
import os
32+
33+
def shared_helper(x):
34+
return x * 2
35+
36+
CONFIG = {'debug': True, 'version': '1.0'}
37+
">>).
38+
39+
%% Create a context - preload is automatically applied
40+
{ok, Ctx} = py_context:new(#{mode => worker}).
41+
42+
%% Use preloaded imports
43+
{ok, <<"{\"a\": 1}">>} = py:eval(Ctx, <<"json.dumps({'a': 1})">>).
44+
45+
%% Use preloaded functions
46+
{ok, 10} = py:eval(Ctx, <<"shared_helper(5)">>).
47+
48+
%% Use preloaded variables
49+
{ok, true} = py:eval(Ctx, <<"CONFIG['debug']">>).
50+
```
51+
52+
## Execution Flow
53+
54+
```
55+
Interpreter/Context Creation
56+
│
57+
▼
58+
apply_registered_paths() ← sys.path updates
59+
│
60+
▼
61+
apply_registered_imports() ← module imports
62+
│
63+
▼
64+
apply_preload() ← preload code execution
65+
│
66+
▼
67+
(interpreter ready)
68+
│
69+
▼
70+
create_local_env() ← copies from interpreter globals
71+
```
72+
73+
## Process Isolation
74+
75+
Each process-local environment gets an isolated copy of preloaded globals:
76+
77+
```erlang
78+
py_preload:set_code(<<"COUNTER = 0">>).
79+
80+
{ok, Ctx1} = py_context:new(#{mode => worker}).
81+
{ok, Ctx2} = py_context:new(#{mode => worker}).
82+
83+
%% Modify in Ctx1
84+
ok = py:exec(Ctx1, <<"COUNTER = 100">>).
85+
{ok, 100} = py:eval(Ctx1, <<"COUNTER">>).
86+
87+
%% Ctx2 still has original value
88+
{ok, 0} = py:eval(Ctx2, <<"COUNTER">>).
89+
```
90+
91+
## Clearing Preload
92+
93+
Clearing preload only affects new contexts:
94+
95+
```erlang
96+
py_preload:set_code(<<"PRELOADED = 42">>).
97+
98+
{ok, Ctx1} = py_context:new(#{mode => worker}).
99+
{ok, 42} = py:eval(Ctx1, <<"PRELOADED">>).
100+
101+
%% Clear preload
102+
py_preload:clear_code().
103+
104+
%% Existing context still has it
105+
{ok, 42} = py:eval(Ctx1, <<"PRELOADED">>).
106+
107+
%% New context does not
108+
{ok, Ctx2} = py_context:new(#{mode => worker}).
109+
{error, _} = py:eval(Ctx2, <<"PRELOADED">>).
110+
```
111+
112+
## Best Practices
113+
114+
1. **Set preload early** - Configure before creating any contexts
115+
2. **Keep it focused** - Only include truly shared code
116+
3. **Avoid side effects** - Preload runs once per interpreter
117+
4. **Use for imports** - Common imports benefit most from preloading
118+
119+
## Limitations
120+
121+
- Changes to preload code don't affect existing contexts
122+
- Same preload applies to all context modes (worker, subinterp, owngil)
123+
- Preload errors during context creation will fail the context

‎src/py_context.erl‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,8 @@ init(Parent, Id, Mode) ->
421421
%% Apply all registered imports and paths to this interpreter
422422
apply_registered_imports(Ref),
423423
apply_registered_paths(Ref),
424+
%% Apply preload code (populates globals for process-local envs)
425+
apply_preload(Ref),
424426
%% For subinterpreters, create a dedicated event worker
425427
EventState = setup_event_worker(Ref, InterpId),
426428
%% For thread-model subinterpreters, spawn a dedicated callback handler
@@ -518,6 +520,13 @@ apply_registered_paths(Ref) ->
518520
Paths -> py_nif:interp_apply_paths(Ref, Paths)
519521
end.
520522

523+
%% @private Apply preload code to the interpreter's globals.
524+
%%
525+
%% Called when a new interpreter is created. The preload code populates
526+
%% the context's globals dict, which process-local environments inherit.
527+
apply_preload(Ref) ->
528+
py_preload:apply_preload(Ref).
529+
521530
%% @private
522531
create_context(worker) ->
523532
py_nif:context_create(worker);

‎src/py_preload.erl‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
%% Copyright 2026 Benoit Chesneau
2+
%%
3+
%% Licensed under the Apache License, Version 2.0 (the "License");
4+
%% you may not use this file except in compliance with the License.
5+
%% You may obtain a copy of the License at
6+
%%
7+
%% http://www.apache.org/licenses/LICENSE-2.0
8+
%%
9+
%% Unless required by applicable law or agreed to in writing, software
10+
%% distributed under the License is distributed on an "AS IS" BASIS,
11+
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
%% See the License for the specific language governing permissions and
13+
%% limitations under the License.
14+
15+
%%% @doc Python preload code registry.
16+
%%%
17+
%%% Allows users to preload Python code that executes during interpreter
18+
%%% initialization. The resulting globals become the base namespace that
19+
%%% process-local environments inherit from.
20+
%%%
21+
%%% == Usage ==
22+
%%%
23+
%%% ```
24+
%%% %% At application startup
25+
%%% py_preload:set_code(<<"
26+
%%% import json
27+
%%% import os
28+
%%%
29+
%%% def shared_helper(x):
30+
%%% return x * 2
31+
%%%
32+
%%% CONFIG = {'debug': True}
33+
%%% ">>).
34+
%%%
35+
%%% %% Later, any context will have these preloaded
36+
%%% {ok, Ctx} = py_context:new(#{mode => worker}),
37+
%%% {ok, 10} = py:eval(Ctx, <<"shared_helper(5)">>).
38+
%%% '''
39+
%%%
40+
%%% == Storage ==
41+
%%%
42+
%%% Uses `persistent_term' for the preload code. Changes only affect
43+
%%% newly created contexts; existing contexts are not modified.
44+
%%%
45+
%%% @end
46+
-module(py_preload).
47+
48+
-export([
49+
set_code/1,
50+
get_code/0,
51+
clear_code/0,
52+
has_preload/0,
53+
apply_preload/1
54+
]).
55+
56+
-define(PRELOAD_KEY, {py_preload, code}).
57+
58+
%% @doc Set preload code to be executed once per interpreter at init.
59+
%%
60+
%% The code is executed in the interpreter's `__main__' namespace.
61+
%% All defined functions, variables, and imports become available
62+
%% in process-local environments.
63+
%%
64+
%% @param Code Python code as binary or iolist
65+
-spec set_code(binary() | iolist()) -> ok.
66+
set_code(Code) when is_binary(Code); is_list(Code) ->
67+
persistent_term:put(?PRELOAD_KEY, iolist_to_binary(Code)).
68+
69+
%% @doc Get the current preload code.
70+
%%
71+
%% @returns The preload code binary, or `undefined' if not set
72+
-spec get_code() -> binary() | undefined.
73+
get_code() ->
74+
try
75+
persistent_term:get(?PRELOAD_KEY)
76+
catch
77+
error:badarg -> undefined
78+
end.
79+
80+
%% @doc Clear the preload code.
81+
%%
82+
%% New contexts will start with empty globals. Existing contexts
83+
%% are not affected.
84+
-spec clear_code() -> ok.
85+
clear_code() ->
86+
try
87+
persistent_term:erase(?PRELOAD_KEY)
88+
catch
89+
error:badarg -> ok
90+
end,
91+
ok.
92+
93+
%% @doc Check if preload code is configured.
94+
-spec has_preload() -> boolean().
95+
has_preload() ->
96+
get_code() =/= undefined.
97+
98+
%% @doc Apply preload code to a context reference.
99+
%%
100+
%% Called internally by `py_context' during context initialization.
101+
%% Executes the preload code in the context's interpreter.
102+
%%
103+
%% @param Ref NIF context reference
104+
%% @returns `ok' if successful or no preload configured, `{error, Reason}' on failure
105+
-spec apply_preload(reference()) -> ok | {error, term()}.
106+
apply_preload(Ref) ->
107+
case get_code() of
108+
undefined -> ok;
109+
Code -> py_nif:context_exec(Ref, Code)
110+
end.

0 commit comments

Comments
 (0)