Skip to content

Commit 3dcec6f

Browse files
committed
Document OWN_GIL and event loop per-process environments
docs/process-bound-envs.md: - Add OWN_GIL Mode section with explicit environment creation - Add Sharing Context, Isolating State examples - Add When to Use Explicit vs Implicit table - Add Event Loop Environments section with examples - Add event_loop_exec/eval usage for defining async functions - Update See Also with OWN_GIL internals link docs/event_loop_architecture.md: - Add Usage section with practical examples - Add Evaluating Expressions examples - Add Process Isolation examples showing namespace independence
1 parent 7921bbe commit 3dcec6f

2 files changed

Lines changed: 217 additions & 1 deletion

File tree

docs/event_loop_architecture.md

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,68 @@ pthread_mutex_unlock PyGILState_Release
247247
248248
Each Erlang process can have an isolated Python namespace within an event loop. These namespaces are tracked in a linked list protected by `namespaces_mutex`.
249249
250+
### Usage
251+
252+
Define functions and state for async tasks in your process's namespace:
253+
254+
```erlang
255+
%% Get event loop reference
256+
{ok, Loop} = py_event_loop:get_loop(),
257+
LoopRef = py_event_loop:get_nif_ref(Loop),
258+
259+
%% Define async functions in this process's namespace
260+
ok = py_nif:event_loop_exec(LoopRef, <<"
261+
import asyncio
262+
263+
async def process_data(items):
264+
results = []
265+
for item in items:
266+
await asyncio.sleep(0.01) # Simulate async I/O
267+
results.append(item * 2)
268+
return results
269+
270+
# State persists across calls
271+
call_count = 0
272+
273+
async def tracked_call(x):
274+
global call_count
275+
call_count += 1
276+
return {'result': x, 'call_number': call_count}
277+
">>),
278+
279+
%% Use the functions via create_task with __main__ module
280+
{ok, Ref1} = py_event_loop:create_task(Loop, '__main__', process_data, [[1,2,3]]),
281+
{ok, [2,4,6]} = py_event_loop:await(Ref1),
282+
283+
%% State is maintained
284+
{ok, Ref2} = py_event_loop:create_task(Loop, '__main__', tracked_call, [42]),
285+
{ok, #{<<"result">> := 42, <<"call_number">> := 1}} = py_event_loop:await(Ref2).
286+
```
287+
288+
### Evaluating Expressions
289+
290+
```erlang
291+
%% Quick evaluation in the process namespace
292+
{ok, 100} = py_nif:event_loop_eval(LoopRef, <<"50 * 2">>),
293+
294+
%% Access previously defined variables
295+
ok = py_nif:event_loop_exec(LoopRef, <<"config = {'timeout': 30}">>),
296+
{ok, #{<<"timeout">> := 30}} = py_nif:event_loop_eval(LoopRef, <<"config">>).
297+
```
298+
299+
### Process Isolation
300+
301+
Each Erlang process has its own isolated namespace:
302+
303+
```erlang
304+
%% Two processes define the same variable name - no conflict
305+
Pids = [spawn(fun() ->
306+
ok = py_nif:event_loop_exec(LoopRef, <<"my_id = ", (integer_to_binary(N))/binary>>),
307+
{ok, N} = py_nif:event_loop_eval(LoopRef, <<"my_id">>),
308+
io:format("Process ~p has my_id = ~p~n", [self(), N])
309+
end) || N <- lists:seq(1, 5)].
310+
```
311+
250312
### Lock Ordering
251313

252314
To prevent ABBA deadlocks, locks must always be acquired in this order:

docs/process-bound-envs.md

Lines changed: 155 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,158 @@ spawn(fun() ->
3939
end).
4040
```
4141

42+
## OWN_GIL Mode
43+
44+
OWN_GIL contexts (Python 3.12+) provide true parallel execution with dedicated pthreads. Process-bound environments work with OWN_GIL, allowing multiple Erlang processes to share a single OWN_GIL context while maintaining isolated Python namespaces.
45+
46+
### Explicit Environment Creation
47+
48+
For OWN_GIL contexts, you can explicitly create and manage environments:
49+
50+
```erlang
51+
%% Create an OWN_GIL context
52+
{ok, Ctx} = py_context:start_link(1, owngil),
53+
54+
%% Create a process-local environment
55+
{ok, Env} = py_context:create_local_env(Ctx),
56+
57+
%% Get the NIF reference for low-level operations
58+
CtxRef = py_context:get_nif_ref(Ctx),
59+
60+
%% Execute code in the isolated environment
61+
ok = py_nif:context_exec(CtxRef, <<"
62+
class MyService:
63+
def __init__(self):
64+
self.counter = 0
65+
def increment(self):
66+
self.counter += 1
67+
return self.counter
68+
69+
service = MyService()
70+
">>, Env),
71+
72+
%% Call functions in the environment
73+
{ok, 1} = py_nif:context_eval(CtxRef, <<"service.increment()">>, #{}, Env),
74+
{ok, 2} = py_nif:context_eval(CtxRef, <<"service.increment()">>, #{}, Env).
75+
```
76+
77+
### Sharing Context, Isolating State
78+
79+
Multiple Erlang processes can share an OWN_GIL context while maintaining isolated namespaces:
80+
81+
```erlang
82+
%% Shared OWN_GIL context
83+
{ok, Ctx} = py_context:start_link(1, owngil),
84+
CtxRef = py_context:get_nif_ref(Ctx),
85+
86+
%% Process A - its own namespace
87+
spawn(fun() ->
88+
{ok, EnvA} = py_context:create_local_env(Ctx),
89+
ok = py_nif:context_exec(CtxRef, <<"x = 'from A'">>, EnvA),
90+
{ok, <<"from A">>} = py_nif:context_eval(CtxRef, <<"x">>, #{}, EnvA)
91+
end),
92+
93+
%% Process B - separate namespace, same context
94+
spawn(fun() ->
95+
{ok, EnvB} = py_context:create_local_env(Ctx),
96+
ok = py_nif:context_exec(CtxRef, <<"x = 'from B'">>, EnvB),
97+
{ok, <<"from B">>} = py_nif:context_eval(CtxRef, <<"x">>, #{}, EnvB)
98+
end).
99+
%% Both execute in parallel on the same OWN_GIL thread, but with isolated state
100+
```
101+
102+
### When to Use Explicit vs Implicit Environments
103+
104+
| Approach | API | Use Case |
105+
|----------|-----|----------|
106+
| **Implicit** | `py:exec/eval/call` | Simple cases, automatic management |
107+
| **Explicit** | `create_local_env` + `py_nif:context_*` | OWN_GIL, fine-grained control, multiple envs per process |
108+
109+
**Use implicit (py:exec)** when:
110+
- Using worker or subinterp modes
111+
- One environment per process is sufficient
112+
- You want automatic lifecycle management
113+
114+
**Use explicit (create_local_env)** when:
115+
- Using OWN_GIL mode for parallel execution
116+
- Need multiple environments in a single process
117+
- Want to pass environments between processes
118+
- Need direct NIF-level control
119+
120+
## Event Loop Environments
121+
122+
The event loop API also supports per-process namespaces. Each Erlang process gets an isolated namespace within the event loop, allowing you to define functions and state that persist across async task calls.
123+
124+
### Defining Functions for Async Tasks
125+
126+
```erlang
127+
%% Get the event loop reference
128+
{ok, Loop} = py_event_loop:get_loop(),
129+
LoopRef = py_event_loop:get_nif_ref(Loop),
130+
131+
%% Define a function in this process's namespace
132+
ok = py_nif:event_loop_exec(LoopRef, <<"
133+
import asyncio
134+
135+
async def my_async_function(x):
136+
await asyncio.sleep(0.1)
137+
return x * 2
138+
139+
counter = 0
140+
141+
async def increment_and_get():
142+
global counter
143+
counter += 1
144+
return counter
145+
">>),
146+
147+
%% Call the function via create_task - uses __main__ module
148+
{ok, Ref} = py_event_loop:create_task(Loop, '__main__', my_async_function, [21]),
149+
{ok, 42} = py_event_loop:await(Ref),
150+
151+
%% State persists across calls
152+
{ok, Ref1} = py_event_loop:create_task(Loop, '__main__', increment_and_get, []),
153+
{ok, 1} = py_event_loop:await(Ref1),
154+
{ok, Ref2} = py_event_loop:create_task(Loop, '__main__', increment_and_get, []),
155+
{ok, 2} = py_event_loop:await(Ref2).
156+
```
157+
158+
### Evaluating Expressions
159+
160+
```erlang
161+
%% Evaluate expressions in the process's namespace
162+
{ok, 42} = py_nif:event_loop_eval(LoopRef, <<"21 * 2">>),
163+
164+
%% Access variables defined via exec
165+
ok = py_nif:event_loop_exec(LoopRef, <<"result = 'computed'">>),
166+
{ok, <<"computed">>} = py_nif:event_loop_eval(LoopRef, <<"result">>).
167+
```
168+
169+
### Process Isolation
170+
171+
Different Erlang processes have isolated event loop namespaces:
172+
173+
```erlang
174+
{ok, Loop} = py_event_loop:get_loop(),
175+
LoopRef = py_event_loop:get_nif_ref(Loop),
176+
177+
%% Process A defines x
178+
spawn(fun() ->
179+
ok = py_nif:event_loop_exec(LoopRef, <<"x = 'A'">>),
180+
{ok, <<"A">>} = py_nif:event_loop_eval(LoopRef, <<"x">>)
181+
end),
182+
183+
%% Process B has its own x
184+
spawn(fun() ->
185+
ok = py_nif:event_loop_exec(LoopRef, <<"x = 'B'">>),
186+
{ok, <<"B">>} = py_nif:event_loop_eval(LoopRef, <<"x">>)
187+
end).
188+
```
189+
190+
### Cleanup
191+
192+
Event loop namespaces are automatically cleaned up when the Erlang process exits. The event loop monitors each process that creates a namespace and removes it on process termination.
193+
42194
## Building Python Actors
43195

44196
The process-bound model enables a pattern we call "Python actors" - Erlang processes that encapsulate Python state and expose it through message passing.
@@ -277,6 +429,8 @@ This design prioritizes safety over avoiding minor memory leaks during edge case
277429

278430
## See Also
279431

432+
- [OWN_GIL Internals](owngil_internals.md) - Architecture and safety mechanisms for OWN_GIL mode
433+
- [Scalability](scalability.md) - Mode comparison (owngil vs subinterp vs worker)
434+
- [Event Loop Architecture](event_loop_architecture.md) - Per-process namespace management
280435
- [Context Affinity](context-affinity.md) - Context binding and routing
281436
- [Scheduling](asyncio.md) - Cooperative scheduling for long operations
282-
- [Scalability](scalability.md) - Multi-context and subinterpreter configurations

0 commit comments

Comments
 (0)