Optimize garbage collection with a generational GC

code-queries/allocations-exceeding-ensure-free.ql

@@ -950,6 +950,20 @@ predicate consumesContextBudget(FunctionCall efCall, FunctionCall consumer) {
         ensureFreeContextVar(efCall)
 }
 
+/**
+ * Holds if `f` tears down a context (e.g. `context_destroy`): it frees the
+ * context rather than re-establishing a usable heap budget on it. Such a
+ * function may reach a memory_ensure_free variant incidentally while draining
+ * signals during teardown, but that does not make a preceding reservation
+ * redundant -- there is no surviving budget for a caller to use. Excluded from
+ * the superseding-reset notion to avoid flagging a deliberate
+ * `ensure_free(ctx, N); ...; context_destroy(ctx);` (e.g. a test that forces a
+ * GC then destroys the context).
+ */
+predicate isContextTeardown(Function f) {
+    f.hasName("context_destroy")
+}
+
 /**
  * Holds if `efCall` is a redundant reserving ensure_free: no allocating call
  * uses its budget, and `supersedingCall` is a subsequent call that resets
@@ -986,8 +1000,10 @@ predicate isRedundantEnsureFree(FunctionCall efCall, FunctionCall supersedingCal
             // A function that internally calls ensure_free on the caller's
             // context (e.g., enif_make_resource), passed that same context as an
             // argument. Uses the ensure_free-only notion: own-heap setup does not
-            // reset this context's budget.
+            // reset this context's budget. Context teardown (context_destroy) is
+            // excluded: it frees the context rather than re-establishing a budget.
             not isEnsureFreeCall(supersedingCall) and
+            not isContextTeardown(supersedingCall.getTarget()) and
             transitivelyCallsEnsureFreeOnly(supersedingCall.getTarget()) and
             supersedingCall.getAnArgument().(VariableAccess).getTarget() = ctxVar
         ) and

doc/src/memory-management.md

@@ -922,3 +922,40 @@ match binaries, as with the case of refc binaries on the process heap.
 #### Deletion
 
 Once all terms have been copied from the old heap to the new heap, and once the MSO list has been swept for unreachable references, the old heap is simply discarded via the `free` function.
+
+### Generational Garbage Collection
+
+The garbage collection described above is a *full sweep*: every live term is copied from the old heap to the new heap and the entire old heap is freed. While correct, this can be expensive for processes with large heaps, because long-lived data that has already survived previous collections must be copied again each time.
+
+AtomVM implements *generational* (or *minor*) garbage collection to reduce this cost, using the same approach as BEAM. The key observation is that most terms die young: they are allocated, used briefly, and become garbage. Terms that have survived at least one collection are likely to survive many more. Generational GC exploits this by dividing the heap into two generations:
+
+* **Young generation**: recently allocated terms, between the *high water mark* and the current heap pointer.
+* **Old (mature) generation**: terms that have survived at least one minor collection, stored in a separate old heap.
+
+#### High Water Mark
+
+After each garbage collection, the heap pointer position is recorded as the *high water mark*. On the next collection, terms allocated below the high water mark (i.e., terms that existed at the time of the previous collection) are considered mature. Terms allocated above the high water mark are young.
+
+#### Minor Collection
+
+During a minor collection:
+
+1. A new young heap is allocated.
+2. Mature terms (below the high water mark) are *promoted*: copied to the old heap rather than the new young heap.
+3. Young terms that are still reachable are copied to the new young heap.
+4. Both the new young heap and the newly promoted old region are scanned for references, since promoted terms may reference young terms and vice versa.
+5. Only the young MSO list is swept; the old MSO list is preserved.
+6. The previous heap is freed, but the old heap persists across minor collections.
+
+Because the old heap is not scanned for garbage during a minor collection, the cost is proportional to the size of the young generation rather than the entire heap.
+
+#### When Full vs. Minor Collection Occurs
+
+AtomVM keeps a counter (`gc_count`) of how many minor collections have occurred since the last full sweep. A full sweep is forced when:
+
+* The process has never been garbage collected (no high water mark exists).
+* `gc_count` reaches the `fullsweep_after` threshold.
+* The old heap does not have enough space to accommodate promoted terms.
+* A `MEMORY_FORCE_SHRINK` request is made (e.g., via `erlang:garbage_collect/0`).
+
+The `fullsweep_after` value can be set per-process via [`spawn_opt`](./programmers-guide.md#spawning-processes) or [`erlang:process_flag/2`](./apidocs/erlang/estdlib/erlang.md#process_flag2). The default value is 65535, meaning full sweeps are infrequent under normal operation. Setting it to `0` disables generational collection entirely, forcing a full sweep on every garbage collection event.

doc/src/programmers-guide.md

@@ -365,6 +365,7 @@ The [options](./apidocs/erlang/estdlib/erlang.md#spawn_option) argument is a pro
 |-----|------------|---------------|-------------|
 | `min_heap_size` | `non_neg_integer()` | none | Minimum heap size of the process.  The heap will shrink no smaller than this size. |
 | `max_heap_size` | `non_neg_integer()` | unbounded | Maximum heap size of the process.  The heap will grow no larger than this size. |
+| `fullsweep_after` | `non_neg_integer()` | 65535 | Maximum number of [minor garbage collections](./memory-management.md#generational-garbage-collection) before a full sweep is forced.  Set to `0` to disable generational garbage collection. |
 | `link` | `boolean()` | `false` | Whether to link the spawned process to the spawning process. |
 | `monitor` | `boolean()` | `false` | Whether to link the spawning process should monitor the spawned process. |
 | `atomvm_heap_growth` | `bounded_free \| minimum \| fibonacci` | `bounded_free` | [Strategy](./memory-management.md#heap-growth-strategies) to grow the heap of the process. |

libs/estdlib/src/erlang.erl

@@ -210,6 +210,7 @@
 -type spawn_option() ::
     {min_heap_size, pos_integer()}
     | {max_heap_size, pos_integer()}
+    | {fullsweep_after, non_neg_integer()}
     | {atomvm_heap_growth, atomvm_heap_growth_strategy()}
     | link
     | monitor.
@@ -1441,7 +1442,9 @@ group_leader(_Leader, _Pid) ->
 %%
 %% @end
 %%-----------------------------------------------------------------------------
--spec process_flag(Flag :: trap_exit, Value :: boolean()) -> pid().
+-spec process_flag
+    (trap_exit, boolean()) -> boolean();
+    (fullsweep_after, non_neg_integer()) -> non_neg_integer().
 process_flag(_Flag, _Value) ->
     erlang:nif_error(undefined).
 

libs/jit/src/jit_aarch64.erl

@@ -77,6 +77,8 @@
     xor_/3
 ]).
 
+-export([dwarf_x_reg_offset/0]).
+
 -ifdef(JIT_DWARF).
 -export([
     dwarf_opcode/2,
@@ -186,25 +188,25 @@
     | {maybe_free_aarch64_register(), '&', non_neg_integer(), '!=', integer()}
     | {{free, aarch64_register()}, '==', {free, aarch64_register()}}.
 
-% ctx->e is 0x28
-% ctx->x is 0x30
+% ctx->e is 0x50
+% ctx->x is 0x58
 -define(WORD_SIZE, 8).
 -define(CTX_REG, r0).
 -define(JITSTATE_REG, r1).
 -define(NATIVE_INTERFACE_REG, r2).
--define(Y_REGS, {?CTX_REG, 16#28}).
--define(X_REG(N), {?CTX_REG, 16#30 + (N * ?WORD_SIZE)}).
--define(CP, {?CTX_REG, 16#B8}).
--define(FP_REGS, {?CTX_REG, 16#C0}).
+-define(Y_REGS, {?CTX_REG, 16#50}).
+-define(X_REG(N), {?CTX_REG, 16#58 + (N * ?WORD_SIZE)}).
+-define(CP, {?CTX_REG, 16#E0}).
+-define(FP_REGS, {?CTX_REG, 16#E8}).
 -define(FP_REG_OFFSET(State, F),
     (F *
         case (State)#state.variant band ?JIT_VARIANT_FLOAT32 of
             0 -> 8;
             _ -> 4
         end)
 ).
--define(BS, {?CTX_REG, 16#C8}).
--define(BS_OFFSET, {?CTX_REG, 16#D0}).
+-define(BS, {?CTX_REG, 16#F0}).
+-define(BS_OFFSET, {?CTX_REG, 16#F8}).
 -define(JITSTATE_MODULE, {?JITSTATE_REG, 0}).
 -define(JITSTATE_CONTINUATION, {?JITSTATE_REG, 16#8}).
 -define(JITSTATE_REDUCTIONCOUNT, {?JITSTATE_REG, 16#10}).
@@ -3111,6 +3113,12 @@ add_label(
 add_label(#state{labels = Labels} = State, Label, Offset) ->
     State#state{labels = Labels#{Label => Offset}}.
 
+%% @doc Byte offset of the `x' register array within the Context struct.
+%% Derived from ?X_REG so it tracks the codegen offset.
+-spec dwarf_x_reg_offset() -> non_neg_integer().
+dwarf_x_reg_offset() ->
+    element(2, ?X_REG(0)).
+
 -ifdef(JIT_DWARF).
 %%-----------------------------------------------------------------------------
 %% @doc Return the DWARF register number for the ctx parameter

libs/jit/src/jit_arm32.erl

@@ -75,6 +75,8 @@
     xor_/3
 ]).
 
+-export([dwarf_x_reg_offset/0]).
+
 -ifdef(JIT_DWARF).
 -export([
     dwarf_opcode/2,
@@ -175,16 +177,16 @@
     | {maybe_free_arm32_register(), '&', non_neg_integer(), '!=', integer()}
     | {{free, arm32_register()}, '==', {free, arm32_register()}}.
 
-% ctx->e is 0x14
-% ctx->x is 0x18
+% ctx->e is 0x28
+% ctx->x is 0x2C
 -define(CTX_REG, r0).
 -define(NATIVE_INTERFACE_REG, r2).
--define(Y_REGS, {?CTX_REG, 16#14}).
--define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}).
--define(CP, {?CTX_REG, 16#5C}).
--define(FP_REGS, {?CTX_REG, 16#60}).
--define(BS, {?CTX_REG, 16#64}).
--define(BS_OFFSET, {?CTX_REG, 16#68}).
+-define(Y_REGS, {?CTX_REG, 16#28}).
+-define(X_REG(N), {?CTX_REG, 16#2C + (N * 4)}).
+-define(CP, {?CTX_REG, 16#70}).
+-define(FP_REGS, {?CTX_REG, 16#74}).
+-define(BS, {?CTX_REG, 16#78}).
+-define(BS_OFFSET, {?CTX_REG, 16#7C}).
 % JITSTATE is on stack, accessed via stack offset
 % These macros now expect a register that contains the jit_state pointer
 -define(JITSTATE_MODULE(Reg), {Reg, 0}).
@@ -3889,6 +3891,12 @@ value_to_contents(Value) -> jit_regs:value_to_contents(Value, ?MAX_REG).
 %% Convert a VM register destination to a contents descriptor.
 vm_dest_to_contents(Dest) -> jit_regs:vm_dest_to_contents(Dest, ?MAX_REG).
 
+%% @doc Byte offset of the `x' register array within the Context struct.
+%% Derived from ?X_REG so it tracks the codegen offset.
+-spec dwarf_x_reg_offset() -> non_neg_integer().
+dwarf_x_reg_offset() ->
+    element(2, ?X_REG(0)).
+
 -ifdef(JIT_DWARF).
 %%-----------------------------------------------------------------------------
 %% @doc Return the DWARF register number for the ctx parameter

libs/jit/src/jit_armv6m.erl

@@ -75,6 +75,8 @@
     xor_/3
 ]).
 
+-export([dwarf_x_reg_offset/0]).
+
 -ifdef(JIT_DWARF).
 -export([
     dwarf_opcode/2,
@@ -196,15 +198,15 @@
     | {{free, armv6m_register()}, '==', {free, armv6m_register()}}.
 
 % ctx->e is 0x28
-% ctx->x is 0x30
+% ctx->x is 0x2C
 -define(CTX_REG, r0).
 -define(NATIVE_INTERFACE_REG, r2).
--define(Y_REGS, {?CTX_REG, 16#14}).
--define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}).
--define(CP, {?CTX_REG, 16#5C}).
--define(FP_REGS, {?CTX_REG, 16#60}).
--define(BS, {?CTX_REG, 16#64}).
--define(BS_OFFSET, {?CTX_REG, 16#68}).
+-define(Y_REGS, {?CTX_REG, 16#28}).
+-define(X_REG(N), {?CTX_REG, 16#2C + (N * 4)}).
+-define(CP, {?CTX_REG, 16#70}).
+-define(FP_REGS, {?CTX_REG, 16#74}).
+-define(BS, {?CTX_REG, 16#78}).
+-define(BS_OFFSET, {?CTX_REG, 16#7C}).
 % JITSTATE is on stack, accessed via stack offset
 % These macros now expect a register that contains the jit_state pointer
 -define(JITSTATE_MODULE(Reg), {Reg, 0}).
@@ -4381,6 +4383,12 @@ add_label(
 add_label(#state{labels = Labels} = State, Label, Offset) ->
     State#state{labels = Labels#{Label => Offset}}.
 
+%% @doc Byte offset of the `x' register array within the Context struct.
+%% Derived from ?X_REG so it tracks the codegen offset.
+-spec dwarf_x_reg_offset() -> non_neg_integer().
+dwarf_x_reg_offset() ->
+    element(2, ?X_REG(0)).
+
 -ifdef(JIT_DWARF).
 %%-----------------------------------------------------------------------------
 %% @doc Return the DWARF register number for the ctx parameter

libs/jit/src/jit_dwarf.erl

@@ -963,11 +963,7 @@ generate_named_var_loc_section(
             WordSize = Backend:word_size(),
             WSBits = WordSize * 8,
             CtxRegNum = Backend:dwarf_ctx_register(),
-            XArrayOffset =
-                case WordSize of
-                    8 -> 16#30;
-                    4 -> 16#18
-                end,
+            XArrayOffset = Backend:dwarf_x_reg_offset(),
             {_LowPC, HighPC} = calculate_address_range(State),
             SortedVars = lists:sort(fun({A, _}, {B, _}) -> A =< B end, Variables),
             SortedFuncs = lists:sort([{Off, FN, Ar} || {Off, FN, Ar} <- Functions, Off >= 0]),
@@ -1141,11 +1137,7 @@ generate_debug_loc_section(#dwarf{reg_locations = RegLocs, backend = Backend}) -
     WordSize = Backend:word_size(),
     WSBits = WordSize * 8,
     CtxRegNum = Backend:dwarf_ctx_register(),
-    XArrayOffset =
-        case WordSize of
-            8 -> 16#30;
-            4 -> 16#18
-        end,
+    XArrayOffset = Backend:dwarf_x_reg_offset(),
 
     % Sort snapshots by offset (they're stored in reverse)
     Sorted = lists:sort(fun({A, _}, {B, _}) -> A =< B end, RegLocs),
@@ -1405,14 +1397,7 @@ generate_type_dies(#dwarf{backend = Backend}, BaseOffset) ->
 
     % Abbrev 8: Context structure type
     % Only include the x array member for now (most important for debugging)
-    XOffset =
-        case Backend of
-            jit_x86_64 -> 16#30;
-            jit_aarch64 -> 16#30;
-            jit_riscv64 -> 16#30;
-            % riscv32 and armv6m
-            _ -> 16#18
-        end,
+    XOffset = Backend:dwarf_x_reg_offset(),
     XMemberDIE = <<
         9,
         "x",

libs/jit/src/jit_riscv32.erl

@@ -77,6 +77,8 @@
     rem_/3
 ]).
 
+-export([dwarf_x_reg_offset/0]).
+
 -ifdef(JIT_DWARF).
 -export([
     dwarf_opcode/2,
@@ -204,16 +206,16 @@
     | {{free, riscv32_register()}, '==', {free, riscv32_register()}}.
 
 % Context offsets (32-bit architecture)
-% ctx->e is 0x14
-% ctx->x is 0x18
+% ctx->e is 0x28
+% ctx->x is 0x2C
 -define(CTX_REG, a0).
 -define(NATIVE_INTERFACE_REG, a2).
--define(Y_REGS, {?CTX_REG, 16#14}).
--define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}).
--define(CP, {?CTX_REG, 16#5C}).
--define(FP_REGS, {?CTX_REG, 16#60}).
--define(BS, {?CTX_REG, 16#64}).
--define(BS_OFFSET, {?CTX_REG, 16#68}).
+-define(Y_REGS, {?CTX_REG, 16#28}).
+-define(X_REG(N), {?CTX_REG, 16#2C + (N * 4)}).
+-define(CP, {?CTX_REG, 16#70}).
+-define(FP_REGS, {?CTX_REG, 16#74}).
+-define(BS, {?CTX_REG, 16#78}).
+-define(BS_OFFSET, {?CTX_REG, 16#7C}).
 -define(JITSTATE_REG, a1).
 -define(RA_REG, ra).
 -define(JITSTATE_MODULE_OFFSET, 0).
@@ -321,6 +323,12 @@ handle_avm_int64_t(State, Value, ArgsT, ArgsRegs, ParamRegs, AvailGP, StackOffse
         State, [LowPart, HighPart | ArgsT], [imm | ArgsRegs], ParamRegs, AvailGP, StackOffset
     ).
 
+%% @doc Byte offset of the `x' register array within the Context struct.
+%% Derived from ?X_REG so it tracks the codegen offset.
+-spec dwarf_x_reg_offset() -> non_neg_integer().
+dwarf_x_reg_offset() ->
+    element(2, ?X_REG(0)).
+
 -ifdef(JIT_DWARF).
 -spec dwarf_ctx_register() -> non_neg_integer().
 dwarf_ctx_register() ->

libs/jit/src/jit_riscv64.erl

@@ -77,6 +77,8 @@
     shift_right_arith/3
 ]).
 
+-export([dwarf_x_reg_offset/0]).
+
 -ifdef(JIT_DWARF).
 -export([
     dwarf_opcode/2,
@@ -211,16 +213,16 @@
     | {{free, riscv64_register()}, '==', {free, riscv64_register()}}.
 
 % Context offsets (64-bit architecture)
-% ctx->e is 0x28
-% ctx->x is 0x30
+% ctx->e is 0x50
+% ctx->x is 0x58
 -define(CTX_REG, a0).
 -define(NATIVE_INTERFACE_REG, a2).
--define(Y_REGS, {?CTX_REG, 16#28}).
--define(X_REG(N), {?CTX_REG, 16#30 + (N * 8)}).
--define(CP, {?CTX_REG, 16#B8}).
--define(FP_REGS, {?CTX_REG, 16#C0}).
--define(BS, {?CTX_REG, 16#C8}).
--define(BS_OFFSET, {?CTX_REG, 16#D0}).
+-define(Y_REGS, {?CTX_REG, 16#50}).
+-define(X_REG(N), {?CTX_REG, 16#58 + (N * 8)}).
+-define(CP, {?CTX_REG, 16#E0}).
+-define(FP_REGS, {?CTX_REG, 16#E8}).
+-define(BS, {?CTX_REG, 16#F0}).
+-define(BS_OFFSET, {?CTX_REG, 16#F8}).
 % JITSTATE is in a1 register (no prolog needed)
 -define(JITSTATE_REG, a1).
 % Return address register
@@ -330,6 +332,12 @@ handle_avm_int64_t(State, Value, ArgsT, ArgsRegs, ParamRegs, AvailGP, StackOffse
         State, [Value | ArgsT], ArgsRegs, ParamRegs, AvailGP, StackOffset
     ).
 
+%% @doc Byte offset of the `x' register array within the Context struct.
+%% Derived from ?X_REG so it tracks the codegen offset.
+-spec dwarf_x_reg_offset() -> non_neg_integer().
+dwarf_x_reg_offset() ->
+    element(2, ?X_REG(0)).
+
 -ifdef(JIT_DWARF).
 -spec dwarf_ctx_register() -> non_neg_integer().
 dwarf_ctx_register() ->