| 1 | module optimize |
| 2 | |
| 3 | import v3.ssa |
| 4 | |
| 5 | // --- Mem2Reg (promote allocas to SSA values + phi nodes) --- |
| 6 | // Flat arrays indexed by value_id / block_id. Phi predecessor operands store |
| 7 | // raw block ids, matching the rest of the v3 SSA (and the arm64 backend's |
| 8 | // emit_phi_edge_copies, which reads operand[odd] as a block id). |
| 9 | struct Mem2RegCtx { |
| 10 | mut: |
| 11 | defs [][]int // alloc_id -> blocks storing to it |
| 12 | uses [][]int // alloc_id -> blocks loading from it |
| 13 | phi_placements [][]int // block_id -> alloc_ids needing a phi here |
| 14 | phi_vals [][]int // block_id -> parallel phi value ids |
| 15 | stacks [][]int // alloc_id -> reaching-definition stack |
| 16 | is_promotable []bool |
| 17 | phi_blocks []int |
| 18 | } |
| 19 | |
| 20 | // array2d_append supports array2d append handling for optimize. |
| 21 | fn array2d_append(mut arr [][]int, idx int, val int) { |
| 22 | mut inner := arr[idx] |
| 23 | inner << val |
| 24 | arr[idx] = inner |
| 25 | } |
| 26 | |
| 27 | // promote_memory_to_register promotes scalar allocas to SSA registers, inserting |
| 28 | // phi nodes at dominance frontiers and renaming load/store chains. |
| 29 | fn promote_memory_to_register(mut m ssa.Module, dom DomInfo, cfg &CfgData) { |
| 30 | n_values := m.values.len |
| 31 | n_blocks := m.blocks.len |
| 32 | |
| 33 | mut ctx := Mem2RegCtx{ |
| 34 | defs: [][]int{len: n_values} |
| 35 | uses: [][]int{len: n_values} |
| 36 | phi_placements: [][]int{len: n_blocks} |
| 37 | phi_vals: [][]int{len: n_blocks} |
| 38 | stacks: [][]int{len: n_values} |
| 39 | is_promotable: []bool{len: n_values} |
| 40 | phi_blocks: []int{} |
| 41 | } |
| 42 | |
| 43 | mut df := [][]int{len: n_blocks} |
| 44 | mut df_blocks := []int{} |
| 45 | mut visited := []bool{len: n_blocks} |
| 46 | mut has_phi := []bool{len: n_blocks} |
| 47 | |
| 48 | for fi in 0 .. m.funcs.len { |
| 49 | mut promotable := []int{} |
| 50 | func := m.funcs[fi] |
| 51 | for blk_id in func.blocks { |
| 52 | blk := m.blocks[blk_id] |
| 53 | for val_id in blk.instrs { |
| 54 | val := m.values[val_id] |
| 55 | if val.kind != .instruction { |
| 56 | continue |
| 57 | } |
| 58 | instr := m.instrs[val.index] |
| 59 | if instr.op == .alloca { |
| 60 | if is_promotable(m, val_id) { |
| 61 | promotable << val_id |
| 62 | ctx.is_promotable[val_id] = true |
| 63 | } |
| 64 | } |
| 65 | if instr.op == .store { |
| 66 | ptr := instr.operands[1] |
| 67 | if ptr < n_values && blk_id !in ctx.defs[ptr] { |
| 68 | array2d_append(mut ctx.defs, ptr, blk_id) |
| 69 | } |
| 70 | } else if instr.op == .load { |
| 71 | ptr := instr.operands[0] |
| 72 | if ptr < n_values && blk_id !in ctx.uses[ptr] { |
| 73 | array2d_append(mut ctx.uses, ptr, blk_id) |
| 74 | } |
| 75 | } |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | compute_dominance_frontier_flat(m, fi, &dom, cfg, mut df, mut df_blocks) |
| 80 | |
| 81 | // Insert phis at iterated dominance frontiers of each promotable alloca. |
| 82 | for alloc_id in promotable { |
| 83 | mut worklist := ctx.defs[alloc_id].clone() |
| 84 | mut visited_list := []int{} |
| 85 | mut has_phi_list := []int{} |
| 86 | for worklist.len > 0 { |
| 87 | b := worklist.pop() |
| 88 | if b < 0 || b >= n_blocks { |
| 89 | continue |
| 90 | } |
| 91 | for d in df[b] { |
| 92 | if !has_phi[d] { |
| 93 | array2d_append(mut ctx.phi_placements, d, alloc_id) |
| 94 | if d !in ctx.phi_blocks { |
| 95 | ctx.phi_blocks << d |
| 96 | } |
| 97 | has_phi[d] = true |
| 98 | has_phi_list << d |
| 99 | if !visited[d] { |
| 100 | visited[d] = true |
| 101 | visited_list << d |
| 102 | worklist << d |
| 103 | } |
| 104 | } |
| 105 | } |
| 106 | } |
| 107 | for d in visited_list { |
| 108 | visited[d] = false |
| 109 | } |
| 110 | for d in has_phi_list { |
| 111 | has_phi[d] = false |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | // Materialize the phi instructions (empty operands, filled during rename). |
| 116 | for blk_id in ctx.phi_blocks { |
| 117 | for alloc_id in ctx.phi_placements[blk_id] { |
| 118 | alloc_val := m.values[alloc_id] |
| 119 | alloc_typ := m.type_store.types[alloc_val.typ] |
| 120 | typ := alloc_typ.elem_type |
| 121 | phi_val := m.add_instr_front(.phi, blk_id, typ, []) |
| 122 | mut pv := m.values[phi_val] |
| 123 | pv.name = '${alloc_val.name}.phi_${blk_id}' |
| 124 | m.values[phi_val] = pv |
| 125 | array2d_append(mut ctx.phi_vals, blk_id, phi_val) |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | // Rename: walk the dominator tree, threading reaching definitions. |
| 130 | if func.blocks.len > 0 { |
| 131 | rename_blocks(mut m, func.blocks[0], mut ctx, dom, cfg) |
| 132 | } |
| 133 | |
| 134 | // Reset per-function state. |
| 135 | for alloc_id in promotable { |
| 136 | ctx.defs[alloc_id] = [] |
| 137 | ctx.uses[alloc_id] = [] |
| 138 | ctx.stacks[alloc_id] = [] |
| 139 | ctx.is_promotable[alloc_id] = false |
| 140 | } |
| 141 | for pb in ctx.phi_blocks { |
| 142 | ctx.phi_placements[pb] = [] |
| 143 | ctx.phi_vals[pb] = [] |
| 144 | } |
| 145 | ctx.phi_blocks = [] |
| 146 | for d in df_blocks { |
| 147 | df[d] = [] |
| 148 | } |
| 149 | df_blocks = [] |
| 150 | } |
| 151 | } |
| 152 | |
| 153 | // is_promotable returns true if every use of the alloca is a direct load/store, |
| 154 | // i.e. the pointer never escapes. Array- and pointer-backed slots stay in memory. |
| 155 | fn is_promotable(m &ssa.Module, alloc_id int) bool { |
| 156 | if alloc_id > 0 && alloc_id < m.values.len { |
| 157 | alloc_typ_id := m.values[alloc_id].typ |
| 158 | if alloc_typ_id > 0 && alloc_typ_id < m.type_store.types.len { |
| 159 | alloc_typ := m.type_store.types[alloc_typ_id] |
| 160 | if alloc_typ.kind == .ptr_t { |
| 161 | elem_typ_id := alloc_typ.elem_type |
| 162 | if elem_typ_id > 0 && elem_typ_id < m.type_store.types.len { |
| 163 | elem_typ := m.type_store.types[elem_typ_id] |
| 164 | // Keep pointer-, array- and aggregate-backed slots in memory. |
| 165 | // Promoting them to SSA values would require the backend to |
| 166 | // thread by-value aggregates through phi copies, which the v3 |
| 167 | // arm64 lowering does not yet handle reliably. |
| 168 | if elem_typ.kind in [.ptr_t, .array_t, .struct_t] { |
| 169 | return false |
| 170 | } |
| 171 | } |
| 172 | } |
| 173 | } |
| 174 | } |
| 175 | |
| 176 | for u in m.values[alloc_id].uses { |
| 177 | if u >= m.values.len { |
| 178 | continue |
| 179 | } |
| 180 | user := m.values[u] |
| 181 | if user.kind != .instruction { |
| 182 | return false |
| 183 | } |
| 184 | instr := m.instrs[user.index] |
| 185 | match instr.op { |
| 186 | .load { |
| 187 | if instr.operands.len == 0 || instr.operands[0] != alloc_id { |
| 188 | return false |
| 189 | } |
| 190 | } |
| 191 | .store { |
| 192 | if instr.operands.len < 2 || instr.operands[1] != alloc_id { |
| 193 | return false |
| 194 | } |
| 195 | } |
| 196 | else { |
| 197 | return false |
| 198 | } |
| 199 | } |
| 200 | } |
| 201 | return true |
| 202 | } |
| 203 | |
| 204 | // compute_dominance_frontier_flat supports compute dominance frontier flat handling for optimize. |
| 205 | fn compute_dominance_frontier_flat(m &ssa.Module, func_idx int, dom &DomInfo, cfg &CfgData, mut df [][]int, mut df_blocks []int) { |
| 206 | func := m.funcs[func_idx] |
| 207 | for blk_id in func.blocks { |
| 208 | if blk_id < 0 || blk_id >= m.blocks.len { |
| 209 | continue |
| 210 | } |
| 211 | num_preds := cfg.preds[blk_id].len |
| 212 | if num_preds >= 2 { |
| 213 | for pi in 0 .. num_preds { |
| 214 | mut runner := cfg.preds[blk_id][pi] |
| 215 | idom := dom.idom[blk_id] |
| 216 | for runner != -1 && runner != idom { |
| 217 | if runner < 0 || runner >= m.blocks.len { |
| 218 | break |
| 219 | } |
| 220 | if blk_id !in df[runner] { |
| 221 | array2d_append(mut df, runner, blk_id) |
| 222 | if runner !in df_blocks { |
| 223 | df_blocks << runner |
| 224 | } |
| 225 | } |
| 226 | if runner == dom.idom[runner] { |
| 227 | break |
| 228 | } |
| 229 | runner = dom.idom[runner] |
| 230 | } |
| 231 | } |
| 232 | } |
| 233 | } |
| 234 | } |
| 235 | |
| 236 | // RenameFrame represents rename frame data used by optimize. |
| 237 | struct RenameFrame { |
| 238 | mut: |
| 239 | blk_id int |
| 240 | child_idx int |
| 241 | pushed_allocs []int |
| 242 | processed bool |
| 243 | } |
| 244 | |
| 245 | // rename_blocks supports rename blocks handling for optimize. |
| 246 | fn rename_blocks(mut m ssa.Module, root_blk int, mut ctx Mem2RegCtx, dom DomInfo, cfg &CfgData) { |
| 247 | mut work := []RenameFrame{} |
| 248 | work << RenameFrame{ |
| 249 | blk_id: root_blk |
| 250 | } |
| 251 | mut visited := []bool{len: m.blocks.len} |
| 252 | |
| 253 | for work.len > 0 { |
| 254 | fi := work.len - 1 |
| 255 | blk_id := work[fi].blk_id |
| 256 | |
| 257 | if !work[fi].processed { |
| 258 | if blk_id >= 0 && blk_id < visited.len && visited[blk_id] { |
| 259 | work.pop() |
| 260 | continue |
| 261 | } |
| 262 | if blk_id >= 0 && blk_id < visited.len { |
| 263 | visited[blk_id] = true |
| 264 | } |
| 265 | mut frame2 := work[fi] |
| 266 | frame2.processed = true |
| 267 | work[fi] = frame2 |
| 268 | |
| 269 | // 1. Push phis defined in this block. |
| 270 | if blk_id < ctx.phi_placements.len && ctx.phi_placements[blk_id].len > 0 { |
| 271 | n_phi := ctx.phi_placements[blk_id].len |
| 272 | for pai in 0 .. n_phi { |
| 273 | alloc_id := ctx.phi_placements[blk_id][pai] |
| 274 | if pai < ctx.phi_vals[blk_id].len { |
| 275 | phi_val_id := ctx.phi_vals[blk_id][pai] |
| 276 | mut new_stack := ctx.stacks[alloc_id].clone() |
| 277 | new_stack << phi_val_id |
| 278 | ctx.stacks[alloc_id] = new_stack |
| 279 | mut wf := work[fi] |
| 280 | wf.pushed_allocs << alloc_id |
| 281 | work[fi] = wf |
| 282 | } |
| 283 | } |
| 284 | } |
| 285 | |
| 286 | // 2. Process instructions: rewrite loads, record store values, nop them. |
| 287 | mut instrs_to_nop := []int{} |
| 288 | blk2 := m.blocks[blk_id] |
| 289 | for val_id in blk2.instrs { |
| 290 | val := m.values[val_id] |
| 291 | if val.kind != .instruction { |
| 292 | continue |
| 293 | } |
| 294 | instr := m.instrs[val.index] |
| 295 | if instr.op == .store { |
| 296 | ptr := instr.operands[1] |
| 297 | if ptr < ctx.is_promotable.len && ctx.is_promotable[ptr] { |
| 298 | mut new_stack := ctx.stacks[ptr].clone() |
| 299 | new_stack << instr.operands[0] |
| 300 | ctx.stacks[ptr] = new_stack |
| 301 | mut wf := work[fi] |
| 302 | wf.pushed_allocs << ptr |
| 303 | work[fi] = wf |
| 304 | instrs_to_nop << val_id |
| 305 | } |
| 306 | } else if instr.op == .load { |
| 307 | ptr := instr.operands[0] |
| 308 | if ptr < ctx.is_promotable.len && ctx.is_promotable[ptr] { |
| 309 | stack := ctx.stacks[ptr] |
| 310 | mut repl := 0 |
| 311 | if stack.len > 0 { |
| 312 | repl = stack.last() |
| 313 | } else { |
| 314 | repl = m.get_or_add_const(val.typ, 'undef') |
| 315 | } |
| 316 | m.replace_uses(val_id, repl) |
| 317 | instrs_to_nop << val_id |
| 318 | } |
| 319 | } else if instr.op == .alloca { |
| 320 | if val_id < ctx.is_promotable.len && ctx.is_promotable[val_id] { |
| 321 | instrs_to_nop << val_id |
| 322 | } |
| 323 | } |
| 324 | } |
| 325 | for vid in instrs_to_nop { |
| 326 | vid_val := m.values[vid] |
| 327 | mut nop_instr := m.instrs[vid_val.index] |
| 328 | nop_instr.op = .bitcast |
| 329 | nop_instr.operands = [] |
| 330 | m.instrs[vid_val.index] = nop_instr |
| 331 | } |
| 332 | |
| 333 | // 3. Fill phi operands in CFG successors (raw block id as predecessor). |
| 334 | for succ_id in cfg.succs[blk_id] { |
| 335 | if succ_id < ctx.phi_placements.len && ctx.phi_placements[succ_id].len > 0 { |
| 336 | n_succ_phi := ctx.phi_placements[succ_id].len |
| 337 | for spai in 0 .. n_succ_phi { |
| 338 | alloc_id := ctx.phi_placements[succ_id][spai] |
| 339 | if spai < ctx.phi_vals[succ_id].len { |
| 340 | vid := ctx.phi_vals[succ_id][spai] |
| 341 | phi_v := m.values[vid] |
| 342 | phi_val := if ctx.stacks[alloc_id].len > 0 { |
| 343 | ctx.stacks[alloc_id].last() |
| 344 | } else { |
| 345 | alloc_v := m.values[alloc_id] |
| 346 | alloc_typ := m.type_store.types[alloc_v.typ] |
| 347 | m.get_or_add_const(alloc_typ.elem_type, 'undef') |
| 348 | } |
| 349 | m.append_phi_operands(phi_v.index, phi_val, blk_id) |
| 350 | } |
| 351 | } |
| 352 | } |
| 353 | } |
| 354 | } |
| 355 | |
| 356 | // 4. Recurse into dominator-tree children. |
| 357 | n_children := dom.dom_tree[blk_id].len |
| 358 | child_idx := work[fi].child_idx |
| 359 | if child_idx < n_children { |
| 360 | child := dom.dom_tree[blk_id][child_idx] |
| 361 | mut frame := work[fi] |
| 362 | frame.child_idx++ |
| 363 | work[fi] = frame |
| 364 | work << RenameFrame{ |
| 365 | blk_id: child |
| 366 | } |
| 367 | } else { |
| 368 | // 5. Pop reaching definitions pushed in this block. |
| 369 | pushed := work[fi].pushed_allocs.clone() |
| 370 | work.pop() |
| 371 | for i := pushed.len - 1; i >= 0; i-- { |
| 372 | mut s := ctx.stacks[pushed[i]].clone() |
| 373 | s.pop() |
| 374 | ctx.stacks[pushed[i]] = s |
| 375 | } |
| 376 | } |
| 377 | } |
| 378 | } |
| 379 | |