vxx2 / vlib / v3 / transform / transform_d_parallel.v
143 lines · 128 sloc · 5.37 KB · 2b81918e62db9ab38787518603a7f93bee80868c
Raw
1module transform
2
3import runtime
4import v3.flat
5
6// Parallel function-body transform. Compiled only when the build defines
7// `parallel` (i.e. `-d parallel`), matching the parallel C codegen path. Each
8// worker transforms a disjoint set of closure-free functions on its own cloned
9// AST + forked TypeChecker, and the master folds the results back together in a
10// fixed order so the build stays deterministic.
11
12const min_parallel_transform_items = 256
13const max_parallel_transform_jobs = 2
14
15$if !windows {
16 // TransformChunkArgs is the payload handed to each worker thread.
17 struct TransformChunkArgs {
18 worker voidptr // &Transformer
19 items_ptr voidptr // &[]FnWorkItem
20 }
21
22 // C.pthread_t declares C pthread t data used by transform.
23 @[typedef]
24 struct C.pthread_t {}
25
26 // C.pthread_create declares the C pthread_create symbol used by transform.
27 fn C.pthread_create(thread &C.pthread_t, attr voidptr, start_routine fn (voidptr) voidptr, arg voidptr) int
28
29 // C.pthread_join declares the C pthread_join symbol used by transform.
30 fn C.pthread_join(thread C.pthread_t, retval voidptr) int
31
32 // C.pthread_attr_init declares the C pthread_attr_init symbol used by transform.
33 fn C.pthread_attr_init(attr voidptr) int
34
35 // C.pthread_attr_setstacksize declares the C pthread_attr_setstacksize symbol used by transform.
36 fn C.pthread_attr_setstacksize(attr voidptr, stacksize usize) int
37
38 // C.pthread_attr_destroy declares the C pthread_attr_destroy symbol used by transform.
39 fn C.pthread_attr_destroy(attr voidptr) int
40
41 // transform_chunk_thread runs one worker's chunk of function bodies.
42 fn transform_chunk_thread(arg voidptr) voidptr {
43 a := unsafe { &TransformChunkArgs(arg) }
44 mut w := unsafe { &Transformer(a.worker) }
45 items := unsafe { &[]FnWorkItem(a.items_ptr) }
46 w.transform_pure_items_serial(*items)
47 return unsafe { nil }
48 }
49}
50
51// run_parallel_transform transforms the closure-free function bodies across
52// threads when there is enough work, otherwise serially. Returns whether threads
53// were actually used.
54fn (mut t Transformer) run_parallel_transform(items []FnWorkItem, base_nodes int, base_children int) bool {
55 $if windows {
56 t.transform_pure_items_serial(items)
57 return false
58 } $else {
59 n_jobs := transform_job_count(runtime.nr_jobs(), items.len)
60 if items.len < min_parallel_transform_items || n_jobs <= 1 {
61 t.transform_pure_items_serial(items)
62 return false
63 }
64 mut chunks := split_work_items(items, n_jobs)
65 chunk_count := chunks.len
66
67 // chunk[0] is transformed by the master on this thread, directly against the
68 // master AST — no clone. Only chunks[1..] get helper threads, each with a
69 // private AST clone + forked TypeChecker. This removes one full base-AST clone
70 // from the peak (each clone is ~one nodes-array; under -gc none they are never
71 // freed, so they also inflate the later cgen peak) and keeps the master thread,
72 // which would otherwise block in join, doing useful work.
73 thread_count := chunk_count - 1
74 mut workers := []voidptr{cap: thread_count}
75 mut worker_asts := []voidptr{cap: thread_count}
76 for _ in 0 .. thread_count {
77 wast := t.clone_ast_base(base_nodes, base_children)
78 worker_asts << voidptr(wast)
79 wtc := t.tc.fork_for_parallel_transform(wast)
80 ww := t.fork_worker(wast, wtc)
81 workers << voidptr(ww)
82 }
83 mut args := []TransformChunkArgs{cap: thread_count}
84 for ci in 0 .. thread_count {
85 args << TransformChunkArgs{
86 worker: workers[ci]
87 items_ptr: unsafe { voidptr(&chunks[ci + 1]) }
88 }
89 }
90
91 mut thread_ids := []C.pthread_t{len: thread_count}
92 attr_buf := [64]u8{}
93 attr := unsafe { voidptr(&attr_buf[0]) }
94 C.pthread_attr_init(attr)
95 // Transform recurses deeply on large expressions; give workers a roomy stack.
96 C.pthread_attr_setstacksize(attr, 64 * 1024 * 1024)
97 for ci in 0 .. thread_count {
98 C.pthread_create(unsafe { &thread_ids[ci] }, attr, transform_chunk_thread,
99 unsafe { voidptr(&args[ci]) })
100 }
101 C.pthread_attr_destroy(attr)
102 // Master transforms chunk[0] in place while the helper threads run. It only
103 // touches its own functions' nodes and the master AST/TypeChecker, all disjoint
104 // from the workers' clones, and never writes the shared (read-only) type tables.
105 // Reset temp_counter to 0 like a freshly forked worker (transform temps are
106 // function-local, so this is collision-free and matches the all-workers output).
107 t.temp_counter = 0
108 t.transform_pure_items_serial(chunks[0])
109 for ci in 0 .. thread_count {
110 C.pthread_join(thread_ids[ci], unsafe { nil })
111 }
112 // Merge helper results in fixed chunk order (chunk[0] is already in place), so
113 // node numbering stays deterministic for reproducible builds.
114 for ci in 0 .. thread_count {
115 ww := unsafe { &Transformer(workers[ci]) }
116 t.merge_worker_used_fns(ww)
117 t.merge_worker(ww, chunks[ci + 1], base_nodes, base_children)
118 // The worker's cloned base AST is no longer needed after merge.
119 unsafe {
120 mut wast := &flat.FlatAst(worker_asts[ci])
121 wast.nodes.free()
122 wast.children.free()
123 }
124 }
125 return true
126 }
127}
128
129// transform_job_count caps the worker count by both the runtime job count and a
130// fixed ceiling (each worker clones the base AST, so more workers cost more memory).
131fn transform_job_count(n_runtime_jobs int, n_items int) int {
132 if n_runtime_jobs <= 0 || n_items <= 0 {
133 return 0
134 }
135 mut n := n_runtime_jobs
136 if n > max_parallel_transform_jobs {
137 n = max_parallel_transform_jobs
138 }
139 if n > n_items {
140 n = n_items
141 }
142 return n
143}
144