medvednikov

/

vqPublic
0 commits21 issue12 pull requests0 contributorsDiscussionsProjectsCI

cgen: use .incbin assembler directives for $embed_file, eliminate the 5MB limit #13

OpenMagicFun1241wants to mergepr/27438intomaster· last Jun 12
7 files changed+293-34
vlib/v/ast/ast.v+4-3

@@ -1092,9 +1092,10 @@ pub mut:

10921092 rpath string // used in the source code, as an ID/key to the embed

10931093 apath string // absolute path during compilation to the resource

10941094 // these are set by gen_embed_file_init in v/gen/c/embed

1095- is_compressed bool

1096- bytes []u8

1097- len int

1095+ is_compressed bool

1096+ bytes []u8

1097+ len int

1098+ compressed_temp_path string // path to compressed temp file (kept for .incbin)

10981099}

10991100

11001101// TemplateLineInfo maps a generated code line to the original template location

vlib/v/builder/builder.v+3-0

@@ -59,6 +59,9 @@ pub mut:

5959 str_args string // for parallel_cc mode only, to know which cc args to use (like -I etc)

6060 last_cc_cmd string // the most recently executed C compiler command; reused to regenerate a #line annotated report

6161 disable_flto bool

62+ embedded_asm map[string]string // filename → .S file content (from GenOutput)

63+ embedded_o_files []string // compiled embed .o file paths

64+ embedded_temp_files []string // temp .bin files for cleanup

6265}

6366

6467struct CFunctionCallCollector {

vlib/v/builder/cbuilder/cbuilder.v+26-0

@@ -71,6 +71,15 @@ pub fn build_c(mut b builder.Builder, v_files []string, out_file string) {

7171 b.stats_lines = output2.count(it == `\n`) + 1

7272 b.stats_bytes = output2.len

7373 }

74+ // Compile embedded .S files to .o and add to Builder

75+ // (skip if parallel_cc already compiled them in gen_c)

76+ if b.embedded_asm.len > 0 && b.embedded_o_files.len == 0 {

77+ b.compile_embedded_asm_files(b.embedded_asm)

78+ }

79+ // Schedule compressed temp files for cleanup

80+ for temp_file in b.embedded_temp_files {

81+ b.pref.cleanup_files << temp_file

82+ }

7483}

7584

7685pub fn gen_c(mut b builder.Builder, v_files []string) strings.Builder {

@@ -85,6 +94,23 @@ pub fn gen_c(mut b builder.Builder, v_files []string) strings.Builder {

8594 result := c.gen(b.parsed_files, mut b.table, b.pref)

8695 util.timing_measure('C GEN')

8796

97+ // Copy embed data from GenOutput onto Builder

98+ if result.embedded_asm.len > 0 {

99+ b.embedded_asm = result.embedded_asm.clone()

100+ }

101+ if result.embedded_temp_files.len > 0 {

102+ b.embedded_temp_files = result.embedded_temp_files

103+ }

104+

105+ // When parallel_cc is active, we must compile embed .S files now

106+ // because parallel_cc runs inside gen_c() before build_c() gets a chance.

107+ if b.pref.parallel_cc && b.embedded_asm.len > 0 {

108+ b.compile_embedded_asm_files(b.embedded_asm)

109+ for temp_file in b.embedded_temp_files {

110+ b.pref.cleanup_files << temp_file

111+ }

112+ }

113+

88114 if b.pref.parallel_cc {

89115 b.cc() // Call it just to gen b.str_args

90116 util.timing_start('Parallel C compilation')

vlib/v/builder/cbuilder/parallel_cc.v+3-0

@@ -154,6 +154,9 @@ fn parallel_cc(mut b builder.Builder, result c.GenOutput) ! {

154154 fo := f.replace('.c', '.o')

155155 ofiles << os.quoted_path(fo)

156156 }

157+ // Note: embedded .o files are NOT added to ofiles here. They are already

158+ // included via ccoptions.o_args (set up in cc() during the parallel_cc

159+ // str_args generation), which get_linker_args() picks up.

157160 obj_files := ofiles.join(' ')

158161

159162 alink := [

vlib/v/builder/cc.v+105-0

@@ -1592,6 +1592,13 @@ pub fn (mut v Builder) cc() {

15921592 v.build_thirdparty_obj_files()

15931593 v.setup_output_name()

15941594

1595+ // Add embedded data object files to the link step

1596+ if v.embedded_o_files.len > 0 {

1597+ for obj_file in v.embedded_o_files {

1598+ v.ccoptions.o_args << os.quoted_path(obj_file)

1599+ }

1600+ }

1601+

15951602 if v.pref.os != .windows && ccompiler.contains('++') {

15961603 cpp_atomic_h_path := '${@VEXEROOT}/thirdparty/stdatomic/nix/cpp/atomic.h'

15971604 if !os.exists(cpp_atomic_h_path) {

@@ -1974,6 +1981,94 @@ fn linux_cross_target_for_arch(arch pref.Arch) !LinuxCrossTarget {

19741981 }

19751982}

19761983

1984+fn find_system_assembler() ?string {

1985+ // Raw `as` is excluded: it doesn't preprocess #if directives in .S files

1986+ // and has no -c flag. We need a compiler (clang/gcc/cc) that can do both.

1987+ for candidate in ['clang', 'gcc', 'cc'] {

1988+ if path := os.find_abs_path_of_executable(candidate) {

1989+ return path

1990+ }

1991+ }

1992+ return none

1993+}

1994+

1995+pub fn (mut b Builder) compile_embedded_asm_files(asm_files map[string]string) {

1996+ if asm_files.len == 0 {

1997+ return

1998+ }

1999+ // Safety checks: these should never trigger (should_use_incbin_embed guards

2000+ // at codegen), but are kept as defense-in-depth.

2001+ if b.pref.os in [.wasm32, .wasm32_emscripten, .wasm32_wasi] {

2002+ verror('embedded file .incbin is not supported on wasm targets')

2003+ }

2004+ if b.pref.build_mode == .build_module {

2005+ verror('embedded file .incbin is not supported in build_module mode (separate .o cannot be merged into cached module)')

2006+ }

2007+ vtmp := os.vtmp_dir()

2008+

2009+ mut asm_cc := b.pref.ccompiler

2010+ if b.pref.ccompiler_type == .tinyc {

2011+ asm_cc = find_system_assembler() or {

2012+ verror('no assembler found for embedded files (TCC cannot assemble .S files); install clang, gcc, cc, or as')

2013+ }

2014+ }

2015+

2016+ for asm_filename, asm_content in asm_files {

2017+ asm_path := os.join_path(vtmp, asm_filename)

2018+ os.write_file(asm_path, asm_content) or {

2019+ verror('could not write embed assembly file: ${err}')

2020+ }

2021+

2022+ obj_path := asm_path.replace('.S', '.o')

2023+ mut asm_args := []string{}

2024+ asm_args << '-c'

2025+

2026+ // Cross-compilation target flags

2027+ // When the target OS/arch differs from the host, the .S file must be

2028+ // assembled for the target, not the host. Without -target, the assembler

2029+ // produces a host-arch .o that fails to link into the target binary.

2030+ if b.pref.os == .linux {

2031+ host_arch := pref.get_host_arch()

2032+ if pref.get_host_os() != .linux || host_arch != b.pref.arch {

2033+ cross_target := linux_cross_target_for_arch(b.pref.arch) or {

2034+ verror('failed to determine linux cross target for embedded assembly: ${err.msg()}')

2035+ return

2036+ }

2037+ asm_args << ['-target', cross_target.triple]

2038+ }

2039+ } else if b.pref.os == .freebsd && pref.get_host_os() != .freebsd {

2040+ asm_args << ['-target', 'x86_64-unknown-freebsd14.0']

2041+ }

2042+

2043+ // macOS architecture

2044+ if b.pref.os == .macos {

2045+ arch_name := darwin_target_arch_name(b.pref.arch)

2046+ if arch_name != '' {

2047+ asm_args << ['-arch', arch_name]

2048+ }

2049+ }

2050+

2051+ asm_args << [os.quoted_path(asm_path), '-o', os.quoted_path(obj_path)]

2052+

2053+ cmd := '${b.quote_compiler_name(asm_cc)} ${asm_args.join(' ')}'

2054+ if b.pref.show_cc {

2055+ println('> Embedded file assembly: ${cmd}')

2056+ }

2057+ res := os.execute(cmd)

2058+ if res.exit_code != 0 {

2059+ verror('Failed to compile embedded file assembly ${asm_filename}:\n${res.output}')

2060+ }

2061+

2062+ b.embedded_o_files << obj_path

2063+

2064+ // Schedule cleanup

2065+ if !b.ccoptions.debug_mode {

2066+ b.pref.cleanup_files << asm_path

2067+ b.pref.cleanup_files << obj_path

2068+ }

2069+ }

2070+}

2071+

19772072fn (mut b Builder) cc_linux_cross() {

19782073 linux_cross_target := linux_cross_target_for_arch(b.pref.arch) or {

19792074 verror(err.msg())

@@ -2099,6 +2194,11 @@ fn (mut b Builder) cc_linux_cross() {

20992194 for eobj in extra_objs {

21002195 linker_args << os.quoted_path(eobj)

21012196 }

2197+ if b.embedded_o_files.len > 0 {

2198+ for embed_obj in b.embedded_o_files {

2199+ linker_args << os.quoted_path(embed_obj)

2200+ }

2201+ }

21022202 // User-defined libraries (e.g. `-lpq` from db.pg) and extra object files

21032203 // must come before the system libraries they depend on. ld.lld resolves

21042204 // references left-to-right, so libpq.a needs to be encountered before

@@ -2195,6 +2295,11 @@ fn (mut b Builder) cc_freebsd_cross() {

21952295 os.quoted_path(obj_file),

21962296 os.quoted_path('${sysroot}/usr/lib/crtn.o'),

21972297 ]

2298+ if b.embedded_o_files.len > 0 {

2299+ for embed_obj in b.embedded_o_files {

2300+ linker_args << os.quoted_path(embed_obj)

2301+ }

2302+ }

21982303 linker_args << '-lc' // needed for fwrite, strlen etc

21992304 linker_args << '-lexecinfo' // needed for backtrace

22002305 linker_args << cflags.c_options_only_object_files() // support custom module defined linker flags

vlib/v/gen/c/cgen.v+29-14

@@ -7,6 +7,7 @@ import os

77import term

88import strings

99import hash.fnv1a

10+import rand

1011import v.ast

1112import v.pref

1213import v.token

@@ -243,6 +244,9 @@ mut:

243244 hotcode_fn_names []string

244245 hotcode_fpaths []string

245246 embedded_files []ast.EmbeddedFile

247+ embedded_asm map[string]string // generated .S file content (filename → content)

248+ embedded_temp_files []string // compressed .bin temp files for cleanup

249+ embed_build_id string // random suffix for .S/.o filenames to avoid parallel-build collisions

246250 sql_i int

247251 sql_stmt_name string

248252 sql_bind_name string

@@ -336,12 +340,15 @@ mut:

336340@[heap]

337341pub struct GenOutput {

338342pub:

339- header string // produced output for out.h (-parallel-cc)

340- res_builder strings.Builder // produced output (complete)

341- out_str string // produced output from g.out

342- out0_str string // helpers output (auto fns, dump fns) for out_0.c (-parallel-cc)

343- extern_str string // extern chunk for (-parallel-cc)

344- out_fn_start_pos []int // fn decl positions

343+ header string

344+ res_builder strings.Builder // produced output (complete)

345+ out_str string // produced output from g.out

346+ out0_str string // helpers output (auto fns, dump fns) for out_0.c (-parallel-cc)

347+ extern_str string // extern chunk for (-parallel-cc)

348+ out_fn_start_pos []int // fn decl positions

349+ embedded_asm map[string]string // filename → .S file content

350+ embedded_temp_files []string // temp .bin files for cleanup

351+ embed_build_id string // random suffix for .S/.o filenames to avoid parallel-build collisions

345352}

346353

347354pub fn gen(files []&ast.File, mut table ast.Table, pref_ &pref.Preferences) GenOutput {

@@ -416,6 +423,7 @@ pub fn gen(files []&ast.File, mut table ast.Table, pref_ &pref.Preferences) GenO

416423 use_segfault_handler: pref_.should_use_segfault_handler()

417424 static_modifier: if pref_.parallel_cc || pref_.is_o { 'static ' } else { '' }

418425 static_non_parallel: if !pref_.parallel_cc { 'static ' } else { '' }

426+ embed_build_id: rand.ulid()

419427 has_reflection: 'v.reflection' in table.modules

420428 has_debugger: 'v.debug' in table.modules

421429 reflection_strings: &reflection_strings

@@ -989,12 +997,15 @@ pub fn gen(files []&ast.File, mut table ast.Table, pref_ &pref.Preferences) GenO

989997 unsafe { g.free_builders() }

990998

991999 return GenOutput{

992- header: header

993- res_builder: b

994- out_str: out_str

995- out0_str: shelpers

996- extern_str: extern_out_str

997- out_fn_start_pos: out_fn_start_pos

1000+ header: header

1001+ res_builder: b

1002+ out_str: out_str

1003+ out0_str: shelpers

1004+ extern_str: extern_out_str

1005+ out_fn_start_pos: out_fn_start_pos

1006+ embedded_asm: g.embedded_asm

1007+ embedded_temp_files: g.embedded_temp_files

1008+ embed_build_id: g.embed_build_id

9981009 }

9991010}

10001011

@@ -1046,6 +1057,7 @@ fn cgen_process_one_file_cb(mut p pool.PoolProcessor, idx int, wid int) voidptr

10461057 module_built: global_g.module_built

10471058 static_non_parallel: global_g.static_non_parallel

10481059 static_modifier: global_g.static_modifier

1060+ embed_build_id: global_g.embed_build_id

10491061 timers: util.new_timers(

10501062 should_print: global_g.timers_should_print

10511063 label: 'cgen_process_one_file_cb idx: ${idx}, wid: ${wid}'

@@ -1149,9 +1161,12 @@ pub fn (mut g Gen) gen_file() {

11491161 // write them to corresponding sections

11501162 g.gen_hash_stmts_in_top()

11511163

1152- // Transfer embedded files

1164+ // Transfer embedded files, deduplicating by hash.

1165+ // Two embeds of the same file with compression get different compressed_temp_path

1166+ // values (random ULIDs), so struct equality would fail to deduplicate them,

1167+ // causing duplicate case labels in generated C. hash() excludes temp paths.

11531168 for path in g.file.embedded_files {

1154- if path !in g.embedded_files {

1169+ if g.embedded_files.all(it.hash() != path.hash()) {

11551170 g.embedded_files << path

11561171 }

11571172 }

vlib/v/gen/c/embed.v+123-17

@@ -2,16 +2,64 @@ module c

22

33import os

44import rand

5+import strings

56import v.ast

67import v.pref

78

8-fn (mut g Gen) should_really_embed_file() bool {

9+fn (g &Gen) should_really_embed_file() bool {

910 if 'embed_only_metadata' in g.pref.compile_defines {

1011 return false

1112 }

1213 return true

1314}

1415

16+fn (g &Gen) should_use_incbin_embed() bool {

17+ if g.pref.out_name.ends_with('.c') || g.pref.generate_c_project != '' || g.pref.should_output_to_stdout() {

18+ return false

19+ }

20+ if g.pref.is_o {

21+ return false

22+ }

23+ // build_module (-usecache) produces a standalone .o per module.

24+ // Embed .o files are separate and would not be merged into the module .o,

25+ // causing unresolved _v_embed_blob_* symbols at final link time.

26+ if g.pref.build_mode == .build_module {

27+ return false

28+ }

29+ if !g.should_really_embed_file() {

30+ return false

31+ }

32+ if g.pref.ccompiler_type == .msvc {

33+ return false

34+ }

35+ if g.pref.os == .windows && pref.get_host_os() != .windows {

36+ return false

37+ }

38+ // wasm targets use a different object format; our .S files have ELF/Mach-O

39+ // directives and would produce host-arch .o files that can't link into wasm.

40+ if g.pref.os in [.wasm32, .wasm32_emscripten, .wasm32_wasi] {

41+ return false

42+ }

43+ if g.pref.ccompiler_type in [.gcc, .clang, .mingw] {

44+ return true

45+ }

46+ if g.pref.ccompiler_type == .tinyc {

47+ // TCC can't assemble .S files itself, so we need a system compiler

48+ // that can handle preprocessor directives in .S files and accepts -c.

49+ // Raw `as` is excluded: it doesn't preprocess #if directives and has no -c flag.

50+ // If none is found, fall back to C hex arrays (slower compile, but correct).

51+ if _ := os.find_abs_path_of_executable('clang') {

52+ return true

53+ } else if _ := os.find_abs_path_of_executable('gcc') {

54+ return true

55+ } else if _ := os.find_abs_path_of_executable('cc') {

56+ return true

57+ }

58+ return false

59+ }

60+ return false

61+}

62+

1563fn (mut g Gen) handle_embedded_files_finish() {

1664 if g.embedded_files.len > 0 {

1765 if g.should_really_embed_file() {

@@ -58,9 +106,13 @@ fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) {

58106 }

59107 []u8{}

60108 }

61- os.rm(cache_path) or {} // clean up

62109 node.embed_file.is_compressed = compressed_bytes.len > 0

63110 && compressed_bytes.len < file_bytes.len

111+ if g.should_use_incbin_embed() && node.embed_file.is_compressed {

112+ node.embed_file.compressed_temp_path = cache_path

113+ } else {

114+ os.rm(cache_path) or {}

115+ }

64116 node.embed_file.bytes = if node.embed_file.is_compressed {

65117 compressed_bytes

66118 } else {

@@ -69,7 +121,9 @@ fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) {

69121 }

70122 }

71123 if node.embed_file.bytes.len > 5242880 {

72- eprintln('embedding of files >= ~5MB is currently not well supported')

124+ if !g.should_use_incbin_embed() {

125+ eprintln('embedding of files >= ~5MB is currently not well supported')

126+ }

73127 }

74128 node.embed_file.len = file_bytes.len

75129 }

@@ -122,7 +176,9 @@ fn (mut g Gen) gen_embedded_metadata() {

122176 } else {

123177 file_size := os.file_size(emfile.apath)

124178 if file_size > 5242880 {

125- eprintln('Warning: embedding of files >= ~5MB is currently not supported')

179+ if !g.should_use_incbin_embed() {

180+ eprintln('Warning: embedding of files >= ~5MB is currently not supported')

181+ }

126182 }

127183 g.embedded_data.writeln('\t\t\tres.len = ${file_size};')

128184 }

@@ -146,27 +202,77 @@ fn (mut g Gen) gen_embedded_data() {

146202 // Declare the index before any generated helper references it, to keep MSVC happy.

147203 index_len := g.embedded_files.len + 1

148204 g.embedded_data.writeln('static const v__embed_file__EmbedFileIndexEntry _v_embed_file_index[${index_len}];')

149- for i, emfile in g.embedded_files {

150- g.embedded_data.write_string('static const unsigned char _v_embed_blob_${i}[${emfile.bytes.len}] = {\n ')

151- for j := 0; j < emfile.bytes.len; j++ {

152- b := emfile.bytes[j].hex()

153- if j < emfile.bytes.len - 1 {

154- g.embedded_data.write_string('0x${b},')

155- } else {

156- g.embedded_data.write_string('0x${b}')

157- }

158- if 0 == ((j + 1) % 16) {

159- g.embedded_data.write_string('\n ')

205+ if g.should_use_incbin_embed() {

206+ // .incbin path: extern declarations + .S file content

207+ for emfile in g.embedded_files {

208+ ef_hash := emfile.hash()

209+ g.embedded_data.writeln('extern const unsigned char _v_embed_blob_${ef_hash}[];')

210+ g.gen_embedded_asm_file(emfile)

211+ }

212+ } else {

213+ // C hex array path: current behavior

214+ for i, emfile in g.embedded_files {

215+ g.embedded_data.write_string('static const unsigned char _v_embed_blob_${i}[${emfile.bytes.len}] = {\n ')

216+ for j := 0; j < emfile.bytes.len; j++ {

217+ b := emfile.bytes[j].hex()

218+ if j < emfile.bytes.len - 1 {

219+ g.embedded_data.write_string('0x${b},')

220+ } else {

221+ g.embedded_data.write_string('0x${b}')

222+ }

223+ if 0 == ((j + 1) % 16) {

224+ g.embedded_data.write_string('\n ')

225+ }

160226 }

227+ g.embedded_data.writeln('\n};')

161228 }

162- g.embedded_data.writeln('\n};')

163229 }

164230 g.embedded_data.writeln('')

165231 g.embedded_data.writeln('static const v__embed_file__EmbedFileIndexEntry _v_embed_file_index[${index_len}] = {')

166232 for i, emfile in g.embedded_files {

167- g.embedded_data.writeln('\t{${i}, { .str=(byteptr)("${cestring(emfile.rpath)}"), .len=${emfile.rpath.len}, .is_lit=1 }, { .str=(byteptr)("${cestring(emfile.compression_type)}"), .len=${emfile.compression_type.len}, .is_lit=1 }, (byteptr)_v_embed_blob_${i}},')

233+ blob_name := if g.should_use_incbin_embed() { emfile.hash().str() } else { i.str() }

234+ g.embedded_data.writeln('\t{${i}, { .str=(byteptr)("${cestring(emfile.rpath)}"), .len=${emfile.rpath.len}, .is_lit=1 }, { .str=(byteptr)("${cestring(emfile.compression_type)}"), .len=${emfile.compression_type.len}, .is_lit=1 }, (byteptr)_v_embed_blob_${blob_name}},')

168235 }

169236 g.embedded_data.writeln('\t{-1, { .str=(byteptr)(""), .len=0, .is_lit=1 }, { .str=(byteptr)(""), .len=0, .is_lit=1 }, NULL}')

170237 g.embedded_data.writeln('};')

171238 // see vlib/v/embed_file/embed_file.v, find_index_entry_by_id/2 and find_index_entry_by_path/2

172239}

240+

241+fn (mut g Gen) gen_embedded_asm_file(emfile ast.EmbeddedFile) {

242+ mut incbin_path := emfile.apath

243+ if emfile.is_compressed {

244+ incbin_path = emfile.compressed_temp_path

245+ }

246+

247+ ef_hash := emfile.hash()

248+ // Include a per-build random suffix in the filename to prevent collisions

249+ // when two V processes embed the same file concurrently (parallel `v test`).

250+ // The symbol names stay deterministic (based on hash only).

251+ asm_filename := '_v_embed_blob_${ef_hash}_${g.embed_build_id}.S'

252+ mut sb := strings.new_builder(512)

253+ sb.writeln('// V embedded file: ${emfile.rpath} (hash ${ef_hash}, uncompressed size ${emfile.len})')

254+ sb.writeln('#if defined(__APPLE__)')

255+ sb.writeln(' .section __TEXT,__const')

256+ sb.writeln(' .globl __v_embed_blob_${ef_hash}')

257+ sb.writeln('__v_embed_blob_${ef_hash}:')

258+ sb.writeln('#elif defined(_WIN32)')

259+ sb.writeln(' .section .rdata')

260+ sb.writeln(' .globl _v_embed_blob_${ef_hash}')

261+ sb.writeln('_v_embed_blob_${ef_hash}:')

262+ sb.writeln('#else')

263+ sb.writeln(' .section .rodata')

264+ sb.writeln(' .globl _v_embed_blob_${ef_hash}')

265+ sb.writeln(' .type _v_embed_blob_${ef_hash}, @object')

266+ sb.writeln('_v_embed_blob_${ef_hash}:')

267+ sb.writeln('#endif')

268+ sb.writeln(' .incbin "${cestring(incbin_path)}"')

269+ sb.writeln('#if !defined(__APPLE__) && !defined(_WIN32)')

270+ sb.writeln(' .size _v_embed_blob_${ef_hash}, ${emfile.bytes.len}')

271+ sb.writeln('#endif')

272+

273+ g.embedded_asm[asm_filename] = sb.str()

274+

275+ if emfile.is_compressed {

276+ g.embedded_temp_files << emfile.compressed_temp_path

277+ }

278+}