From ace63594bf4d6a2b532fd480a53ce6d4e706ef7e Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sat, 4 Dec 2021 18:43:19 +0100 Subject: [PATCH] all: support `$embed_file('embed.vv', .zlib)` (#12654) --- cmd/tools/vcompress.v | 44 +++++++++ cmd/v/v.v | 1 + doc/docs.md | 11 +++ vlib/v/ast/ast.v | 14 ++- vlib/v/checker/checker.v | 26 +++-- .../tests/embed_unknown_compress_type.out | 7 ++ .../tests/embed_unknown_compress_type.vv | 5 + vlib/v/embed_file/decoder.v | 17 ++++ vlib/v/embed_file/embed_file.v | 47 ++++++---- vlib/v/gen/c/cgen.v | 2 +- vlib/v/gen/c/comptime.v | 4 +- vlib/v/gen/c/coutput_test.v | 32 +++++-- vlib/v/gen/c/embed.v | 94 +++++++++++++++---- vlib/v/gen/c/testdata/embed.c.must_have | 4 +- .../c/testdata/embed_with_prod.c.must_have | 10 +- vlib/v/gen/c/testdata/embed_with_prod.out | 7 ++ ...with_prod_and_several_decoders.c.must_have | 11 +++ .../embed_with_prod_and_several_decoders.out | 23 +++++ .../embed_with_prod_and_several_decoders.vv | 44 +++++++++ .../testdata/embed_with_prod_zlib.c.must_have | 32 +++++++ .../v/gen/c/testdata/embed_with_prod_zlib.out | 7 ++ vlib/v/gen/c/testdata/embed_with_prod_zlib.vv | 8 ++ vlib/v/parser/comptime.v | 13 +++ .../embed_file/zlib/embed_file_zlib.v | 14 +++ 24 files changed, 408 insertions(+), 69 deletions(-) create mode 100644 cmd/tools/vcompress.v create mode 100644 vlib/v/checker/tests/embed_unknown_compress_type.out create mode 100644 vlib/v/checker/tests/embed_unknown_compress_type.vv create mode 100644 vlib/v/embed_file/decoder.v create mode 100644 vlib/v/gen/c/testdata/embed_with_prod.out create mode 100644 vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.c.must_have create mode 100644 vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.out create mode 100644 vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.vv create mode 100644 vlib/v/gen/c/testdata/embed_with_prod_zlib.c.must_have create mode 100644 vlib/v/gen/c/testdata/embed_with_prod_zlib.out create mode 100644 vlib/v/gen/c/testdata/embed_with_prod_zlib.vv create mode 100644 vlib/v/preludes/embed_file/zlib/embed_file_zlib.v diff --git a/cmd/tools/vcompress.v b/cmd/tools/vcompress.v new file mode 100644 index 000000000..bcd18e8f9 --- /dev/null +++ b/cmd/tools/vcompress.v @@ -0,0 +1,44 @@ +module main + +import compress.zlib +import os + +enum CompressionType { + zlib +} + +fn main() { + if os.args.len != 5 { + eprintln('v compress ') + eprintln('supported types: zlib') + exit(1) + } + compression_type := match os.args[2] { + 'zlib' { + CompressionType.zlib + } + else { + eprintln('unsupported type: ${os.args[1]}') + exit(1) + } + } + path := os.args[3] + content := os.read_bytes(path) or { + eprintln('unable to read "$path": $err') + exit(1) + } + compressed := match compression_type { + .zlib { + zlib.compress(content) or { + eprintln('compression error: $err') + exit(1) + } + } + } + out_path := os.args[4] + + os.write_file_array(out_path, compressed) or { + eprintln('failed to write "$out_path": $err') + exit(1) + } +} diff --git a/cmd/v/v.v b/cmd/v/v.v index a522bc0c2..66b354383 100644 --- a/cmd/v/v.v +++ b/cmd/v/v.v @@ -20,6 +20,7 @@ const ( 'build-vbinaries', 'check-md', 'complete', + 'compress', 'doc', 'doctor', 'fmt', diff --git a/doc/docs.md b/doc/docs.md index 3a1a3e0b7..9235269b9 100644 --- a/doc/docs.md +++ b/doc/docs.md @@ -4982,6 +4982,17 @@ executable, increasing your binary size, but making it more self contained and thus easier to distribute. In this case, `embedded_file.data()` will cause *no IO*, and it will always return the same data. +`$embed_file` supports compression of the embedded file when compiling with `-prod`. +Currently only one compression type is supported: `zlib` + +```v ignore +import os +fn main() { + embedded_file := $embed_file('v.png', .zlib) // compressed using zlib + os.write_file('exported.png', embedded_file.to_string()) ? +} +``` + #### `$tmpl` for embedding and parsing V template files V has a simple template language for text and html templates, and they can easily diff --git a/vlib/v/ast/ast.v b/vlib/v/ast/ast.v index d7b669d86..6b78086dc 100644 --- a/vlib/v/ast/ast.v +++ b/vlib/v/ast/ast.v @@ -646,8 +646,14 @@ pub mut: pub struct EmbeddedFile { pub: - rpath string // used in the source code, as an ID/key to the embed - apath string // absolute path during compilation to the resource + rpath string // used in the source code, as an ID/key to the embed + apath string // absolute path during compilation to the resource + compression_type string +pub mut: + // these are set by gen_embed_file_init in v/gen/c/embed + is_compressed bool + bytes []byte + len int } // Each V source file is represented by one File structure. @@ -1542,8 +1548,7 @@ pub: is_vweb bool vweb_tmpl File // - is_embed bool - embed_file EmbeddedFile + is_embed bool // is_env bool env_pos token.Position @@ -1554,6 +1559,7 @@ pub mut: result_type Type env_value string args []CallArg + embed_file EmbeddedFile } pub struct None { diff --git a/vlib/v/checker/checker.v b/vlib/v/checker/checker.v index a36e5920f..b72815353 100644 --- a/vlib/v/checker/checker.v +++ b/vlib/v/checker/checker.v @@ -20,20 +20,21 @@ const int_min = int(0x80000000) const int_max = int(0x7FFFFFFF) const ( - valid_comptime_if_os = ['windows', 'ios', 'macos', 'mach', 'darwin', 'hpux', 'gnu', + valid_comptime_if_os = ['windows', 'ios', 'macos', 'mach', 'darwin', 'hpux', 'gnu', 'qnx', 'linux', 'freebsd', 'openbsd', 'netbsd', 'bsd', 'dragonfly', 'android', 'solaris', 'haiku', 'serenity', 'vinix'] - valid_comptime_if_compilers = ['gcc', 'tinyc', 'clang', 'mingw', 'msvc', 'cplusplus'] - valid_comptime_if_platforms = ['amd64', 'i386', 'aarch64', 'arm64', 'arm32', 'rv64', 'rv32'] - valid_comptime_if_cpu_features = ['x64', 'x32', 'little_endian', 'big_endian'] - valid_comptime_if_other = ['js', 'debug', 'prod', 'test', 'glibc', 'prealloc', + valid_comptime_compression_types = ['none', 'zlib'] + valid_comptime_if_compilers = ['gcc', 'tinyc', 'clang', 'mingw', 'msvc', 'cplusplus'] + valid_comptime_if_platforms = ['amd64', 'i386', 'aarch64', 'arm64', 'arm32', 'rv64', 'rv32'] + valid_comptime_if_cpu_features = ['x64', 'x32', 'little_endian', 'big_endian'] + valid_comptime_if_other = ['js', 'debug', 'prod', 'test', 'glibc', 'prealloc', 'no_bounds_checking', 'freestanding', 'threads', 'js_node', 'js_browser', 'js_freestanding'] - valid_comptime_not_user_defined = all_valid_comptime_idents() - array_builtin_methods = ['filter', 'clone', 'repeat', 'reverse', 'map', 'slice', + valid_comptime_not_user_defined = all_valid_comptime_idents() + array_builtin_methods = ['filter', 'clone', 'repeat', 'reverse', 'map', 'slice', 'sort', 'contains', 'index', 'wait', 'any', 'all', 'first', 'last', 'pop'] - reserved_type_names = ['bool', 'char', 'i8', 'i16', 'int', 'i64', 'byte', 'u16', + reserved_type_names = ['bool', 'char', 'i8', 'i16', 'int', 'i64', 'byte', 'u16', 'u32', 'u64', 'f32', 'f64', 'map', 'string', 'rune'] - vroot_is_deprecated_message = '@VROOT is deprecated, use @VMODROOT or @VEXEROOT instead' + vroot_is_deprecated_message = '@VROOT is deprecated, use @VMODROOT or @VEXEROOT instead' ) fn all_valid_comptime_idents() []string { @@ -5866,7 +5867,12 @@ fn (mut c Checker) comptime_call(mut node ast.ComptimeCall) ast.Type { return ast.string_type } if node.is_embed { - c.file.embedded_files << node.embed_file + // c.file.embedded_files << node.embed_file + if node.embed_file.compression_type !in checker.valid_comptime_compression_types { + supported := checker.valid_comptime_compression_types.map('.$it').join(', ') + c.error('not supported compression type: .${node.embed_file.compression_type}. supported: $supported', + node.pos) + } return c.table.find_type_idx('v.embed_file.EmbedFileData') } if node.is_vweb { diff --git a/vlib/v/checker/tests/embed_unknown_compress_type.out b/vlib/v/checker/tests/embed_unknown_compress_type.out new file mode 100644 index 000000000..1baeeb688 --- /dev/null +++ b/vlib/v/checker/tests/embed_unknown_compress_type.out @@ -0,0 +1,7 @@ +vlib/v/checker/tests/embed_unknown_compress_type.vv:3:10: error: not supported compression type: .this_is_not_a_valid_compression_type. supported: .none, .zlib + 1 | + 2 | fn main() { + 3 | test := $embed_file('embed_unknown_compress_type.vv', .this_is_not_a_valid_compression_type) + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + 4 | println(test.to_string()) + 5 | } diff --git a/vlib/v/checker/tests/embed_unknown_compress_type.vv b/vlib/v/checker/tests/embed_unknown_compress_type.vv new file mode 100644 index 000000000..b7f610bb5 --- /dev/null +++ b/vlib/v/checker/tests/embed_unknown_compress_type.vv @@ -0,0 +1,5 @@ + +fn main() { + test := $embed_file('embed_unknown_compress_type.vv', .this_is_not_a_valid_compression_type) + println(test.to_string()) +} diff --git a/vlib/v/embed_file/decoder.v b/vlib/v/embed_file/decoder.v new file mode 100644 index 000000000..99b296e53 --- /dev/null +++ b/vlib/v/embed_file/decoder.v @@ -0,0 +1,17 @@ +[has_globals] +module embed_file + +interface Decoder { + decompress([]byte) ?[]byte +} + +struct EmbedFileDecoders { +mut: + decoders map[string]Decoder +} + +__global g_embed_file_decoders = &EmbedFileDecoders{} + +pub fn register_decoder(compression_type string, decoder Decoder) { + g_embed_file_decoders.decoders[compression_type] = decoder +} diff --git a/vlib/v/embed_file/embed_file.v b/vlib/v/embed_file/embed_file.v index 22ea45807..26fc834e8 100644 --- a/vlib/v/embed_file/embed_file.v +++ b/vlib/v/embed_file/embed_file.v @@ -9,7 +9,8 @@ pub const ( // https://github.com/vlang/rfcs/blob/master/embedding_resources.md // EmbedFileData encapsulates functionality for the `$embed_file()` compile time call. pub struct EmbedFileData { - apath string + apath string + compression_type string mut: compressed &byte uncompressed &byte @@ -29,6 +30,7 @@ pub fn (mut ed EmbedFileData) free() { unsafe { ed.path.free() ed.apath.free() + ed.compression_type.free() if ed.free_compressed { free(ed.compressed) ed.compressed = &byte(0) @@ -59,25 +61,31 @@ pub fn (original &EmbedFileData) to_bytes() []byte { pub fn (mut ed EmbedFileData) data() &byte { if !isnil(ed.uncompressed) { return ed.uncompressed + } + if isnil(ed.uncompressed) && !isnil(ed.compressed) { + decoder := g_embed_file_decoders.decoders[ed.compression_type] or { + panic('EmbedFileData error: unknown compression of "$ed.path": "$ed.compression_type"') + } + compressed := unsafe { ed.compressed.vbytes(ed.len) } + decompressed := decoder.decompress(compressed) or { + panic('EmbedFileData error: decompression of "$ed.path" failed: $err') + } + unsafe { + ed.uncompressed = &byte(memdup(decompressed.data, ed.len)) + } } else { - if isnil(ed.uncompressed) && !isnil(ed.compressed) { - // TODO implement uncompression - // See also C Gen.gen_embedded_data() where the compression should occur. - ed.uncompressed = ed.compressed - } else { - mut path := os.resource_abs_path(ed.path) + mut path := os.resource_abs_path(ed.path) + if !os.is_file(path) { + path = ed.apath if !os.is_file(path) { - path = ed.apath - if !os.is_file(path) { - panic('EmbedFileData error: files "$ed.path" and "$ed.apath" do not exist') - } - } - bytes := os.read_bytes(path) or { - panic('EmbedFileData error: "$path" could not be read: $err') + panic('EmbedFileData error: files "$ed.path" and "$ed.apath" do not exist') } - ed.uncompressed = bytes.data - ed.free_uncompressed = true } + bytes := os.read_bytes(path) or { + panic('EmbedFileData error: "$path" could not be read: $err') + } + ed.uncompressed = bytes.data + ed.free_uncompressed = true } return ed.uncompressed } @@ -91,19 +99,20 @@ pub fn (mut ed EmbedFileData) data() &byte { pub struct EmbedFileIndexEntry { id int path string + algo string data &byte } // find_index_entry_by_path is used internally by the V compiler: -pub fn find_index_entry_by_path(start voidptr, path string) &EmbedFileIndexEntry { +pub fn find_index_entry_by_path(start voidptr, path string, algo string) &EmbedFileIndexEntry { mut x := &EmbedFileIndexEntry(start) - for !(x.path == path || isnil(x.data)) { + for x.id >= 0 && x.data != 0 && (x.algo != algo || x.path != path) { unsafe { x++ } } $if debug_embed_file_in_prod ? { - eprintln('>> v.embed_file find_index_entry_by_path ${ptr_str(start)}, path: "$path" => ${ptr_str(x)}') + eprintln('>> v.embed_file find_index_entry_by_path ${ptr_str(start)}, id: $x.id, path: "$path", algo: "$algo" => ${ptr_str(x)}') } return x } diff --git a/vlib/v/gen/c/cgen.v b/vlib/v/gen/c/cgen.v index e3678fe2f..8ed8d8d3c 100644 --- a/vlib/v/gen/c/cgen.v +++ b/vlib/v/gen/c/cgen.v @@ -3828,7 +3828,7 @@ fn (mut g Gen) expr(node ast.Expr) { } ast.Comment {} ast.ComptimeCall { - g.comptime_call(node) + g.comptime_call(mut node) } ast.ComptimeSelector { g.comptime_selector(node) diff --git a/vlib/v/gen/c/comptime.v b/vlib/v/gen/c/comptime.v index e3ebc6513..4bcd86bf2 100644 --- a/vlib/v/gen/c/comptime.v +++ b/vlib/v/gen/c/comptime.v @@ -28,10 +28,10 @@ fn (mut g Gen) comptime_selector(node ast.ComptimeSelector) { g.expr(node.field_expr) } -fn (mut g Gen) comptime_call(node ast.ComptimeCall) { +fn (mut g Gen) comptime_call(mut node ast.ComptimeCall) { if node.is_embed { // $embed_file('/path/to/file') - g.gen_embed_file_init(node) + g.gen_embed_file_init(mut node) return } if node.method_name == 'env' { diff --git a/vlib/v/gen/c/coutput_test.v b/vlib/v/gen/c/coutput_test.v index 59df1aae3..214971415 100644 --- a/vlib/v/gen/c/coutput_test.v +++ b/vlib/v/gen/c/coutput_test.v @@ -12,6 +12,8 @@ const testdata_folder = os.join_path(vroot, 'vlib', 'v', 'gen', 'c', 'testdata') const diff_cmd = diff.find_working_diff_command() or { '' } +const show_compilation_output = os.getenv('VTEST_SHOW_COMPILATION_OUTPUT').int() == 1 + fn mm(s string) string { return term.colorize(term.magenta, s) } @@ -34,9 +36,13 @@ fn test_out_files() ? { mut total_errors := 0 for out_path in paths { basename, path, relpath, out_relpath := target2paths(out_path, '.out') - print(mm('v run $relpath') + ' == ${mm(out_relpath)} ') pexe := os.join_path(output_path, '${basename}.exe') - compilation := os.execute('"$vexe" -o "$pexe" "$path"') + // + file_options := get_file_options(path) + alloptions := '-o "$pexe" $file_options.vflags' + print(mm('v $alloptions run $relpath') + ' == ${mm(out_relpath)} ') + // + compilation := os.execute('"$vexe" $alloptions "$path"') ensure_compilation_succeeded(compilation) res := os.execute(pexe) if res.exit_code < 0 { @@ -99,12 +105,14 @@ fn test_c_must_have_files() ? { } paths := vtest.filter_vtest_only(tests, basepath: testdata_folder) mut total_errors := 0 + mut failed_descriptions := []string{cap: paths.len} for must_have_path in paths { basename, path, relpath, must_have_relpath := target2paths(must_have_path, '.c.must_have') file_options := get_file_options(path) alloptions := '-o - $file_options.vflags' - print(mm('v $alloptions $relpath') + - ' matches all line patterns in ${mm(must_have_relpath)} ') + description := mm('v $alloptions $relpath') + + ' matches all line patterns in ${mm(must_have_relpath)} ' + print(description) cmd := '$vexe $alloptions $path' compilation := os.execute(cmd) ensure_compilation_succeeded(compilation) @@ -122,6 +130,9 @@ fn test_c_must_have_files() ? { eprintln('$must_have_path:${idx_expected_line + 1}: expected match error:') eprintln('`$cmd` did NOT produce expected line:') eprintln(term.colorize(term.red, eline)) + if description !in failed_descriptions { + failed_descriptions << description + } total_errors++ continue } @@ -129,8 +140,10 @@ fn test_c_must_have_files() ? { if nmatches == expected_lines.len { println(term.green('OK')) } else { - eprintln('> ALL lines:') - eprintln(compilation.output) + if show_compilation_output { + eprintln('> ALL lines:') + eprintln(compilation.output) + } eprintln('--------- failed patterns: -------------------------------------------') for fpattern in failed_patterns { eprintln(fpattern) @@ -138,6 +151,13 @@ fn test_c_must_have_files() ? { eprintln('----------------------------------------------------------------------') } } + if failed_descriptions.len > 0 { + eprintln('--------- failed commands: -------------------------------------------') + for fd in failed_descriptions { + eprintln(' > $fd') + } + eprintln('----------------------------------------------------------------------') + } assert total_errors == 0 } diff --git a/vlib/v/gen/c/embed.v b/vlib/v/gen/c/embed.v index 9ecb9bc41..0afd2ca49 100644 --- a/vlib/v/gen/c/embed.v +++ b/vlib/v/gen/c/embed.v @@ -1,7 +1,9 @@ module c import os +import rand import v.ast +import v.pref fn (mut g Gen) embed_file_is_prod_mode() bool { if g.pref.is_prod || 'debug_embed_file_in_prod' in g.pref.compile_defines { @@ -11,7 +13,51 @@ fn (mut g Gen) embed_file_is_prod_mode() bool { } // gen_embed_file_struct generates C code for `$embed_file('...')` calls. -fn (mut g Gen) gen_embed_file_init(node ast.ComptimeCall) { +fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) { + if g.embed_file_is_prod_mode() { + file_bytes := os.read_bytes(node.embed_file.apath) or { + panic('unable to read file: "$node.embed_file.rpath') + } + + if node.embed_file.compression_type == 'none' { + node.embed_file.bytes = file_bytes + } else { + cache_dir := os.join_path(os.vmodules_dir(), 'cache', 'embed_file') + cache_key := rand.ulid() + // cache_key := md5.hexhash(node.embed_file.apath) + if !os.exists(cache_dir) { + os.mkdir_all(cache_dir) or { panic(err) } + } + cache_path := os.join_path(cache_dir, cache_key) + + vexe := pref.vexe_path() + result := os.execute('"$vexe" compress $node.embed_file.compression_type "$node.embed_file.apath" "$cache_path"') + if result.exit_code != 0 { + eprintln('unable to compress file "$node.embed_file.rpath": $result.output') + node.embed_file.bytes = file_bytes + } else { + compressed_bytes := os.read_bytes(cache_path) or { + eprintln('unable to read compressed file') + { + } + []byte{} + } + os.rm(cache_path) or {} // clean up + node.embed_file.is_compressed = compressed_bytes.len > 0 + && compressed_bytes.len < file_bytes.len + node.embed_file.bytes = if node.embed_file.is_compressed { + compressed_bytes + } else { + file_bytes + } + } + } + if node.embed_file.bytes.len > 5242880 { + eprintln('embedding of files >= ~5MB is currently not well supported') + } + node.embed_file.len = file_bytes.len + } + g.writeln('(v__embed_file__EmbedFileData){') g.writeln('\t\t.path = ${ctoslit(node.embed_file.rpath)},') if g.embed_file_is_prod_mode() { @@ -20,27 +66,36 @@ fn (mut g Gen) gen_embed_file_init(node ast.ComptimeCall) { } else { g.writeln('\t\t.apath = ${ctoslit(node.embed_file.apath)},') } - file_size := os.file_size(node.embed_file.apath) - if file_size > 5242880 { - eprintln('Warning: embedding of files >= ~5MB is currently not supported') - } if g.embed_file_is_prod_mode() { - // Use function generated in Gen.gen_embedded_data() - g.writeln('\t\t.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)})->data,') + // use function generated in Gen.gen_embedded_data() + if node.embed_file.is_compressed { + g.writeln('\t\t.compression_type = ${ctoslit(node.embed_file.compression_type)},') + g.writeln('\t\t.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)}, ${ctoslit(node.embed_file.compression_type)})->data,') + g.writeln('\t\t.uncompressed = NULL,') + } else { + g.writeln('\t\t.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)}, ${ctoslit(node.embed_file.compression_type)})->data,') + } + } else { + g.writeln('\t\t.uncompressed = NULL,') } - g.writeln('\t\t.uncompressed = NULL,') g.writeln('\t\t.free_compressed = 0,') g.writeln('\t\t.free_uncompressed = 0,') - g.writeln('\t\t.len = $file_size') + if g.embed_file_is_prod_mode() { + g.writeln('\t\t.len = $node.embed_file.len') + } else { + file_size := os.file_size(node.embed_file.apath) + if file_size > 5242880 { + eprintln('Warning: embedding of files >= ~5MB is currently not supported') + } + g.writeln('\t\t.len = $file_size') + } g.writeln('} // \$embed_file("$node.embed_file.apath")') + + g.file.embedded_files << node.embed_file } // gen_embedded_data embeds data into the V target executable. fn (mut g Gen) gen_embedded_data() { - /* - TODO implement compression. - See also the vlib/embed module where decompression should occur. - */ /* TODO implement support for large files - right now the setup has problems // with even just 10 - 50 MB files - the problem is both in V and C compilers. @@ -48,11 +103,10 @@ fn (mut g Gen) gen_embedded_data() { // like the `rcc` tool in Qt? */ for i, emfile in g.embedded_files { - fbytes := os.read_bytes(emfile.apath) or { panic('Error while embedding file: $err') } - g.embedded_data.write_string('static const unsigned char _v_embed_blob_$i[$fbytes.len] = {\n ') - for j := 0; j < fbytes.len; j++ { - b := fbytes[j].hex() - if j < fbytes.len - 1 { + g.embedded_data.write_string('static const unsigned char _v_embed_blob_$i[$emfile.bytes.len] = {\n ') + for j := 0; j < emfile.bytes.len; j++ { + b := emfile.bytes[j].hex() + if j < emfile.bytes.len - 1 { g.embedded_data.write_string('0x$b,') } else { g.embedded_data.write_string('0x$b') @@ -66,9 +120,9 @@ fn (mut g Gen) gen_embedded_data() { g.embedded_data.writeln('') g.embedded_data.writeln('const v__embed_file__EmbedFileIndexEntry _v_embed_file_index[] = {') for i, emfile in g.embedded_files { - g.embedded_data.writeln('\t{$i, { .str=(byteptr)("${cestring(emfile.rpath)}"), .len=$emfile.rpath.len, .is_lit=1 }, _v_embed_blob_$i},') + g.embedded_data.writeln('\t{$i, { .str=(byteptr)("${cestring(emfile.rpath)}"), .len=$emfile.rpath.len, .is_lit=1 }, { .str=(byteptr)("${cestring(emfile.compression_type)}"), .len=$emfile.compression_type.len, .is_lit=1 }, _v_embed_blob_$i},') } - g.embedded_data.writeln('\t{-1, { .str=(byteptr)(""), .len=0, .is_lit=1 }, NULL}') + g.embedded_data.writeln('\t{-1, { .str=(byteptr)(""), .len=0, .is_lit=1 }, { .str=(byteptr)(""), .len=0, .is_lit=1 }, NULL}') g.embedded_data.writeln('};') // see vlib/v/embed_file/embed_file.v, find_index_entry_by_id/2 and find_index_entry_by_path/2 } diff --git a/vlib/v/gen/c/testdata/embed.c.must_have b/vlib/v/gen/c/testdata/embed.c.must_have index 78b83cb31..fd6037a51 100644 --- a/vlib/v/gen/c/testdata/embed.c.must_have +++ b/vlib/v/gen/c/testdata/embed.c.must_have @@ -7,11 +7,11 @@ struct v__embed_file__EmbedFileIndexEntry { string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed); void v__embed_file__EmbedFileData_free(v__embed_file__EmbedFileData* ed); byte* v__embed_file__EmbedFileData_data(v__embed_file__EmbedFileData* ed); -v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path); +v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo); string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) { string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) { -v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path) { +v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) { v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){ .path = _SLIT("embed.vv"), diff --git a/vlib/v/gen/c/testdata/embed_with_prod.c.must_have b/vlib/v/gen/c/testdata/embed_with_prod.c.must_have index 8c96f2a78..c245ab904 100644 --- a/vlib/v/gen/c/testdata/embed_with_prod.c.must_have +++ b/vlib/v/gen/c/testdata/embed_with_prod.c.must_have @@ -5,8 +5,8 @@ static const unsigned char _v_embed_blob_0[138] = { 0x66,0x6e,0x20,0x6d,0x61,0x69,0x6e,0x28,0x29,0x20,0x7b,0x0a,0x09,0x6d,0x75,0x74, const v__embed_file__EmbedFileIndexEntry _v_embed_file_index[] = { - {0, { .str=(byteptr)("embed.vv"), .len=8, .is_lit=1 }, _v_embed_blob_0}, - {-1, { .str=(byteptr)(""), .len=0, .is_lit=1 }, NULL} + {0, { .str=(byteptr)("embed.vv"), .len=8, .is_lit=1 }, { .str=(byteptr)("none"), .len=4, .is_lit=1 }, _v_embed_blob_0}, + {-1, { .str=(byteptr)(""), .len=0, .is_lit=1 }, { .str=(byteptr)(""), .len=0, .is_lit=1 }, NULL} }; typedef struct v__embed_file__EmbedFileData v__embed_file__EmbedFileData; @@ -18,15 +18,15 @@ struct v__embed_file__EmbedFileIndexEntry { string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed); void v__embed_file__EmbedFileData_free(v__embed_file__EmbedFileData* ed); byte* v__embed_file__EmbedFileData_data(v__embed_file__EmbedFileData* ed); -v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path); +v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo); string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) { string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) { -v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path) { +v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) { v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){ .path = _SLIT("embed.vv"), .apath = _SLIT(""), -.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"))->data, +.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data, // Initializations for module v.embed_file : diff --git a/vlib/v/gen/c/testdata/embed_with_prod.out b/vlib/v/gen/c/testdata/embed_with_prod.out new file mode 100644 index 000000000..358dbffc9 --- /dev/null +++ b/vlib/v/gen/c/testdata/embed_with_prod.out @@ -0,0 +1,7 @@ +fn main() { + mut my_source := $embed_file('embed.vv') + assert my_source.len > 0 + s := my_source.to_string() + assert s.len > 0 + print(s) +} diff --git a/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.c.must_have b/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.c.must_have new file mode 100644 index 000000000..5d8c9aebf --- /dev/null +++ b/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.c.must_have @@ -0,0 +1,11 @@ + +{0, { .str=(byteptr)("embed.vv"), .len=8, .is_lit=1 }, { .str=(byteptr)("zlib"), .len=4, .is_lit=1 }, _v_embed_blob_0}, +{1, { .str=(byteptr)("embed.vv"), .len=8, .is_lit=1 }, { .str=(byteptr)("none"), .len=4, .is_lit=1 }, _v_embed_blob_1}, + +VV_LOCAL_SYMBOL void v__preludes__embed_file__zlib__init(void); +VV_LOCAL_SYMBOL Option_Array_byte v__preludes__embed_file__zlib__ZLibDecoder_decompress(v__preludes__embed_file__zlib__ZLibDecoder _d1, Array_byte data) { += compress__zlib__decompress(data); + +.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data, +.compression_type = _SLIT("zlib") +.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data, diff --git a/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.out b/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.out new file mode 100644 index 000000000..c77f4dc3f --- /dev/null +++ b/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.out @@ -0,0 +1,23 @@ +fn main() { + mut my_source := $embed_file('embed.vv') + assert my_source.len > 0 + s := my_source.to_string() + assert s.len > 0 + print(s) +} +-------------------------------------- +fn main() { + mut my_source := $embed_file('embed.vv') + assert my_source.len > 0 + s := my_source.to_string() + assert s.len > 0 + print(s) +} +-------------------------------------- +fn main() { + mut my_source := $embed_file('embed.vv') + assert my_source.len > 0 + s := my_source.to_string() + assert s.len > 0 + print(s) +} diff --git a/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.vv b/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.vv new file mode 100644 index 000000000..650b03798 --- /dev/null +++ b/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.vv @@ -0,0 +1,44 @@ +// vtest vflags: -prod +// NB: asserts are stripped with -prod => instead use plain if{panic()} +fn main() { + mut my_source := $embed_file('embed.vv', .zlib) + mut my_source_2 := $embed_file('embed.vv', .zlib) + if my_source.len <= 0 { + panic('my_source.len <= 0') + } + s := my_source.to_string() + if s.len <= 0 { + panic('s.len <= 0') + } + print(s) + + println('--------------------------------------') + print( my_source_2.to_string() ) + println('--------------------------------------') + + mut my_uncompressed_source := $embed_file('embed.vv') + if my_uncompressed_source.len <= 0 { + panic('my_uncompressed_source.len <= 0') + } + us :=my_uncompressed_source.to_string() + if us.len <= 0 { + panic('us.len <= 0') + } + print(us) + + // + + if us != s { + println('us != s') + } + + // + mut my_uncompressed_source_2 := $embed_file('embed.vv') + if my_uncompressed_source.len != my_uncompressed_source_2.len { + panic('my_uncompressed_source.len != my_uncompressed_source_2.len') + } + + if my_source.len != my_source_2.len { + panic('my_source.len != my_source_2.len') + } +} diff --git a/vlib/v/gen/c/testdata/embed_with_prod_zlib.c.must_have b/vlib/v/gen/c/testdata/embed_with_prod_zlib.c.must_have new file mode 100644 index 000000000..f08654a42 --- /dev/null +++ b/vlib/v/gen/c/testdata/embed_with_prod_zlib.c.must_have @@ -0,0 +1,32 @@ +#define _VPROD (1) + +// V embedded data: +static const unsigned char _v_embed_blob_0[121] = { +0x78,0x01,0x05,0x40,0x4b,0x0a,0x83,0x30,0x10,0x5d,0x67,0x4e,0xf1,0x16,0x05,0x93, + +const v__embed_file__EmbedFileIndexEntry _v_embed_file_index[] = { + {0, { .str=(byteptr)("embed.vv"), .len=8, .is_lit=1 }, { .str=(byteptr)("zlib"), .len=4, .is_lit=1 }, _v_embed_blob_0}, + {-1, { .str=(byteptr)(""), .len=0, .is_lit=1 }, { .str=(byteptr)(""), .len=0, .is_lit=1 }, NULL} +}; + +typedef struct v__embed_file__EmbedFileData v__embed_file__EmbedFileData; +typedef struct v__embed_file__EmbedFileIndexEntry v__embed_file__EmbedFileIndexEntry; + +struct v__embed_file__EmbedFileData { +struct v__embed_file__EmbedFileIndexEntry { + +string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed); +void v__embed_file__EmbedFileData_free(v__embed_file__EmbedFileData* ed); +byte* v__embed_file__EmbedFileData_data(v__embed_file__EmbedFileData* ed); +v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo); + +string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) { +string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) { +v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) { + +v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){ +.path = _SLIT("embed.vv"), +.apath = _SLIT(""), +.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data, + +// Initializations for module v.embed_file : diff --git a/vlib/v/gen/c/testdata/embed_with_prod_zlib.out b/vlib/v/gen/c/testdata/embed_with_prod_zlib.out new file mode 100644 index 000000000..358dbffc9 --- /dev/null +++ b/vlib/v/gen/c/testdata/embed_with_prod_zlib.out @@ -0,0 +1,7 @@ +fn main() { + mut my_source := $embed_file('embed.vv') + assert my_source.len > 0 + s := my_source.to_string() + assert s.len > 0 + print(s) +} diff --git a/vlib/v/gen/c/testdata/embed_with_prod_zlib.vv b/vlib/v/gen/c/testdata/embed_with_prod_zlib.vv new file mode 100644 index 000000000..9b11972d7 --- /dev/null +++ b/vlib/v/gen/c/testdata/embed_with_prod_zlib.vv @@ -0,0 +1,8 @@ +// vtest vflags: -prod +fn main() { + mut my_source := $embed_file('embed.vv', .zlib) + assert my_source.len > 0 + s := my_source.to_string() + assert s.len > 0 + print(s) +} diff --git a/vlib/v/parser/comptime.v b/vlib/v/parser/comptime.v index 3c6a0a857..a8adb2f09 100644 --- a/vlib/v/parser/comptime.v +++ b/vlib/v/parser/comptime.v @@ -95,6 +95,14 @@ fn (mut p Parser) comptime_call() ast.ComptimeCall { if !is_html { p.check(.string) } + mut embed_compression_type := 'none' + if is_embed_file { + if p.tok.kind == .comma { + p.check(.comma) + p.check(.dot) + embed_compression_type = p.check_name() + } + } p.check(.rpar) // $embed_file('/path/to/file') if is_embed_file { @@ -126,12 +134,17 @@ fn (mut p Parser) comptime_call() ast.ComptimeCall { } } p.register_auto_import('v.preludes.embed_file') + if embed_compression_type == 'zlib' + && (p.pref.is_prod || 'debug_embed_file_in_prod' in p.pref.compile_defines) { + p.register_auto_import('v.preludes.embed_file.zlib') + } return ast.ComptimeCall{ scope: 0 is_embed: true embed_file: ast.EmbeddedFile{ rpath: literal_string_param apath: epath + compression_type: embed_compression_type } pos: start_pos.extend(p.prev_tok.position()) } diff --git a/vlib/v/preludes/embed_file/zlib/embed_file_zlib.v b/vlib/v/preludes/embed_file/zlib/embed_file_zlib.v new file mode 100644 index 000000000..243e042a2 --- /dev/null +++ b/vlib/v/preludes/embed_file/zlib/embed_file_zlib.v @@ -0,0 +1,14 @@ +module zlib + +import compress.zlib +import v.embed_file + +struct ZLibDecoder {} + +fn (_ ZLibDecoder) decompress(data []byte) ?[]byte { + return zlib.decompress(data) +} + +fn init() { + embed_file.register_decoder('zlib', embed_file.Decoder(ZLibDecoder{})) +} -- 2.30.2