cgen,builder,markused,vcache: make -usecache correct for multi-module builds (fix #27592)

vlib/v/ast/ast.v

@@ -1123,6 +1123,7 @@ pub mut:
 pub struct EmbeddedFile {
 pub:
 	compression_type string
+	source_file      string // path of the .v file containing the $embed_file call; the anchor for relative-path resolution (a const may be checked from an importing file's context, so Checker.file is not reliable here)
 pub mut:
 	rpath string // used in the source code, as an ID/key to the embed
 	apath string // absolute path during compilation to the resource
@@ -1165,6 +1166,7 @@ pub mut:
 	used_imports          []string
 	implied_imports       []string                  // ​imports that the user's code uses but omitted to import explicitly, used by `vfmt`
 	embedded_files        []EmbeddedFile            // list of files to embed in the binary
+	embedded_apaths       []string                  // absolute paths of $embed_file targets (deduped, checker-resolved); drives -usecache invalidation of embedded assets
 	imported_symbols      map[string]string         // used for `import {symbol}`, it maps symbol => module.symbol
 	imported_symbols_trie token.KeywordsMatcherTrie // constructed from imported_symbols, to accelerate presense checks
 	imported_symbols_used map[string]bool

vlib/v/builder/cc.v

@@ -1321,6 +1321,11 @@ fn (mut v Builder) setup_output_name() {
 			get_dsc_content('PREF.PATH: ${v.pref.path}\nVOPTS: ${v.pref.cache_manager.vopts}\n')) or {
 			panic(err)
 		}
+		// Record the content hashes of every $embed_file target that went into this
+		// cached object, so rebuild_cached_module can invalidate it when an embedded
+		// asset changes (embedded assets are invisible to the .v source hashes).
+		v.pref.cache_manager.mod_save(v.pref.path, '.embeds.txt', v.pref.path,
+			embeds_manifest(v.parsed_files)) or { panic(err) }
 	}
 	if os.is_dir(v.pref.out_name) {
 		verror('${os.quoted_path(v.pref.out_name)} is a directory')

vlib/v/builder/rebuilding.v

@@ -27,19 +27,54 @@ pub fn (mut b Builder) rebuild_modules() {
 	$if trace_invalidations ? {
 		eprintln('> rebuild_modules all_files: ${all_files}')
 	}
-	invalidations := b.find_invalidated_modules_by_files(all_files)
+	invalidations, snew_hashes := b.find_invalidated_modules_by_files(all_files)
 	$if trace_invalidations ? {
 		eprintln('> rebuild_modules invalidations: ${invalidations}')
 	}
+	mut stale_object_might_survive := false
 	if invalidations.len > 0 {
 		vexe := pref.vexe_path()
 		for imp in invalidations {
-			b.v_build_module(vexe, imp)
+			rc := b.v_build_module(vexe, imp)
+			if rc != 0 {
+				// CACHE-POISONING GUARD: a failed module rebuild used to be
+				// silently ignored while the new source hashes were ALREADY saved —
+				// every later build then saw "hashes unchanged", skipped invalidation,
+				// and linked the STALE cached object forever (until a manual cache
+				// wipe). Observed as a permanent all-tests link failure
+				// (_builtin__init_consts unresolved); an ABI-compatible variant would
+				// be a silently WRONG binary. Aborting outright is too strict — the
+				// invalidator sometimes names paths that are not real buildable
+				// modules (e.g. a folder of standalone _test.v files) whose rebuild
+				// has always failed harmlessly and which have no cached object to go
+				// stale. The precise guard: a failure is only dangerous if a stale
+				// object could still be SERVED — so remove the module's cached object;
+				// once nothing stale can be linked, recording the new source hashes is
+				// safe. Only if the stale object cannot be removed do we skip the hash
+				// save (forcing re-detection + retry on the next run). A genuinely
+				// needed-but-missing object is rebuilt on demand by
+				// rebuild_cached_module, which fails loudly.
+				stale_o := b.pref.cache_manager.mod_postfix_with_key2cpath(imp, '.o', imp)
+				if os.exists(stale_o) {
+					os.rm(stale_o) or { stale_object_might_survive = true }
+					eprintln('warning: `v build-module ${imp}` failed (exit code ${rc}); removed its cached object so nothing stale can be linked.')
+				} else {
+					vcache.dlog('| Builder.' + @FN,
+						'build-module failed for ${imp} (rc=${rc}); no cached object under this key — harmless (not a servable module)')
+				}
+			}
 		}
 	}
+	if !stale_object_might_survive {
+		// Persist the new source hashes only now — after every invalidated module was
+		// rebuilt (or its stale object provably removed) — so a failed rebuild can
+		// never poison the cache state.
+		mut cm := vcache.new_cache_manager(all_files)
+		cm.save('.hashes', 'all_files', snew_hashes) or {}
+	}
 }
 
-pub fn (mut b Builder) find_invalidated_modules_by_files(all_files []string) []string {
+pub fn (mut b Builder) find_invalidated_modules_by_files(all_files []string) ([]string, string) {
 	util.timing_start('${@METHOD} source_hashing')
 	mut new_hashes := map[string]string{}
 	mut old_hashes := map[string]string{}
@@ -72,7 +107,9 @@ pub fn (mut b Builder) find_invalidated_modules_by_files(all_files []string) []s
 	// eprintln('new_hashes: ${new_hashes}')
 	// eprintln('> new_hashes != old_hashes: ' + ( old_hashes != new_hashes ).str())
 	// eprintln(snew_hashes)
-	cm.save('.hashes', 'all_files', snew_hashes) or {}
+	// NOTE: the new hashes are deliberately NOT saved here. The caller
+	// saves them only after every invalidated module has been rebuilt
+	// successfully; saving eagerly poisoned the cache on any rebuild failure.
 	util.timing_measure('${@METHOD} source_hashing')
 
 	mut invalidations := []string{}
@@ -182,10 +219,10 @@ pub fn (mut b Builder) find_invalidated_modules_by_files(all_files []string) []s
 		}
 		util.timing_measure('${@METHOD} rebuilding')
 	}
-	return invalidations
+	return invalidations, snew_hashes
 }
 
-fn (mut b Builder) v_build_module(vexe string, imp_path string) {
+fn (mut b Builder) v_build_module(vexe string, imp_path string) int {
 	pwd := os.getwd()
 	defer {
 		os.chdir(pwd) or {}
@@ -200,22 +237,103 @@ fn (mut b Builder) v_build_module(vexe string, imp_path string) {
 	$if trace_v_build_module ? {
 		eprintln('> Builder.v_build_module: ${rebuild_cmd}')
 	}
-	// eprintln('> Builder.v_build_module: ${rebuild_cmd}')
-	os.system(rebuild_cmd)
+	// The child must reconstruct EXACTLY the parent's build configuration from
+	// the relayed b.pref.build_options — the parent already folded VFLAGS into
+	// those while parsing its own args. Letting the child re-apply VFLAGS on
+	// top duplicates its values (e.g. `-cflags x` becomes `x x`), so the child
+	// derives a DIFFERENT vcache key than the one the parent looks up, and the
+	// freshly built module object is never found ('could not rebuild cache
+	// module ... does not exist yet' — hit by any -usecache build running
+	// under a VFLAGS environment, e.g. the sanitized CI jobs).
+	old_vflags := os.getenv('VFLAGS')
+	if old_vflags != '' {
+		os.setenv('VFLAGS', '', true)
+	}
+	// The exit status is the caller's problem now: ignoring it while
+	// the hashes were saved eagerly permanently poisoned the module cache.
+	rc := os.system(rebuild_cmd)
+	if old_vflags != '' {
+		os.setenv('VFLAGS', old_vflags, true)
+	}
+	return rc
+}
+
+// embed_file_content_hash returns the invalidation hash for one embedded-asset
+// path; an unreadable file hashes to a constant that can never match a real
+// content hash, so it always invalidates.
+fn embed_file_content_hash(apath string) string {
+	econtent := util.read_file(apath) or { return 'unreadable' }
+	return hash.sum64_string(econtent, 7).hex_full()
+}
+
+// embeds_manifest serialises every checker-resolved $embed_file target of
+// `parsed_files` as `<content-hash> <absolute-path>` lines. Saved next to a
+// cached module object (see setup_output_name); verified by
+// cached_module_embeds_fresh before a cached object is served.
+fn embeds_manifest(parsed_files []&ast.File) string {
+	mut seen := map[string]bool{}
+	mut lines := []string{}
+	for pf in parsed_files {
+		for ea in pf.embedded_apaths {
+			if seen[ea] {
+				continue
+			}
+			seen[ea] = true
+			lines << '${embed_file_content_hash(ea)} ${ea}'
+		}
+	}
+	lines.sort()
+	return lines.join('\n')
+}
+
+// cached_module_embeds_fresh reports whether every embedded asset recorded in
+// the module's .embeds.txt manifest still has the content it was built with.
+// The .v source hashes cannot see $embed_file targets, so without this check a
+// cached object silently keeps serving stale embedded data after the asset
+// changes. A missing manifest means the object was built by this compiler
+// build with no embeds recorded (the compiler-identity salt namespaces away
+// pre-manifest objects), so it is trivially fresh.
+fn (mut b Builder) cached_module_embeds_fresh(imp_path string) bool {
+	manifest := b.pref.cache_manager.mod_load(imp_path, '.embeds.txt', imp_path) or { return true }
+	for line in manifest.split_into_lines() {
+		if line == '' {
+			continue
+		}
+		recorded := line.all_before(' ')
+		apath := line.all_after(' ')
+		if embed_file_content_hash(apath) != recorded {
+			vcache.dlog('| Builder.' + @FN, 'stale embedded asset: ${apath} (module ${imp_path})')
+			return false
+		}
+	}
+	return true
 }
 
 fn (mut b Builder) rebuild_cached_module(vexe string, imp_path string) string {
-	res := b.pref.cache_manager.mod_exists(imp_path, '.o', imp_path) or {
-		if b.pref.is_verbose {
+	mut existing := ''
+	if res := b.pref.cache_manager.mod_exists(imp_path, '.o', imp_path) {
+		existing = res
+	}
+	if existing != '' && b.cached_module_embeds_fresh(imp_path) {
+		return existing
+	}
+	if b.pref.is_verbose {
+		if existing == '' {
 			println('Cached ${imp_path} .o file not found... Building .o file for ${imp_path}')
+		} else {
+			println('Cached ${imp_path} .o file has stale embedded assets... Rebuilding .o file for ${imp_path}')
 		}
-		b.v_build_module(vexe, imp_path)
-		rebuilt_o := b.pref.cache_manager.mod_exists(imp_path, '.o', imp_path) or {
-			panic('could not rebuild cache module for ${imp_path}, error: ${err.msg()}')
-		}
-		return rebuilt_o
 	}
-	return res
+	rc := b.v_build_module(vexe, imp_path)
+	if rc != 0 && existing != '' {
+		// A failed rebuild must not keep serving the stale object (the same
+		// poisoning class as the eager-hash-save bug, cx #151).
+		os.rm(existing) or {}
+	}
+	rebuilt_o := b.pref.cache_manager.mod_exists(imp_path, '.o', imp_path) or {
+		panic('could not rebuild cache module for ${imp_path}, error: ${err.msg()}')
+	}
+	return rebuilt_o
 }
 
 fn (mut b Builder) handle_usecache(vexe string) {
@@ -227,7 +345,15 @@ fn (mut b Builder) handle_usecache(vexe string) {
 	builtin_obj_path := b.rebuild_cached_module(vexe, 'vlib/builtin')
 	libs << builtin_obj_path
 	for ast_file in b.parsed_files {
-		if b.pref.is_test && ast_file.mod.name != 'main' {
+		// Cache the test's own non-main modules. Apply the same guards as the import
+		// loop below: builtin (and its parts: strconv/strings/math.bits/...) are already
+		// inside builtin.o, and a module already queued must not be built again — without
+		// these guards a test build caches builtin a SECOND time via its absolute path
+		// (a086..module.builtin.o AND 7c16..module.<abspath>.builtin.o), linking both and
+		// duplicating every builtin symbol (~9000 "duplicate symbol" errors at link).
+		if b.pref.is_test && ast_file.mod.name != 'main' && ast_file.mod.name != 'help'
+			&& !util.module_is_builtin(ast_file.mod.name) && ast_file.mod.name !in built_modules
+			&& !util.should_bundle_module(ast_file.mod.name) {
 			imp_path := b.find_module_path(ast_file.mod.name, ast_file.path) or {
 				verror('cannot import module "${ast_file.mod.name}" (not found)')
 				break

vlib/v/checker/comptime.v

@@ -476,11 +476,7 @@ fn (mut c Checker) comptime_call(mut node ast.ComptimeCall) ast.Type {
 			escaped_path = c.resolve_pseudo_variables(escaped_path, node.pos) or {
 				return ast.string_type
 			}
-			abs_path := os.real_path(escaped_path)
-			// check absolute path first
-			if !os.exists(abs_path) {
-				// ... look relative to the source file:
-				escaped_path = os.real_path(os.join_path_single(os.dir(c.file.path), escaped_path))
+			if os.is_abs_path(escaped_path) {
 				if !os.exists(escaped_path) {
 					c.error('"${escaped_path}" does not exist so it cannot be embedded', node.pos)
 					return ast.string_type
@@ -489,11 +485,50 @@ fn (mut c Checker) comptime_call(mut node ast.ComptimeCall) ast.Type {
 					c.error('"${escaped_path}" is not a file so it cannot be embedded', node.pos)
 					return ast.string_type
 				}
+				escaped_path = os.real_path(escaped_path)
 			} else {
-				escaped_path = abs_path
+				// Per the documented semantics ("Paths can be absolute or relative to the
+				// source file"), a relative path binds to the file next to the SOURCE first.
+				// Probing the process CWD first made the embedded content depend on where
+				// the compiler happened to run: the -usecache `v build-module` child runs
+				// chdir-ed to VROOT, so a CWD-relative twin of the file there silently
+				// replaced the intended source-relative one in the cached module object,
+				// while the main program TU embedded the right one. CWD-relative resolution
+				// stays as a fallback for backwards compatibility.
+				// The anchor is the file DECLARING the call (recorded by the parser): a
+				// module's const initializer can be checked from an importing file's
+				// context, so c.file would be the wrong anchor there.
+				anchor := if node.embed_file.source_file != '' {
+					node.embed_file.source_file
+				} else {
+					c.file.path
+				}
+				src_relative := os.real_path(os.join_path_single(os.dir(anchor), escaped_path))
+				if os.exists(src_relative) && os.is_file(src_relative) {
+					escaped_path = src_relative
+				} else {
+					cwd_relative := os.real_path(escaped_path)
+					if !os.exists(cwd_relative) {
+						c.error('"${src_relative}" does not exist so it cannot be embedded',
+							node.pos)
+						return ast.string_type
+					}
+					if !os.is_file(cwd_relative) {
+						c.error('"${cwd_relative}" is not a file so it cannot be embedded',
+							node.pos)
+						return ast.string_type
+					}
+					escaped_path = cwd_relative
+				}
 			}
 			node.embed_file.rpath = raw_path
 			node.embed_file.apath = escaped_path
+			// Record the resolved target on the file, so the module cache can detect a
+			// changed embedded asset (invisible to the .v source hashes) — consumed by
+			// Builder.rebuild_cached_module via the .embeds.txt manifest.
+			if escaped_path !in c.file.embedded_apaths {
+				c.file.embedded_apaths << escaped_path
+			}
 		}
 		// c.file.embedded_files << node.embed_file
 		if node.embed_file.compression_type !in ast.valid_comptime_compression_types {

vlib/v/gen/c/auto_str_methods.v

@@ -31,8 +31,8 @@ fn (mut g Gen) gen_str_default(sym ast.TypeSymbol, styp string, str_fn_name stri
 	} else {
 		verror('could not generate string method for type `${styp}`')
 	}
-	g.definitions.writeln('string ${str_fn_name}(${styp} it);')
-	g.auto_str_funcs.writeln('string ${str_fn_name}(${styp} it) {')
+	g.definitions.writeln('${g.static_non_parallel}string ${str_fn_name}(${styp} it);')
+	g.auto_str_funcs.writeln('${g.static_non_parallel}string ${str_fn_name}(${styp} it) {')
 	if convertor == 'bool' {
 		g.auto_str_funcs.writeln('\tstring tmp1 = builtin__string__plus(_S("${styp}("), ((${convertor})it ? _S("true") : _S("false")));')
 	} else {
@@ -226,10 +226,10 @@ fn (mut g Gen) gen_str_for_option(typ ast.Type, styp string, str_fn_name string,
 	parent_str_fn_name := g.get_str_fn(parent_type)
 	parent_cast_type := g.auto_str_storage_cast_type(parent_type)
 
-	g.definitions.writeln('string ${str_fn_name}(${styp} it);')
-	g.auto_str_funcs.writeln('string ${str_fn_name}(${styp} it) { return indent_${str_fn_name}(it, 0); }')
-	g.definitions.writeln('string indent_${str_fn_name}(${styp} it, ${ast.int_type_name} indent_count);')
-	g.auto_str_funcs.writeln('string indent_${str_fn_name}(${styp} it, ${ast.int_type_name} indent_count) {')
+	g.definitions.writeln('${g.static_non_parallel}string ${str_fn_name}(${styp} it);')
+	g.auto_str_funcs.writeln('${g.static_non_parallel}string ${str_fn_name}(${styp} it) { return indent_${str_fn_name}(it, 0); }')
+	g.definitions.writeln('${g.static_non_parallel}string indent_${str_fn_name}(${styp} it, ${ast.int_type_name} indent_count);')
+	g.auto_str_funcs.writeln('${g.static_non_parallel}string indent_${str_fn_name}(${styp} it, ${ast.int_type_name} indent_count) {')
 	g.auto_str_funcs.writeln('\tstring res;')
 	g.auto_str_funcs.writeln('\tif (it.state == 0) {')
 	deref := if typ.is_ptr() && !typ.has_flag(.option_mut_param_t) {
@@ -276,10 +276,10 @@ fn (mut g Gen) gen_str_for_result(typ ast.Type, styp string, str_fn_name string)
 	parent_str_fn_name := g.get_str_fn(parent_type)
 	parent_cast_type := g.auto_str_storage_cast_type(parent_type)
 
-	g.definitions.writeln('string ${str_fn_name}(${styp} it);')
-	g.auto_str_funcs.writeln('string ${str_fn_name}(${styp} it) { return indent_${str_fn_name}(it, 0); }')
-	g.definitions.writeln('string indent_${str_fn_name}(${styp} it, ${ast.int_type_name} indent_count);')
-	g.auto_str_funcs.writeln('string indent_${str_fn_name}(${styp} it, ${ast.int_type_name} indent_count) {')
+	g.definitions.writeln('${g.static_non_parallel}string ${str_fn_name}(${styp} it);')
+	g.auto_str_funcs.writeln('${g.static_non_parallel}string ${str_fn_name}(${styp} it) { return indent_${str_fn_name}(it, 0); }')
+	g.definitions.writeln('${g.static_non_parallel}string indent_${str_fn_name}(${styp} it, ${ast.int_type_name} indent_count);')
+	g.auto_str_funcs.writeln('${g.static_non_parallel}string indent_${str_fn_name}(${styp} it, ${ast.int_type_name} indent_count) {')
 	g.auto_str_funcs.writeln('\tstring res;')
 	g.auto_str_funcs.writeln('\tif (!it.is_error) {')
 	if sym.kind == .string {