✨ Configurations embed and retrieve (#21321, #23303)

buildroot/share/PlatformIO/scripts/SAMD51_grandcentral_m4.py

@@ -6,7 +6,6 @@ import pioutil
 if pioutil.is_pio_build():
 	from os.path import join, isfile
 	import shutil
-	from pprint import pprint
 
 	Import("env")
 

buildroot/share/PlatformIO/scripts/common-dependencies.py

@@ -192,63 +192,6 @@ if pioutil.is_pio_build():
 				lib_ignore = env.GetProjectOption('lib_ignore') + [feat['lib_ignore']]
 				set_env_field('lib_ignore', lib_ignore)
 
-	#
-	# Find a compiler, considering the OS
-	#
-	ENV_BUILD_PATH = os.path.join(env.Dictionary('PROJECT_BUILD_DIR'), env['PIOENV'])
-	GCC_PATH_CACHE = os.path.join(ENV_BUILD_PATH, ".gcc_path")
-	def search_compiler():
-		try:
-			filepath = env.GetProjectOption('custom_gcc')
-			blab("Getting compiler from env")
-			return filepath
-		except:
-			pass
-
-		if os.path.exists(GCC_PATH_CACHE):
-			with open(GCC_PATH_CACHE, 'r') as f:
-				return f.read()
-
-		# Find the current platform compiler by searching the $PATH
-		# which will be in a platformio toolchain bin folder
-		path_regex = re.escape(env['PROJECT_PACKAGES_DIR'])
-
-		# See if the environment provides a default compiler
-		try:
-			gcc = env.GetProjectOption('custom_deps_gcc')
-		except:
-			gcc = "g++"
-
-		if env['PLATFORM'] == 'win32':
-			path_separator = ';'
-			path_regex += r'.*\\bin'
-			gcc += ".exe"
-		else:
-			path_separator = ':'
-			path_regex += r'/.+/bin'
-
-		# Search for the compiler
-		for pathdir in env['ENV']['PATH'].split(path_separator):
-			if not re.search(path_regex, pathdir, re.IGNORECASE):
-				continue
-			for filepath in os.listdir(pathdir):
-				if not filepath.endswith(gcc):
-					continue
-				# Use entire path to not rely on env PATH
-				filepath = os.path.sep.join([pathdir, filepath])
-				# Cache the g++ path to no search always
-				if os.path.exists(ENV_BUILD_PATH):
-					with open(GCC_PATH_CACHE, 'w+') as f:
-						f.write(filepath)
-
-				return filepath
-
-		filepath = env.get('CXX')
-		if filepath == 'CC':
-			filepath = gcc
-		blab("Couldn't find a compiler! Fallback to %s" % filepath)
-		return filepath
-
 	#
 	# Use the compiler to get a list of all enabled features
 	#
@@ -257,25 +200,8 @@ if pioutil.is_pio_build():
 			return
 
 		# Process defines
-		build_flags = env.get('BUILD_FLAGS')
-		build_flags = env.ParseFlagsExtended(build_flags)
-
-		cxx = search_compiler()
-		cmd = ['"' + cxx + '"']
-
-		# Build flags from board.json
-		#if 'BOARD' in env:
-		#	cmd += [env.BoardConfig().get("build.extra_flags")]
-		for s in build_flags['CPPDEFINES']:
-			if isinstance(s, tuple):
-				cmd += ['-D' + s[0] + '=' + str(s[1])]
-			else:
-				cmd += ['-D' + s]
-
-		cmd += ['-D__MARLIN_DEPS__ -w -dM -E -x c++ buildroot/share/PlatformIO/scripts/common-dependencies.h']
-		cmd = ' '.join(cmd)
-		blab(cmd, 4)
-		define_list = subprocess.check_output(cmd, shell=True).splitlines()
+		from preprocessor import run_preprocessor
+		define_list = run_preprocessor(env)
 		marlin_features = {}
 		for define in define_list:
 			feature = define[8:].strip().decode().split(' ')
@@ -310,9 +236,18 @@ if pioutil.is_pio_build():
 	except:
 		pass
 
+	#
 	# Add a method for other PIO scripts to query enabled features
+	#
 	env.AddMethod(MarlinFeatureIsEnabled)
 
+	#
 	# Add dependencies for enabled Marlin features
+	#
 	apply_features_config()
 	force_ignore_unused_libs()
+
+	#print(env.Dump())
+
+	from signature import compute_build_signature
+	compute_build_signature(env)

buildroot/share/PlatformIO/scripts/mc-apply.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+#
+# Create a Configuration from marlin_config.json
+#
+import json
+import sys
+import shutil
+import re
+
+opt_output = '--opt' in sys.argv
+output_suffix = '.sh' if opt_output else '' if '--bare-output' in sys.argv else '.gen'
+
+try:
+	with open('marlin_config.json', 'r') as infile:
+		conf = json.load(infile)
+		for key in conf:
+			# We don't care about the hash when restoring here
+			if key == '__INITIAL_HASH':
+				continue
+			if key == 'VERSION':
+				for k, v in sorted(conf[key].items()):
+					print(k + ': ' + v)
+				continue
+			# The key is the file name, so let's build it now
+			outfile = open('Marlin/' + key + output_suffix, 'w')
+			for k, v in sorted(conf[key].items()):
+				# Make define line now
+				if opt_output:
+					if v != '':
+						if '"' in v:
+							v = "'%s'" % v
+						elif ' ' in v:
+							v = '"%s"' % v
+						define = 'opt_set ' + k + ' ' + v + '\n'
+					else:
+						define = 'opt_enable ' + k + '\n'
+				else:
+					define = '#define ' + k + ' ' + v + '\n'
+				outfile.write(define)
+			outfile.close()
+
+			# Try to apply changes to the actual configuration file (in order to keep useful comments)
+			if output_suffix != '':
+				# Move the existing configuration so it doesn't interfere
+				shutil.move('Marlin/' + key, 'Marlin/' + key + '.orig')
+				infile_lines = open('Marlin/' + key + '.orig', 'r').read().split('\n')
+				outfile = open('Marlin/' + key, 'w')
+				for line in infile_lines:
+					sline = line.strip(" \t\n\r")
+					if sline[:7] == "#define":
+						# Extract the key here (we don't care about the value)
+						kv = sline[8:].strip().split(' ')
+						if kv[0] in conf[key]:
+							outfile.write('#define ' + kv[0] + ' ' + conf[key][kv[0]] + '\n')
+							# Remove the key from the dict, so we can still write all missing keys at the end of the file
+							del conf[key][kv[0]]
+						else:
+							outfile.write(line + '\n')
+					else:
+						outfile.write(line + '\n')
+				# Process any remaining defines here
+				for k, v in sorted(conf[key].items()):
+					define = '#define ' + k + ' ' + v + '\n'
+					outfile.write(define)
+				outfile.close()
+
+			print('Output configuration written to: ' + 'Marlin/' + key + output_suffix)
+except:
+	print('No marlin_config.json found.')

buildroot/share/PlatformIO/scripts/preprocessor.py

@@ -0,0 +1,99 @@
+#
+# preprocessor.py
+#
+import subprocess,os,re
+
+verbose = 0
+
+def blab(str):
+	if verbose:
+		print(str)
+
+################################################################################
+#
+# Invoke GCC to run the preprocessor and extract enabled features
+#
+preprocessor_cache = {}
+def run_preprocessor(env, fn=None):
+	filename = fn or 'buildroot/share/PlatformIO/scripts/common-dependencies.h'
+	if filename in preprocessor_cache:
+		return preprocessor_cache[filename]
+
+	# Process defines
+	build_flags = env.get('BUILD_FLAGS')
+	build_flags = env.ParseFlagsExtended(build_flags)
+
+	cxx = search_compiler(env)
+	cmd = ['"' + cxx + '"']
+
+	# Build flags from board.json
+	#if 'BOARD' in env:
+	#	cmd += [env.BoardConfig().get("build.extra_flags")]
+	for s in build_flags['CPPDEFINES']:
+		if isinstance(s, tuple):
+			cmd += ['-D' + s[0] + '=' + str(s[1])]
+		else:
+			cmd += ['-D' + s]
+
+	cmd += ['-D__MARLIN_DEPS__ -w -dM -E -x c++']
+	depcmd = cmd + [ filename ]
+	cmd = ' '.join(depcmd)
+	blab(cmd)
+	define_list = subprocess.check_output(cmd, shell=True).splitlines()
+	preprocessor_cache[filename] = define_list
+	return define_list
+
+
+################################################################################
+#
+# Find a compiler, considering the OS
+#
+def search_compiler(env):
+
+	ENV_BUILD_PATH = os.path.join(env.Dictionary('PROJECT_BUILD_DIR'), env['PIOENV'])
+	GCC_PATH_CACHE = os.path.join(ENV_BUILD_PATH, ".gcc_path")
+
+	try:
+		filepath = env.GetProjectOption('custom_gcc')
+		blab("Getting compiler from env")
+		return filepath
+	except:
+		pass
+
+	if os.path.exists(GCC_PATH_CACHE):
+		blab("Getting g++ path from cache")
+		with open(GCC_PATH_CACHE, 'r') as f:
+			return f.read()
+
+	# Find the current platform compiler by searching the $PATH
+	# which will be in a platformio toolchain bin folder
+	path_regex = re.escape(env['PROJECT_PACKAGES_DIR'])
+	gcc = "g++"
+	if env['PLATFORM'] == 'win32':
+		path_separator = ';'
+		path_regex += r'.*\\bin'
+		gcc += ".exe"
+	else:
+		path_separator = ':'
+		path_regex += r'/.+/bin'
+
+	# Search for the compiler
+	for pathdir in env['ENV']['PATH'].split(path_separator):
+		if not re.search(path_regex, pathdir, re.IGNORECASE):
+			continue
+		for filepath in os.listdir(pathdir):
+			if not filepath.endswith(gcc):
+				continue
+			# Use entire path to not rely on env PATH
+			filepath = os.path.sep.join([pathdir, filepath])
+			# Cache the g++ path to no search always
+			if os.path.exists(ENV_BUILD_PATH):
+				blab("Caching g++ for current env")
+				with open(GCC_PATH_CACHE, 'w+') as f:
+					f.write(filepath)
+
+			return filepath
+
+	filepath = env.get('CXX')
+	blab("Couldn't find a compiler! Fallback to %s" % filepath)
+	return filepath

buildroot/share/PlatformIO/scripts/signature.py

@@ -0,0 +1,176 @@
+#
+# signature.py
+#
+import os,subprocess,re,json,hashlib
+
+#
+# The dumbest preprocessor in the world
+# Extract macro name from an header file and store them in an array
+# No processing is done here, so they are raw values here and it does not match what actually enabled
+# in the file (since you can have #if SOMETHING_UNDEFINED / #define BOB / #endif)
+# But it's useful to filter the useful macro spit out by the preprocessor from noise from the system
+# headers.
+#
+def extract_defines(filepath):
+	f = open(filepath, encoding="utf8").read().split("\n")
+	a = []
+	for line in f:
+		sline = line.strip(" \t\n\r")
+		if sline[:7] == "#define":
+			# Extract the key here (we don't care about the value)
+			kv = sline[8:].strip().split(' ')
+			a.append(kv[0])
+	return a
+
+# Compute the SHA256 hash of a file
+def get_file_sha256sum(filepath):
+	sha256_hash = hashlib.sha256()
+	with open(filepath,"rb") as f:
+		# Read and update hash string value in blocks of 4K
+		for byte_block in iter(lambda: f.read(4096),b""):
+			sha256_hash.update(byte_block)
+	return sha256_hash.hexdigest()
+
+#
+# Compress a JSON file into a zip file
+#
+import zipfile
+def compress_file(filepath, outputbase):
+	with zipfile.ZipFile(outputbase + '.zip', 'w', compression=zipfile.ZIP_BZIP2, compresslevel=9) as zipf:
+		zipf.write(filepath, compress_type=zipfile.ZIP_BZIP2, compresslevel=9)
+
+#
+# Compute the build signature. The idea is to extract all defines in the configuration headers
+# to build a unique reversible signature from this build so it can be included in the binary
+# We can reverse the signature to get a 1:1 equivalent configuration file
+#
+def compute_build_signature(env):
+	if 'BUILD_SIGNATURE' in env:
+		return
+
+	# Definitions from these files will be kept
+	files_to_keep = [ 'Marlin/Configuration.h', 'Marlin/Configuration_adv.h' ]
+
+	build_dir=os.path.join(env['PROJECT_BUILD_DIR'], env['PIOENV'])
+
+	# Check if we can skip processing
+	hashes = ''
+	for header in files_to_keep:
+		hashes += get_file_sha256sum(header)[0:10]
+
+	marlin_json = os.path.join(build_dir, 'marlin_config.json')
+	marlin_zip = os.path.join(build_dir, 'mc')
+
+	# Read existing config file
+	try:
+		with open(marlin_json, 'r') as infile:
+			conf = json.load(infile)
+			if conf['__INITIAL_HASH'] == hashes:
+				# Same configuration, skip recomputing the building signature
+				compress_file(marlin_json, marlin_zip)
+				return
+	except:
+		pass
+
+	# Get enabled config options based on preprocessor
+	from preprocessor import run_preprocessor
+	complete_cfg = run_preprocessor(env)
+
+	# Dumb #define extraction from the configuration files
+	real_defines = {}
+	all_defines = []
+	for header in files_to_keep:
+		defines = extract_defines(header)
+		# To filter only the define we want
+		all_defines = all_defines + defines
+		# To remember from which file it cames from
+		real_defines[header.split('/')[-1]] = defines
+
+	r = re.compile(r"\(+(\s*-*\s*_.*)\)+")
+
+	# First step is to collect all valid macros
+	defines = {}
+	for line in complete_cfg:
+
+		# Split the define from the value
+		key_val = line[8:].strip().decode().split(' ')
+		key, value = key_val[0], ' '.join(key_val[1:])
+
+		# Ignore values starting with two underscore, since it's low level
+		if len(key) > 2 and key[0:2] == "__" :
+			continue
+		# Ignore values containing a parenthesis (likely a function macro)
+		if '(' in key and ')' in key:
+			continue
+
+		# Then filter dumb values
+		if r.match(value):
+			continue
+
+		defines[key] = value if len(value) else ""
+
+	if not 'CONFIGURATION_EMBEDDING' in defines:
+		return
+
+	# Second step is to filter useless macro
+	resolved_defines = {}
+	for key in defines:
+		# Remove all boards now
+		if key[0:6] == "BOARD_" and key != "BOARD_INFO_NAME":
+			continue
+		# Remove all keys ending by "_NAME" as it does not make a difference to the configuration
+		if key[-5:] == "_NAME" and key != "CUSTOM_MACHINE_NAME":
+			continue
+		# Remove all keys ending by "_T_DECLARED" as it's a copy of not important system stuff
+		if key[-11:] == "_T_DECLARED":
+			continue
+		# Remove keys that are not in the #define list in the Configuration list
+		if not (key in all_defines) and key != "DETAILED_BUILD_VERSION" and key != "STRING_DISTRIBUTION_DATE":
+			continue
+
+		# Don't be that smart guy here
+		resolved_defines[key] = defines[key]
+
+	# Generate a build signature now
+	# We are making an object that's a bit more complex than a basic dictionary here
+	data = {}
+	data['__INITIAL_HASH'] = hashes
+	# First create a key for each header here
+	for header in real_defines:
+		data[header] = {}
+
+	# Then populate the object where each key is going to (that's a O(N^2) algorithm here...)
+	for key in resolved_defines:
+		for header in real_defines:
+			if key in real_defines[header]:
+				data[header][key] = resolved_defines[key]
+
+	# Append the source code version and date
+	data['VERSION'] = {}
+	data['VERSION']['DETAILED_BUILD_VERSION'] = resolved_defines['DETAILED_BUILD_VERSION']
+	data['VERSION']['STRING_DISTRIBUTION_DATE'] = resolved_defines['STRING_DISTRIBUTION_DATE']
+	try:
+		curver = subprocess.check_output(["git", "describe", "--match=NeVeRmAtCh", "--always"]).strip()
+		data['VERSION']['GIT_REF'] = curver.decode()
+	except:
+		pass
+
+	with open(marlin_json, 'w') as outfile:
+		json.dump(data, outfile, separators=(',', ':'))
+
+	# Compress the JSON file as much as we can
+	compress_file(marlin_json, marlin_zip)
+
+	# Generate a C source file for storing this array
+	with open('Marlin/src/mczip.h','wb') as result_file:
+		result_file.write(b'#warning "Generated file \'mc.zip\' is embedded"\n')
+		result_file.write(b'const unsigned char mc_zip[] PROGMEM = {\n ')
+		count = 0
+		for b in open(os.path.join(build_dir, 'mc.zip'), 'rb').read():
+			result_file.write(b' 0x%02X,' % b)
+			count += 1
+			if (count % 16 == 0):
+			 	result_file.write(b'\n ')
+		if (count % 16):
+			result_file.write(b'\n')
+		result_file.write(b'};\n')