✨ Configurations embed and retrieve (#21321, #23303)

.gitignore

@@ -22,6 +22,9 @@
 # Generated files
 _Version.h
 bdf2u8g
+marlin_config.json
+mczip.h
+*.gen
 *.sublime-workspace
 
 #

Marlin/Configuration_adv.h

@@ -1593,6 +1593,14 @@
     #define SD_FIRMWARE_UPDATE_INACTIVE_VALUE 0xFF
   #endif
 
+  /**
+   * Enable this option if you have more than ~3K of unused flash space.
+   * Marlin will embed all settings in the firmware binary as compressed data.
+   * Use 'M503 C' to write the settings out to the SD Card as 'mc.zip'.
+   * See docs/ConfigEmbedding.md for details on how to use 'mc-apply.py'.
+   */
+  //#define CONFIGURATION_EMBEDDING
+
   // Add an optimized binary file transfer mode, initiated with 'M28 B1'
   //#define BINARY_FILE_TRANSFER
 

Marlin/src/gcode/eeprom/M500-M504.cpp

@@ -25,6 +25,11 @@
 #include "../../core/serial.h"
 #include "../../inc/MarlinConfig.h"
 
+#if ENABLED(CONFIGURATION_EMBEDDING)
+  #include "../../sd/SdBaseFile.h"
+  #include "../../mczip.h"
+#endif
+
 /**
  * M500: Store settings in EEPROM
  */
@@ -50,9 +55,22 @@ void GcodeSuite::M502() {
 
   /**
    * M503: print settings currently in memory
+   *
+   * With CONFIGURATION_EMBEDDING:
+   *   C<flag> : Save the full Marlin configuration to SD Card as "mc.zip"
    */
   void GcodeSuite::M503() {
     (void)settings.report(!parser.boolval('S', true));
+
+    #if ENABLED(CONFIGURATION_EMBEDDING)
+      if (parser.seen_test('C')) {
+        SdBaseFile file;
+        const uint16_t size = sizeof(mc_zip);
+        // Need to create the config size on the SD card
+        if (file.open("mc.zip", O_WRITE|O_CREAT) && file.write(pgm_read_ptr(mc_zip), size) != -1 && file.close())
+          SERIAL_ECHO_MSG("Configuration saved as 'mc.zip'");
+      }
+    #endif
   }
 
 #endif // !DISABLE_M503

Marlin/src/gcode/host/M115.cpp

@@ -24,7 +24,6 @@
 #include "../../inc/MarlinConfig.h"
 #include "../queue.h"           // for getting the command port
 
-
 #if ENABLED(M115_GEOMETRY_REPORT)
   #include "../../module/motion.h"
 #endif
@@ -33,13 +32,25 @@
   #include "../../feature/caselight.h"
 #endif
 
+//#define MINIMAL_CAP_LINES // Don't even mention the disabled capabilities
+
 #if ENABLED(EXTENDED_CAPABILITIES_REPORT)
-  static void cap_line(FSTR_P const name, bool ena=false) {
+  #if ENABLED(MINIMAL_CAP_LINES)
+    #define cap_line(S,C) if (C) _cap_line(S)
+    static void _cap_line(FSTR_P const name) {
+      SERIAL_ECHOPGM("Cap:");
+      SERIAL_ECHOF(name);
+      SERIAL_ECHOLNPGM(":1");
+    }
+  #else
+    #define cap_line(V...) _cap_line(V)
+    static void _cap_line(FSTR_P const name, bool ena=false) {
       SERIAL_ECHOPGM("Cap:");
       SERIAL_ECHOF(name);
       SERIAL_CHAR(':', '0' + ena);
       SERIAL_EOL();
     }
+  #endif
 #endif
 
 /**
@@ -167,6 +178,9 @@ void GcodeSuite::M115() {
     // MEATPACK Compression
     cap_line(F("MEATPACK"), SERIAL_IMPL.has_feature(port, SerialFeature::MeatPack));
 
+    // CONFIG_EXPORT
+    cap_line(F("CONFIG_EXPORT"), ENABLED(CONFIG_EMBED_AND_SAVE_TO_SD));
+
     // Machine Geometry
     #if ENABLED(M115_GEOMETRY_REPORT)
       const xyz_pos_t bmin = { 0, 0, 0 },

Marlin/src/inc/Conditionals_adv.h

@@ -1004,3 +1004,9 @@
 #if EITHER(MEATPACK_ON_SERIAL_PORT_1, MEATPACK_ON_SERIAL_PORT_2)
   #define HAS_MEATPACK 1
 #endif
+
+// AVR are (usually) too limited in resources to store the configuration into the binary
+#if !defined(FORCE_CONFIG_EMBED) && (defined(__AVR__) || DISABLED(SDSUPPORT) || EITHER(SDCARD_READONLY, DISABLE_M503))
+  #undef CONFIGURATION_EMBEDDING
+  #define CANNOT_EMBED_CONFIGURATION defined(__AVR__)
+#endif

Marlin/src/inc/Warnings.cpp

@@ -549,3 +549,7 @@
 #elif !USE_SENSORLESS && ENABLED(USES_DIAG_PINS)
   #warning "Driver DIAG pins must be physically removed unless SENSORLESS_HOMING is enabled. (See https://bit.ly/2ZPRlt0)"
 #endif
+
+#if CANNOT_EMBED_CONFIGURATION
+  #warning "Disabled CONFIGURATION_EMBEDDING because the target usually has less flash storage. Define FORCE_CONFIG_EMBED to override."
+#endif

buildroot/bin/restore_configs

@@ -2,4 +2,4 @@
 
 git checkout Marlin/Configuration*.h 2>/dev/null
 git checkout Marlin/src/pins/ramps/pins_RAMPS.h 2>/dev/null
-rm -f Marlin/_Bootscreen.h Marlin/_Statusscreen.h
+rm -f Marlin/_Bootscreen.h Marlin/_Statusscreen.h marlin_config.json .pio/build/mc.zip

buildroot/share/PlatformIO/scripts/SAMD51_grandcentral_m4.py

@@ -6,7 +6,6 @@ import pioutil
 if pioutil.is_pio_build():
 	from os.path import join, isfile
 	import shutil
-	from pprint import pprint
 
 	Import("env")
 

buildroot/share/PlatformIO/scripts/common-dependencies.py

@@ -192,63 +192,6 @@ if pioutil.is_pio_build():
 				lib_ignore = env.GetProjectOption('lib_ignore') + [feat['lib_ignore']]
 				set_env_field('lib_ignore', lib_ignore)
 
-	#
-	# Find a compiler, considering the OS
-	#
-	ENV_BUILD_PATH = os.path.join(env.Dictionary('PROJECT_BUILD_DIR'), env['PIOENV'])
-	GCC_PATH_CACHE = os.path.join(ENV_BUILD_PATH, ".gcc_path")
-	def search_compiler():
-		try:
-			filepath = env.GetProjectOption('custom_gcc')
-			blab("Getting compiler from env")
-			return filepath
-		except:
-			pass
-
-		if os.path.exists(GCC_PATH_CACHE):
-			with open(GCC_PATH_CACHE, 'r') as f:
-				return f.read()
-
-		# Find the current platform compiler by searching the $PATH
-		# which will be in a platformio toolchain bin folder
-		path_regex = re.escape(env['PROJECT_PACKAGES_DIR'])
-
-		# See if the environment provides a default compiler
-		try:
-			gcc = env.GetProjectOption('custom_deps_gcc')
-		except:
-			gcc = "g++"
-
-		if env['PLATFORM'] == 'win32':
-			path_separator = ';'
-			path_regex += r'.*\\bin'
-			gcc += ".exe"
-		else:
-			path_separator = ':'
-			path_regex += r'/.+/bin'
-
-		# Search for the compiler
-		for pathdir in env['ENV']['PATH'].split(path_separator):
-			if not re.search(path_regex, pathdir, re.IGNORECASE):
-				continue
-			for filepath in os.listdir(pathdir):
-				if not filepath.endswith(gcc):
-					continue
-				# Use entire path to not rely on env PATH
-				filepath = os.path.sep.join([pathdir, filepath])
-				# Cache the g++ path to no search always
-				if os.path.exists(ENV_BUILD_PATH):
-					with open(GCC_PATH_CACHE, 'w+') as f:
-						f.write(filepath)
-
-				return filepath
-
-		filepath = env.get('CXX')
-		if filepath == 'CC':
-			filepath = gcc
-		blab("Couldn't find a compiler! Fallback to %s" % filepath)
-		return filepath
-
 	#
 	# Use the compiler to get a list of all enabled features
 	#
@@ -257,25 +200,8 @@ if pioutil.is_pio_build():
 			return
 
 		# Process defines
-		build_flags = env.get('BUILD_FLAGS')
+		from preprocessor import run_preprocessor
-		build_flags = env.ParseFlagsExtended(build_flags)
+		define_list = run_preprocessor(env)
-
-		cxx = search_compiler()
-		cmd = ['"' + cxx + '"']
-
-		# Build flags from board.json
-		#if 'BOARD' in env:
-		#	cmd += [env.BoardConfig().get("build.extra_flags")]
-		for s in build_flags['CPPDEFINES']:
-			if isinstance(s, tuple):
-				cmd += ['-D' + s[0] + '=' + str(s[1])]
-			else:
-				cmd += ['-D' + s]
-
-		cmd += ['-D__MARLIN_DEPS__ -w -dM -E -x c++ buildroot/share/PlatformIO/scripts/common-dependencies.h']
-		cmd = ' '.join(cmd)
-		blab(cmd, 4)
-		define_list = subprocess.check_output(cmd, shell=True).splitlines()
 		marlin_features = {}
 		for define in define_list:
 			feature = define[8:].strip().decode().split(' ')
@@ -310,9 +236,18 @@ if pioutil.is_pio_build():
 	except:
 		pass
 
+	#
 	# Add a method for other PIO scripts to query enabled features
+	#
 	env.AddMethod(MarlinFeatureIsEnabled)
 
+	#
 	# Add dependencies for enabled Marlin features
+	#
 	apply_features_config()
 	force_ignore_unused_libs()
+
+	#print(env.Dump())
+
+	from signature import compute_build_signature
+	compute_build_signature(env)

buildroot/share/PlatformIO/scripts/mc-apply.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+#
+# Create a Configuration from marlin_config.json
+#
+import json
+import sys
+import shutil
+import re
+
+opt_output = '--opt' in sys.argv
+output_suffix = '.sh' if opt_output else '' if '--bare-output' in sys.argv else '.gen'
+
+try:
+	with open('marlin_config.json', 'r') as infile:
+		conf = json.load(infile)
+		for key in conf:
+			# We don't care about the hash when restoring here
+			if key == '__INITIAL_HASH':
+				continue
+			if key == 'VERSION':
+				for k, v in sorted(conf[key].items()):
+					print(k + ': ' + v)
+				continue
+			# The key is the file name, so let's build it now
+			outfile = open('Marlin/' + key + output_suffix, 'w')
+			for k, v in sorted(conf[key].items()):
+				# Make define line now
+				if opt_output:
+					if v != '':
+						if '"' in v:
+							v = "'%s'" % v
+						elif ' ' in v:
+							v = '"%s"' % v
+						define = 'opt_set ' + k + ' ' + v + '\n'
+					else:
+						define = 'opt_enable ' + k + '\n'
+				else:
+					define = '#define ' + k + ' ' + v + '\n'
+				outfile.write(define)
+			outfile.close()
+
+			# Try to apply changes to the actual configuration file (in order to keep useful comments)
+			if output_suffix != '':
+				# Move the existing configuration so it doesn't interfere
+				shutil.move('Marlin/' + key, 'Marlin/' + key + '.orig')
+				infile_lines = open('Marlin/' + key + '.orig', 'r').read().split('\n')
+				outfile = open('Marlin/' + key, 'w')
+				for line in infile_lines:
+					sline = line.strip(" \t\n\r")
+					if sline[:7] == "#define":
+						# Extract the key here (we don't care about the value)
+						kv = sline[8:].strip().split(' ')
+						if kv[0] in conf[key]:
+							outfile.write('#define ' + kv[0] + ' ' + conf[key][kv[0]] + '\n')
+							# Remove the key from the dict, so we can still write all missing keys at the end of the file
+							del conf[key][kv[0]]
+						else:
+							outfile.write(line + '\n')
+					else:
+						outfile.write(line + '\n')
+				# Process any remaining defines here
+				for k, v in sorted(conf[key].items()):
+					define = '#define ' + k + ' ' + v + '\n'
+					outfile.write(define)
+				outfile.close()
+
+			print('Output configuration written to: ' + 'Marlin/' + key + output_suffix)
+except:
+	print('No marlin_config.json found.')

buildroot/share/PlatformIO/scripts/preprocessor.py

@@ -0,0 +1,99 @@
+#
+# preprocessor.py
+#
+import subprocess,os,re
+
+verbose = 0
+
+def blab(str):
+	if verbose:
+		print(str)
+
+################################################################################
+#
+# Invoke GCC to run the preprocessor and extract enabled features
+#
+preprocessor_cache = {}
+def run_preprocessor(env, fn=None):
+	filename = fn or 'buildroot/share/PlatformIO/scripts/common-dependencies.h'
+	if filename in preprocessor_cache:
+		return preprocessor_cache[filename]
+
+	# Process defines
+	build_flags = env.get('BUILD_FLAGS')
+	build_flags = env.ParseFlagsExtended(build_flags)
+
+	cxx = search_compiler(env)
+	cmd = ['"' + cxx + '"']
+
+	# Build flags from board.json
+	#if 'BOARD' in env:
+	#	cmd += [env.BoardConfig().get("build.extra_flags")]
+	for s in build_flags['CPPDEFINES']:
+		if isinstance(s, tuple):
+			cmd += ['-D' + s[0] + '=' + str(s[1])]
+		else:
+			cmd += ['-D' + s]
+
+	cmd += ['-D__MARLIN_DEPS__ -w -dM -E -x c++']
+	depcmd = cmd + [ filename ]
+	cmd = ' '.join(depcmd)
+	blab(cmd)
+	define_list = subprocess.check_output(cmd, shell=True).splitlines()
+	preprocessor_cache[filename] = define_list
+	return define_list
+
+
+################################################################################
+#
+# Find a compiler, considering the OS
+#
+def search_compiler(env):
+
+	ENV_BUILD_PATH = os.path.join(env.Dictionary('PROJECT_BUILD_DIR'), env['PIOENV'])
+	GCC_PATH_CACHE = os.path.join(ENV_BUILD_PATH, ".gcc_path")
+
+	try:
+		filepath = env.GetProjectOption('custom_gcc')
+		blab("Getting compiler from env")
+		return filepath
+	except:
+		pass
+
+	if os.path.exists(GCC_PATH_CACHE):
+		blab("Getting g++ path from cache")
+		with open(GCC_PATH_CACHE, 'r') as f:
+			return f.read()
+
+	# Find the current platform compiler by searching the $PATH
+	# which will be in a platformio toolchain bin folder
+	path_regex = re.escape(env['PROJECT_PACKAGES_DIR'])
+	gcc = "g++"
+	if env['PLATFORM'] == 'win32':
+		path_separator = ';'
+		path_regex += r'.*\\bin'
+		gcc += ".exe"
+	else:
+		path_separator = ':'
+		path_regex += r'/.+/bin'
+
+	# Search for the compiler
+	for pathdir in env['ENV']['PATH'].split(path_separator):
+		if not re.search(path_regex, pathdir, re.IGNORECASE):
+			continue
+		for filepath in os.listdir(pathdir):
+			if not filepath.endswith(gcc):
+				continue
+			# Use entire path to not rely on env PATH
+			filepath = os.path.sep.join([pathdir, filepath])
+			# Cache the g++ path to no search always
+			if os.path.exists(ENV_BUILD_PATH):
+				blab("Caching g++ for current env")
+				with open(GCC_PATH_CACHE, 'w+') as f:
+					f.write(filepath)
+
+			return filepath
+
+	filepath = env.get('CXX')
+	blab("Couldn't find a compiler! Fallback to %s" % filepath)
+	return filepath

buildroot/share/PlatformIO/scripts/signature.py

@@ -0,0 +1,176 @@
+#
+# signature.py
+#
+import os,subprocess,re,json,hashlib
+
+#
+# The dumbest preprocessor in the world
+# Extract macro name from an header file and store them in an array
+# No processing is done here, so they are raw values here and it does not match what actually enabled
+# in the file (since you can have #if SOMETHING_UNDEFINED / #define BOB / #endif)
+# But it's useful to filter the useful macro spit out by the preprocessor from noise from the system
+# headers.
+#
+def extract_defines(filepath):
+	f = open(filepath, encoding="utf8").read().split("\n")
+	a = []
+	for line in f:
+		sline = line.strip(" \t\n\r")
+		if sline[:7] == "#define":
+			# Extract the key here (we don't care about the value)
+			kv = sline[8:].strip().split(' ')
+			a.append(kv[0])
+	return a
+
+# Compute the SHA256 hash of a file
+def get_file_sha256sum(filepath):
+	sha256_hash = hashlib.sha256()
+	with open(filepath,"rb") as f:
+		# Read and update hash string value in blocks of 4K
+		for byte_block in iter(lambda: f.read(4096),b""):
+			sha256_hash.update(byte_block)
+	return sha256_hash.hexdigest()
+
+#
+# Compress a JSON file into a zip file
+#
+import zipfile
+def compress_file(filepath, outputbase):
+	with zipfile.ZipFile(outputbase + '.zip', 'w', compression=zipfile.ZIP_BZIP2, compresslevel=9) as zipf:
+		zipf.write(filepath, compress_type=zipfile.ZIP_BZIP2, compresslevel=9)
+
+#
+# Compute the build signature. The idea is to extract all defines in the configuration headers
+# to build a unique reversible signature from this build so it can be included in the binary
+# We can reverse the signature to get a 1:1 equivalent configuration file
+#
+def compute_build_signature(env):
+	if 'BUILD_SIGNATURE' in env:
+		return
+
+	# Definitions from these files will be kept
+	files_to_keep = [ 'Marlin/Configuration.h', 'Marlin/Configuration_adv.h' ]
+
+	build_dir=os.path.join(env['PROJECT_BUILD_DIR'], env['PIOENV'])
+
+	# Check if we can skip processing
+	hashes = ''
+	for header in files_to_keep:
+		hashes += get_file_sha256sum(header)[0:10]
+
+	marlin_json = os.path.join(build_dir, 'marlin_config.json')
+	marlin_zip = os.path.join(build_dir, 'mc')
+
+	# Read existing config file
+	try:
+		with open(marlin_json, 'r') as infile:
+			conf = json.load(infile)
+			if conf['__INITIAL_HASH'] == hashes:
+				# Same configuration, skip recomputing the building signature
+				compress_file(marlin_json, marlin_zip)
+				return
+	except:
+		pass
+
+	# Get enabled config options based on preprocessor
+	from preprocessor import run_preprocessor
+	complete_cfg = run_preprocessor(env)
+
+	# Dumb #define extraction from the configuration files
+	real_defines = {}
+	all_defines = []
+	for header in files_to_keep:
+		defines = extract_defines(header)
+		# To filter only the define we want
+		all_defines = all_defines + defines
+		# To remember from which file it cames from
+		real_defines[header.split('/')[-1]] = defines
+
+	r = re.compile(r"\(+(\s*-*\s*_.*)\)+")
+
+	# First step is to collect all valid macros
+	defines = {}
+	for line in complete_cfg:
+
+		# Split the define from the value
+		key_val = line[8:].strip().decode().split(' ')
+		key, value = key_val[0], ' '.join(key_val[1:])
+
+		# Ignore values starting with two underscore, since it's low level
+		if len(key) > 2 and key[0:2] == "__" :
+			continue
+		# Ignore values containing a parenthesis (likely a function macro)
+		if '(' in key and ')' in key:
+			continue
+
+		# Then filter dumb values
+		if r.match(value):
+			continue
+
+		defines[key] = value if len(value) else ""
+
+	if not 'CONFIGURATION_EMBEDDING' in defines:
+		return
+
+	# Second step is to filter useless macro
+	resolved_defines = {}
+	for key in defines:
+		# Remove all boards now
+		if key[0:6] == "BOARD_" and key != "BOARD_INFO_NAME":
+			continue
+		# Remove all keys ending by "_NAME" as it does not make a difference to the configuration
+		if key[-5:] == "_NAME" and key != "CUSTOM_MACHINE_NAME":
+			continue
+		# Remove all keys ending by "_T_DECLARED" as it's a copy of not important system stuff
+		if key[-11:] == "_T_DECLARED":
+			continue
+		# Remove keys that are not in the #define list in the Configuration list
+		if not (key in all_defines) and key != "DETAILED_BUILD_VERSION" and key != "STRING_DISTRIBUTION_DATE":
+			continue
+
+		# Don't be that smart guy here
+		resolved_defines[key] = defines[key]
+
+	# Generate a build signature now
+	# We are making an object that's a bit more complex than a basic dictionary here
+	data = {}
+	data['__INITIAL_HASH'] = hashes
+	# First create a key for each header here
+	for header in real_defines:
+		data[header] = {}
+
+	# Then populate the object where each key is going to (that's a O(N^2) algorithm here...)
+	for key in resolved_defines:
+		for header in real_defines:
+			if key in real_defines[header]:
+				data[header][key] = resolved_defines[key]
+
+	# Append the source code version and date
+	data['VERSION'] = {}
+	data['VERSION']['DETAILED_BUILD_VERSION'] = resolved_defines['DETAILED_BUILD_VERSION']
+	data['VERSION']['STRING_DISTRIBUTION_DATE'] = resolved_defines['STRING_DISTRIBUTION_DATE']
+	try:
+		curver = subprocess.check_output(["git", "describe", "--match=NeVeRmAtCh", "--always"]).strip()
+		data['VERSION']['GIT_REF'] = curver.decode()
+	except:
+		pass
+
+	with open(marlin_json, 'w') as outfile:
+		json.dump(data, outfile, separators=(',', ':'))
+
+	# Compress the JSON file as much as we can
+	compress_file(marlin_json, marlin_zip)
+
+	# Generate a C source file for storing this array
+	with open('Marlin/src/mczip.h','wb') as result_file:
+		result_file.write(b'#warning "Generated file \'mc.zip\' is embedded"\n')
+		result_file.write(b'const unsigned char mc_zip[] PROGMEM = {\n ')
+		count = 0
+		for b in open(os.path.join(build_dir, 'mc.zip'), 'rb').read():
+			result_file.write(b' 0x%02X,' % b)
+			count += 1
+			if (count % 16 == 0):
+			 	result_file.write(b'\n ')
+		if (count % 16):
+			result_file.write(b'\n')
+		result_file.write(b'};\n')

docs/ConfigEmbedding.md

@@ -0,0 +1,19 @@
+# Configuration Embedding
+
+Starting with version 2.0.9.3, Marlin automatically extracts the configuration used to generate the firmware and stores it in the firmware binary. This is enabled by defining `CONFIGURATION_EMBEDDING` in `Configuration_adv.h`.
+
+## How it's done
+To create the embedded configuration, we do a compiler pass to process the Configuration files and extract all active options. The active options are parsed into key/value pairs, serialized to JSON format, and stored in a file called `marlin_config.json`, which also includes specific build information (like the git revision, the build date, and some version information. The JSON file is then compressed in a ZIP archive called `.pio/build/mc.zip` which is converted into a C array and stored in a C++ file called `mc.h` which is included in the build.
+
+## Extracting configurations from a Marlin binary
+To get the configuration out of a binary firmware, you'll need a non-write-protected SD card inserted into the printer while running the firmware.
+Send the command `M503 C` to write the file `mc.zip` to the SD card. Copy the file to your computer, ideally in the same folder as the Marlin repository.
+
+Run the following commands to extract and apply the configuration:
+```
+$ git checkout -f
+$ unzip mc.zip
+$ python buildroot/share/PlatformIO/scripts/mc-apply.py
+```
+
+This will attempt to update the configuration files to match the settings used for the original build. It will also dump the git reference used to build the code (which may be accessible if the firmware was built from the main repository. As a fallback it also includes the `STRING_DISTRIBUTION_DATE` which is unlikely to be modified in a fork).