🔨 Simplify scripts with pathlib (#24574)

This commit is contained in:
Scott Lahteine
2022-08-01 01:14:58 -05:00
parent c3f2586445
commit 3b30951e83
14 changed files with 182 additions and 196 deletions

View File

@ -3,30 +3,29 @@
#
import pioutil
if pioutil.is_pio_build():
import os,shutil,marlin
from SCons.Script import DefaultEnvironment
from platformio import util
import shutil,marlin
from pathlib import Path
env = DefaultEnvironment()
Import("env")
platform = env.PioPlatform()
board = env.BoardConfig()
FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoststm32-maple")
assert os.path.isdir(FRAMEWORK_DIR)
FRAMEWORK_DIR = Path(platform.get_package_dir("framework-arduinoststm32-maple"))
assert FRAMEWORK_DIR.is_dir()
source_root = os.path.join("buildroot", "share", "PlatformIO", "variants")
assert os.path.isdir(source_root)
source_root = Path("buildroot/share/PlatformIO/variants")
assert source_root.is_dir()
variant = board.get("build.variant")
variant_dir = os.path.join(FRAMEWORK_DIR, "STM32F1", "variants", variant)
variant_dir = FRAMEWORK_DIR / "STM32F1/variants" / variant
source_dir = os.path.join(source_root, variant)
assert os.path.isdir(source_dir)
source_dir = source_root / variant
assert source_dir.is_dir()
if os.path.isdir(variant_dir):
if variant_dir.is_dir():
shutil.rmtree(variant_dir)
if not os.path.isdir(variant_dir):
os.mkdir(variant_dir)
if not variant_dir.is_dir():
variant_dir.mkdir()
marlin.copytree(source_dir, variant_dir)

View File

@ -4,9 +4,7 @@
#
import pioutil
if pioutil.is_pio_build():
import os,random,struct,uuid,marlin
# Relocate firmware from 0x08000000 to 0x08008800
marlin.relocate_firmware("0x08008800")
import struct,uuid
def calculate_crc(contents, seed):
accumulating_xor_value = seed;
@ -105,13 +103,13 @@ if pioutil.is_pio_build():
# Encrypt ${PROGNAME}.bin and save it as 'update.cbd'
def encrypt(source, target, env):
firmware = open(target[0].path, "rb")
update = open(target[0].dir.path + '/update.cbd', "wb")
length = os.path.getsize(target[0].path)
encrypt_file(firmware, update, length)
firmware.close()
update.close()
from pathlib import Path
fwpath = Path(target[0].path)
fwsize = fwpath.stat().st_size
fwfile = fwpath.open("rb")
upfile = Path(target[0].dir.path, 'update.cbd').open("wb")
encrypt_file(fwfile, upfile, fwsize)
import marlin
marlin.relocate_firmware("0x08008800")
marlin.add_post_action(encrypt);

View File

@ -5,45 +5,49 @@
import pioutil
if pioutil.is_pio_build():
Import("env")
import os,requests,zipfile,tempfile,shutil
import requests,zipfile,tempfile,shutil
from pathlib import Path
url = "https://github.com/makerbase-mks/Mks-Robin-Nano-Marlin2.0-Firmware/archive/0263cdaccf.zip"
deps_path = env.Dictionary("PROJECT_LIBDEPS_DIR")
zip_path = os.path.join(deps_path, "mks-assets.zip")
assets_path = os.path.join(env.Dictionary("PROJECT_BUILD_DIR"), env.Dictionary("PIOENV"), "assets")
deps_path = Path(env.Dictionary("PROJECT_LIBDEPS_DIR"))
zip_path = deps_path / "mks-assets.zip"
assets_path = Path(env.Dictionary("PROJECT_BUILD_DIR"), env.Dictionary("PIOENV"), "assets")
def download_mks_assets():
print("Downloading MKS Assets")
r = requests.get(url, stream=True)
# the user may have a very clean workspace,
# so create the PROJECT_LIBDEPS_DIR directory if not exits
if os.path.exists(deps_path) == False:
os.mkdir(deps_path)
with open(zip_path, 'wb') as fd:
if not deps_path.exists():
deps_path.mkdir()
with zip_path.open('wb') as fd:
for chunk in r.iter_content(chunk_size=128):
fd.write(chunk)
def copy_mks_assets():
print("Copying MKS Assets")
output_path = tempfile.mkdtemp()
output_path = Path(tempfile.mkdtemp())
zip_obj = zipfile.ZipFile(zip_path, 'r')
zip_obj.extractall(output_path)
zip_obj.close()
if os.path.exists(assets_path) == True and os.path.isdir(assets_path) == False:
os.unlink(assets_path)
if os.path.exists(assets_path) == False:
os.mkdir(assets_path)
if assets_path.exists() and not assets_path.is_dir():
assets_path.unlink()
if not assets_path.exists():
assets_path.mkdir()
base_path = ''
for filename in os.listdir(output_path):
for filename in output_path.iterdir():
base_path = filename
for filename in os.listdir(os.path.join(output_path, base_path, 'Firmware', 'mks_font')):
shutil.copy(os.path.join(output_path, base_path, 'Firmware', 'mks_font', filename), assets_path)
for filename in os.listdir(os.path.join(output_path, base_path, 'Firmware', 'mks_pic')):
shutil.copy(os.path.join(output_path, base_path, 'Firmware', 'mks_pic', filename), assets_path)
fw_path = (output_path / base_path / 'Firmware')
font_path = fw_path / 'mks_font'
for filename in font_path.iterdir():
shutil.copy(font_path / filename, assets_path)
pic_path = fw_path / 'mks_pic'
for filename in pic_path.iterdir():
shutil.copy(pic_path / filename, assets_path)
shutil.rmtree(output_path, ignore_errors=True)
if os.path.exists(zip_path) == False:
if not zip_path.exists():
download_mks_assets()
if os.path.exists(assets_path) == False:
if not assets_path.exists():
copy_mks_assets()

View File

@ -7,16 +7,14 @@
#
import pioutil
if pioutil.is_pio_build():
import os,shutil,marlin
from SCons.Script import DefaultEnvironment
from platformio import util
env = DefaultEnvironment()
import shutil,marlin
from pathlib import Path
#
# Get the platform name from the 'platform_packages' option,
# or look it up by the platform.class.name.
#
env = marlin.env
platform = env.PioPlatform()
from platformio.package.meta import PackageSpec
@ -37,8 +35,8 @@ if pioutil.is_pio_build():
if platform_name in [ "usb-host-msc", "usb-host-msc-cdc-msc", "usb-host-msc-cdc-msc-2", "usb-host-msc-cdc-msc-3", "tool-stm32duino", "biqu-bx-workaround", "main" ]:
platform_name = "framework-arduinoststm32"
FRAMEWORK_DIR = platform.get_package_dir(platform_name)
assert os.path.isdir(FRAMEWORK_DIR)
FRAMEWORK_DIR = Path(platform.get_package_dir(platform_name))
assert FRAMEWORK_DIR.is_dir()
board = env.BoardConfig()
@ -47,14 +45,14 @@ if pioutil.is_pio_build():
#series = mcu_type[:7].upper() + "xx"
# Prepare a new empty folder at the destination
variant_dir = os.path.join(FRAMEWORK_DIR, "variants", variant)
if os.path.isdir(variant_dir):
variant_dir = FRAMEWORK_DIR / "variants" / variant
if variant_dir.is_dir():
shutil.rmtree(variant_dir)
if not os.path.isdir(variant_dir):
os.mkdir(variant_dir)
if not variant_dir.is_dir():
variant_dir.mkdir()
# Source dir is a local variant sub-folder
source_dir = os.path.join("buildroot/share/PlatformIO/variants", variant)
assert os.path.isdir(source_dir)
source_dir = Path("buildroot/share/PlatformIO/variants", variant)
assert source_dir.is_dir()
marlin.copytree(source_dir, variant_dir)

View File

@ -4,37 +4,32 @@
#
import pioutil
if pioutil.is_pio_build():
import os,marlin
# Append ${PROGNAME}.bin firmware after bootloader and save it as 'jgaurora_firmware.bin'
def addboot(source, target, env):
firmware = open(target[0].path, "rb")
lengthfirmware = os.path.getsize(target[0].path)
bootloader_bin = "buildroot/share/PlatformIO/scripts/" + "jgaurora_bootloader.bin"
bootloader = open(bootloader_bin, "rb")
lengthbootloader = os.path.getsize(bootloader_bin)
from pathlib import Path
firmware_with_boothloader_bin = target[0].dir.path + '/firmware_with_bootloader.bin'
if os.path.exists(firmware_with_boothloader_bin):
os.remove(firmware_with_boothloader_bin)
firmwareimage = open(firmware_with_boothloader_bin, "wb")
position = 0
while position < lengthbootloader:
byte = bootloader.read(1)
firmwareimage.write(byte)
position += 1
position = 0
while position < lengthfirmware:
byte = firmware.read(1)
firmwareimage.write(byte)
position += 1
bootloader.close()
firmware.close()
firmwareimage.close()
fw_path = Path(target[0].path)
fwb_path = fw_path.parent / 'firmware_with_bootloader.bin'
with fwb_path.open("wb") as fwb_file:
bl_path = Path("buildroot/share/PlatformIO/scripts/jgaurora_bootloader.bin")
bl_file = bl_path.open("rb")
while True:
b = bl_file.read(1)
if b == b'': break
else: fwb_file.write(b)
firmware_without_bootloader_bin = target[0].dir.path + '/firmware_for_sd_upload.bin'
if os.path.exists(firmware_without_bootloader_bin):
os.remove(firmware_without_bootloader_bin)
os.rename(target[0].path, firmware_without_bootloader_bin)
#os.rename(target[0].dir.path+'/firmware_with_bootloader.bin', target[0].dir.path+'/firmware.bin')
with fw_path.open("rb") as fw_file:
while True:
b = fw_file.read(1)
if b == b'': break
else: fwb_file.write(b)
fws_path = Path(target[0].dir.path, 'firmware_for_sd_upload.bin')
if fws_path.exists():
fws_path.unlink()
fw_path.rename(fws_path)
import marlin
marlin.add_post_action(addboot);

View File

@ -8,10 +8,8 @@
import pioutil
if pioutil.is_pio_build():
import os,marlin
Import("env")
from SCons.Script import DefaultEnvironment
board = DefaultEnvironment().BoardConfig()
board = marlin.env.BoardConfig()
def encryptByte(byte):
byte = 0xFF & ((byte << 6) | (byte >> 2))

View File

@ -2,21 +2,18 @@
# marlin.py
# Helper module with some commonly-used functions
#
import os,shutil
import shutil
from pathlib import Path
from SCons.Script import DefaultEnvironment
env = DefaultEnvironment()
from os.path import join
def copytree(src, dst, symlinks=False, ignore=None):
for item in os.listdir(src):
s = join(src, item)
d = join(dst, item)
if os.path.isdir(s):
shutil.copytree(s, d, symlinks, ignore)
for item in src.iterdir():
if item.is_dir():
shutil.copytree(item, dst / item.name, symlinks, ignore)
else:
shutil.copy2(s, d)
shutil.copy2(item, dst / item.name)
def replace_define(field, value):
for define in env['CPPDEFINES']:
@ -34,7 +31,7 @@ def relocate_vtab(address):
# Replace the existing -Wl,-T with the given ldscript path
def custom_ld_script(ldname):
apath = os.path.abspath("buildroot/share/PlatformIO/ldscripts/" + ldname)
apath = str(Path("buildroot/share/PlatformIO/ldscripts", ldname).resolve())
for i, flag in enumerate(env["LINKFLAGS"]):
if "-Wl,-T" in flag:
env["LINKFLAGS"][i] = "-Wl,-T" + apath
@ -52,15 +49,15 @@ def encrypt_mks(source, target, env, new_name):
mf = env["MARLIN_FEATURES"]
if "FIRMWARE_BIN" in mf: new_name = mf["FIRMWARE_BIN"]
fwpath = target[0].path
fwfile = open(fwpath, "rb")
enfile = open(target[0].dir.path + "/" + new_name, "wb")
length = os.path.getsize(fwpath)
fwpath = Path(target[0].path)
fwfile = fwpath.open("rb")
enfile = Path(target[0].dir.path, new_name).open("wb")
length = fwpath.stat().st_size
position = 0
try:
while position < length:
byte = fwfile.read(1)
if position >= 320 and position < 31040:
if 320 <= position < 31040:
byte = chr(ord(byte) ^ key[position & 31])
if sys.version_info[0] > 2:
byte = bytes(byte, 'latin1')
@ -69,7 +66,7 @@ def encrypt_mks(source, target, env, new_name):
finally:
fwfile.close()
enfile.close()
os.remove(fwpath)
fwpath.unlink()
def add_post_action(action):
env.AddPostAction(join("$BUILD_DIR", "${PROGNAME}.bin"), action);
env.AddPostAction(str(Path("$BUILD_DIR", "${PROGNAME}.bin")), action);

View File

@ -10,12 +10,10 @@
#
import pioutil
if pioutil.is_pio_build():
import os,sys,marlin
Import("env")
from SCons.Script import DefaultEnvironment
board = DefaultEnvironment().BoardConfig()
import sys,marlin
env = marlin.env
board = env.BoardConfig()
board_keys = board.get("build").keys()
#
@ -56,7 +54,7 @@ if pioutil.is_pio_build():
if 'rename' in board_keys:
def rename_target(source, target, env):
firmware = os.path.join(target[0].dir.path, board.get("build.rename"))
os.replace(target[0].path, firmware)
from pathlib import Path
Path(target[0].path).replace(Path(target[0].dir.path, board.get("build.rename")))
marlin.add_post_action(rename_target)

View File

@ -6,10 +6,12 @@ import pioutil
if pioutil.is_pio_build():
import os,re,sys
from pathlib import Path
Import("env")
def get_envs_for_board(board):
with open(os.path.join("Marlin", "src", "pins", "pins.h"), "r") as file:
ppath = Path("Marlin/src/pins/pins.h")
with ppath.open() as file:
if sys.platform == 'win32':
envregex = r"(?:env|win):"
@ -77,9 +79,10 @@ if pioutil.is_pio_build():
#
# Check for Config files in two common incorrect places
#
for p in [ env['PROJECT_DIR'], os.path.join(env['PROJECT_DIR'], "config") ]:
for f in [ "Configuration.h", "Configuration_adv.h" ]:
if os.path.isfile(os.path.join(p, f)):
epath = Path(env['PROJECT_DIR'])
for p in [ epath, epath / "config" ]:
for f in ("Configuration.h", "Configuration_adv.h"):
if (p / f).is_file():
err = "ERROR: Config files found in directory %s. Please move them into the Marlin subfolder." % p
raise SystemExit(err)
@ -87,12 +90,12 @@ if pioutil.is_pio_build():
# Find the name.cpp.o or name.o and remove it
#
def rm_ofile(subdir, name):
build_dir = os.path.join(env['PROJECT_BUILD_DIR'], build_env);
for outdir in [ build_dir, os.path.join(build_dir, "debug") ]:
for ext in [ ".cpp.o", ".o" ]:
fpath = os.path.join(outdir, "src", "src", subdir, name + ext)
if os.path.exists(fpath):
os.remove(fpath)
build_dir = Path(env['PROJECT_BUILD_DIR'], build_env);
for outdir in (build_dir, build_dir / "debug"):
for ext in (".cpp.o", ".o"):
fpath = outdir / "src/src" / subdir / (name + ext)
if fpath.exists():
fpath.unlink()
#
# Give warnings on every build
@ -109,13 +112,13 @@ if pioutil.is_pio_build():
# Check for old files indicating an entangled Marlin (mixing old and new code)
#
mixedin = []
p = os.path.join(env['PROJECT_DIR'], "Marlin", "src", "lcd", "dogm")
p = Path(env['PROJECT_DIR'], "Marlin/src/lcd/dogm")
for f in [ "ultralcd_DOGM.cpp", "ultralcd_DOGM.h" ]:
if os.path.isfile(os.path.join(p, f)):
if (p / f).is_file():
mixedin += [ f ]
p = os.path.join(env['PROJECT_DIR'], "Marlin", "src", "feature", "bedlevel", "abl")
p = Path(env['PROJECT_DIR'], "Marlin/src/feature/bedlevel/abl")
for f in [ "abl.cpp", "abl.h" ]:
if os.path.isfile(os.path.join(p, f)):
if (p / f).is_file():
mixedin += [ f ]
if mixedin:
err = "ERROR: Old files fell into your Marlin folder. Remove %s and try again" % ", ".join(mixedin)

View File

@ -1,7 +1,7 @@
#
# preprocessor.py
#
import subprocess,os,re
import subprocess,re
nocache = 1
verbose = 0
@ -54,51 +54,41 @@ def run_preprocessor(env, fn=None):
#
def search_compiler(env):
ENV_BUILD_PATH = os.path.join(env['PROJECT_BUILD_DIR'], env['PIOENV'])
GCC_PATH_CACHE = os.path.join(ENV_BUILD_PATH, ".gcc_path")
from pathlib import Path, PurePath
ENV_BUILD_PATH = Path(env['PROJECT_BUILD_DIR'], env['PIOENV'])
GCC_PATH_CACHE = ENV_BUILD_PATH / ".gcc_path"
try:
filepath = env.GetProjectOption('custom_gcc')
gccpath = env.GetProjectOption('custom_gcc')
blab("Getting compiler from env")
return filepath
return gccpath
except:
pass
# Warning: The cached .gcc_path will obscure a newly-installed toolkit
if not nocache and os.path.exists(GCC_PATH_CACHE):
if not nocache and GCC_PATH_CACHE.exists():
blab("Getting g++ path from cache")
with open(GCC_PATH_CACHE, 'r') as f:
return f.read()
return GCC_PATH_CACHE.read_text()
# Find the current platform compiler by searching the $PATH
# which will be in a platformio toolchain bin folder
path_regex = re.escape(env['PROJECT_PACKAGES_DIR'])
gcc = "g++"
# Use any item in $PATH corresponding to a platformio toolchain bin folder
path_separator = ':'
gcc_exe = '*g++'
if env['PLATFORM'] == 'win32':
path_separator = ';'
path_regex += r'.*\\bin'
gcc += ".exe"
else:
path_separator = ':'
path_regex += r'/.+/bin'
gcc_exe += ".exe"
# Search for the compiler
for pathdir in env['ENV']['PATH'].split(path_separator):
if not re.search(path_regex, pathdir, re.IGNORECASE):
continue
for filepath in os.listdir(pathdir):
if not filepath.endswith(gcc):
continue
# Use entire path to not rely on env PATH
filepath = os.path.sep.join([pathdir, filepath])
# Cache the g++ path to no search always
if not nocache and os.path.exists(ENV_BUILD_PATH):
blab("Caching g++ for current env")
with open(GCC_PATH_CACHE, 'w+') as f:
f.write(filepath)
# Search for the compiler in PATH
for ppath in map(Path, env['ENV']['PATH'].split(path_separator)):
if ppath.match(env['PROJECT_PACKAGES_DIR'] + "/**/bin"):
for gpath in ppath.glob(gcc_exe):
gccpath = str(gpath.resolve())
# Cache the g++ path to no search always
if not nocache and ENV_BUILD_PATH.exists():
blab("Caching g++ for current env")
GCC_PATH_CACHE.write_text(gccpath)
return gccpath
return filepath
filepath = env.get('CXX')
blab("Couldn't find a compiler! Fallback to %s" % filepath)
return filepath
gccpath = env.get('CXX')
blab("Couldn't find a compiler! Fallback to %s" % gccpath)
return gccpath