alexei.dolgolyov
feb91ad281
Lint & Test / test (push) Successful in 2m6s
The 'cmd <<EOF || { ... }' pattern confuses bash's parser — the closing
brace of the inline error block collides with the function's closing
brace. Rewrote to capture the python script into a local var via
$(cat <<EOF), then run it with -c and a plain 'if !' guard.
274 lines
11 KiB
Bash
274 lines
11 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# Shared build functions for LedGrab distribution packaging.
|
|
# Sourced by build-dist.sh (Linux) and build-dist-windows.sh (Windows).
|
|
#
|
|
# Expected variables set by the caller before sourcing:
|
|
# SCRIPT_DIR, BUILD_DIR, DIST_DIR, SERVER_DIR, APP_DIR
|
|
|
|
# ── Version detection ────────────────────────────────────────
|
|
|
|
detect_version() {
|
|
# Usage: detect_version [explicit_version]
|
|
local version="${1:-}"
|
|
|
|
if [ -z "$version" ]; then
|
|
version=$(git describe --tags --exact-match 2>/dev/null || true)
|
|
fi
|
|
if [ -z "$version" ]; then
|
|
version="${GITEA_REF_NAME:-${GITHUB_REF_NAME:-}}"
|
|
fi
|
|
if [ -z "$version" ]; then
|
|
version=$(grep -oP '^version\s*=\s*"\K[^"]+' "$SERVER_DIR/pyproject.toml" 2>/dev/null || echo "0.0.0")
|
|
fi
|
|
|
|
VERSION_CLEAN="${version#v}"
|
|
|
|
# Normalize non-PEP440 version labels (e.g. "dev", "nightly", "snapshot")
|
|
# to a valid PEP440 dev release. Without this, pip/setuptools rejects the
|
|
# pyproject.toml with: `project.version` must be pep440.
|
|
if ! [[ "$VERSION_CLEAN" =~ ^[0-9]+(\.[0-9]+)*((a|b|rc|\.dev|\.post)[0-9]+)*(\+[a-zA-Z0-9.]+)?$ ]]; then
|
|
echo " Warning: '$VERSION_CLEAN' is not PEP440-compliant, using 0.0.0.dev0"
|
|
VERSION_CLEAN="0.0.0.dev0"
|
|
fi
|
|
|
|
# Stamp the resolved version into pyproject.toml so that
|
|
# importlib.metadata reads the correct value at runtime.
|
|
sed -i "s/^version = .*/version = \"${VERSION_CLEAN}\"/" "$SERVER_DIR/pyproject.toml"
|
|
}
|
|
|
|
# ── Clean previous build ─────────────────────────────────────
|
|
|
|
clean_dist() {
|
|
if [ -d "$DIST_DIR" ]; then
|
|
echo " Cleaning previous build..."
|
|
rm -rf "$DIST_DIR"
|
|
fi
|
|
mkdir -p "$DIST_DIR"
|
|
}
|
|
|
|
# ── Build frontend ───────────────────────────────────────────
|
|
|
|
build_frontend() {
|
|
echo " Building frontend bundle..."
|
|
(cd "$SERVER_DIR" && npm ci --loglevel error && npm run build) 2>&1 | {
|
|
grep -v 'RemoteException' || true
|
|
}
|
|
}
|
|
|
|
# ── Copy application files ───────────────────────────────────
|
|
|
|
copy_app_files() {
|
|
echo " Copying application files..."
|
|
mkdir -p "$APP_DIR"
|
|
|
|
cp -r "$SERVER_DIR/src" "$APP_DIR/src"
|
|
cp -r "$SERVER_DIR/config" "$APP_DIR/config"
|
|
mkdir -p "$DIST_DIR/data" "$DIST_DIR/logs"
|
|
|
|
# Clean up source maps and __pycache__
|
|
find "$APP_DIR" -name "*.map" -delete 2>/dev/null || true
|
|
find "$APP_DIR" -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
|
|
}
|
|
|
|
# ── Site-packages cleanup ────────────────────────────────────
|
|
#
|
|
# Strips tests, type stubs, unused submodules, and debug symbols
|
|
# from the installed site-packages directory.
|
|
#
|
|
# Args:
|
|
# $1 — path to site-packages directory
|
|
# $2 — native extension suffix: "pyd" (Windows) or "so" (Linux)
|
|
# $3 — native lib suffix for OpenCV ffmpeg: "dll" or "so"
|
|
|
|
cleanup_site_packages() {
|
|
local sp_dir="$1"
|
|
local ext_suffix="${2:-so}"
|
|
local lib_suffix="${3:-so}"
|
|
|
|
echo " Cleaning up site-packages to reduce size..."
|
|
|
|
# ── Generic cleanup ──────────────────────────────────────
|
|
find "$sp_dir" -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
|
|
find "$sp_dir" -type d -name tests -exec rm -rf {} + 2>/dev/null || true
|
|
find "$sp_dir" -type d -name test -exec rm -rf {} + 2>/dev/null || true
|
|
find "$sp_dir" -type d -name "*.dist-info" -exec rm -rf {} + 2>/dev/null || true
|
|
find "$sp_dir" -name "*.pyi" -delete 2>/dev/null || true
|
|
|
|
# ── pip / setuptools (not needed at runtime) ─────────────
|
|
rm -rf "$sp_dir"/pip "$sp_dir"/pip-* 2>/dev/null || true
|
|
rm -rf "$sp_dir"/setuptools "$sp_dir"/setuptools-* "$sp_dir"/pkg_resources 2>/dev/null || true
|
|
rm -rf "$sp_dir"/_distutils_hack 2>/dev/null || true
|
|
|
|
# ── OpenCV ───────────────────────────────────────────────
|
|
local cv2_dir="$sp_dir/cv2"
|
|
if [ -d "$cv2_dir" ]; then
|
|
# Remove ffmpeg (28 MB on Windows), Haar cascades, dev files
|
|
rm -f "$cv2_dir"/opencv_videoio_ffmpeg*."$lib_suffix" 2>/dev/null || true
|
|
rm -rf "$cv2_dir/data" "$cv2_dir/gapi" "$cv2_dir/misc" "$cv2_dir/utils" 2>/dev/null || true
|
|
rm -rf "$cv2_dir/typing_stubs" "$cv2_dir/typing" 2>/dev/null || true
|
|
fi
|
|
|
|
# ── NumPy ────────────────────────────────────────────────
|
|
# Only strip modules that are safely unused by numpy's own import chain.
|
|
# DO NOT strip: lib, linalg, ma, matrixlib — numpy.__init__ imports them
|
|
# transitively (e.g. matrixlib → defmatrix → linalg), so removing any of
|
|
# these breaks `import numpy` itself, cascading into every downstream
|
|
# module. Learned the hard way in the v0.0.0.dev0 Windows build.
|
|
for mod in polynomial distutils f2py typing _pyinstaller; do
|
|
rm -rf "$sp_dir/numpy/$mod" 2>/dev/null || true
|
|
done
|
|
rm -rf "$sp_dir/numpy/tests" "$sp_dir/numpy/*/tests" 2>/dev/null || true
|
|
|
|
# ── Pillow (only used for system tray icon) ──────────────
|
|
rm -rf "$sp_dir/PIL/tests" 2>/dev/null || true
|
|
# Remove unused image format plugins (keep JPEG, PNG, ICO, BMP)
|
|
for plugin in Eps Gif Tiff Webp Psd Pcx Xbm Xpm Dds Ftex Gbr Grib \
|
|
Icns Im Imt Iptc McIrdas Mpo Msp Pcd Pixar Ppm Sgi \
|
|
Spider Sun Tga Wal Wmf; do
|
|
rm -f "$sp_dir/PIL/${plugin}ImagePlugin.py" 2>/dev/null || true
|
|
rm -f "$sp_dir/PIL/${plugin}ImagePlugin.pyc" 2>/dev/null || true
|
|
done
|
|
|
|
# ── zeroconf ─────────────────────────────────────────────
|
|
# DO NOT strip zeroconf/_services — the compiled Cython _listener.pyd
|
|
# imports from it, and the import fails at runtime with:
|
|
# ModuleNotFoundError: No module named 'zeroconf._services'
|
|
# Same class of bug as numpy — "presumed unused" submodule is actually
|
|
# imported internally by the package's own compiled code.
|
|
|
|
# ── Strip debug symbols ──────────────────────────────────
|
|
if command -v strip &>/dev/null; then
|
|
echo " Stripping debug symbols from .$ext_suffix files..."
|
|
find "$sp_dir" -name "*.$ext_suffix" -exec strip --strip-debug {} \; 2>/dev/null || true
|
|
fi
|
|
|
|
# ── Remove wled_controller if pip-installed ───────────────
|
|
rm -rf "$sp_dir"/wled_controller* "$sp_dir"/wled*.dist-info 2>/dev/null || true
|
|
|
|
local cleaned_size
|
|
cleaned_size=$(du -sh "$sp_dir" | cut -f1)
|
|
echo " Site-packages after cleanup: $cleaned_size"
|
|
}
|
|
|
|
# ── Pre-compile .py → .pyc (keep sources) ────────────────────
|
|
#
|
|
# MUST run AFTER cleanup_site_packages. Speeds up first startup by
|
|
# producing .pyc alongside .py. We deliberately do NOT delete .py
|
|
# sources afterwards:
|
|
#
|
|
# 1. OpenCV's loader does literal file I/O on cv2/config.py (not
|
|
# an import) — stripping it breaks `import cv2` with:
|
|
# "OpenCV loader: missing configuration file: ['config.py']".
|
|
# 2. Other packages may do similar tricks (inspect.getsource,
|
|
# runtime introspection, __file__-relative data loading).
|
|
# 3. The size saving (~30%) isn't worth the whack-a-mole of
|
|
# shipping broken installers. We already hit this with
|
|
# numpy.linalg and zeroconf._services — enough incidents.
|
|
#
|
|
# Args:
|
|
# $1 — directory to compile (site-packages or app/src)
|
|
# $2 — python executable to use (default: python3)
|
|
|
|
compile_and_strip_sources() {
|
|
local target_dir="$1"
|
|
local py_cmd="${2:-python3}"
|
|
|
|
if [ ! -d "$target_dir" ]; then
|
|
return 0
|
|
fi
|
|
|
|
echo " Pre-compiling Python bytecode in $(basename "$target_dir")..."
|
|
"$py_cmd" -m compileall -b -q "$target_dir" 2>/dev/null || {
|
|
echo " ERROR: compileall failed for $target_dir — aborting"
|
|
return 1
|
|
}
|
|
# Drop __pycache__ to save the duplicated PEP-3147 copies; the
|
|
# `-b` flag above placed legacy .pyc next to each .py, so nothing
|
|
# of value is lost here.
|
|
find "$target_dir" -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
|
|
}
|
|
|
|
# ── Import smoke test ────────────────────────────────────────
|
|
#
|
|
# Verifies that every top-level dependency that wled_controller actually
|
|
# uses can be imported from the stripped site-packages. Catches regressions
|
|
# where cleanup_site_packages removes a submodule that turns out to be
|
|
# imported internally by the package (e.g. numpy.linalg, zeroconf._services).
|
|
# Failing here is cheap; failing on a user's machine after install is not.
|
|
#
|
|
# Args:
|
|
# $1 — path to site-packages to test against
|
|
# $2 — python executable
|
|
# $3 — (optional) extra PYTHONPATH entry (e.g. app/src for wled_controller)
|
|
|
|
smoke_test_imports() {
|
|
local sp_dir="$1"
|
|
local py_cmd="${2:-python3}"
|
|
local extra_path="${3:-}"
|
|
|
|
echo " Running import smoke test..."
|
|
local pypath="$sp_dir"
|
|
if [ -n "$extra_path" ]; then
|
|
pypath="$extra_path:$sp_dir"
|
|
fi
|
|
|
|
# Modules that MUST import cleanly IF PRESENT. We don't enforce
|
|
# installation — Pillow for example is only a Windows dep. But if a
|
|
# module's top-level package dir exists in site-packages and we
|
|
# can't import it, that's a broken install and we abort.
|
|
local smoke_script
|
|
smoke_script=$(cat <<'PYEOF'
|
|
import importlib
|
|
import os
|
|
import sys
|
|
|
|
sp_dir = sys.argv[1]
|
|
|
|
# (module_name, site-packages path to check for presence)
|
|
candidates = [
|
|
('numpy', 'numpy'),
|
|
('numpy.linalg', 'numpy/linalg'),
|
|
('numpy.lib', 'numpy/lib'),
|
|
('numpy.matrixlib', 'numpy/matrixlib'),
|
|
('cv2', 'cv2'),
|
|
('fastapi', 'fastapi'),
|
|
('uvicorn', 'uvicorn'),
|
|
('starlette', 'starlette'),
|
|
('pydantic', 'pydantic'),
|
|
('zeroconf', 'zeroconf'),
|
|
('zeroconf._services', 'zeroconf/_services'),
|
|
('PIL', 'PIL'),
|
|
('PIL.Image', 'PIL'),
|
|
('yaml', 'yaml'),
|
|
]
|
|
|
|
tested = 0
|
|
skipped = 0
|
|
failed = []
|
|
for mod, path in candidates:
|
|
if not os.path.exists(os.path.join(sp_dir, path)):
|
|
skipped += 1
|
|
continue
|
|
try:
|
|
importlib.import_module(mod)
|
|
tested += 1
|
|
except Exception as e:
|
|
failed.append(f'{mod}: {type(e).__name__}: {e}')
|
|
|
|
if failed:
|
|
print('SMOKE TEST FAILED:', file=sys.stderr)
|
|
for f in failed:
|
|
print(f' {f}', file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
print(f' Smoke test passed ({tested} imported, {skipped} not installed)')
|
|
PYEOF
|
|
)
|
|
|
|
if ! PYTHONPATH="$pypath" "$py_cmd" -c "$smoke_script" "$sp_dir"; then
|
|
echo " ERROR: smoke test failed — site-packages is broken, aborting build"
|
|
return 1
|
|
fi
|
|
}
|