From 32b6d1a04e2e612bb6c4fff5d245daf74b03f1de Mon Sep 17 00:00:00 2001 From: nmstudioo Date: Fri, 13 Nov 2026 01:18:39 +0600 Subject: [PATCH 1/2] feat(dsp): add validated NextGEN IRS tail-trim candidate --- .github/workflows/ci.yml | 10 + build/generator/build_irs_nextgen.py | 591 +++++++++++++++++++++++++++ 2 files changed, 601 insertions(+) create mode 100644 build/generator/build_irs_nextgen.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c3e81b..b679d27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,16 @@ jobs: run: python3 build/build.py --out /tmp/rel - name: Validate acceptance result (fresh or restored from cache) run: python3 tests/check_acceptance.py /tmp/rel/acceptance.json + - name: NextGEN IRS builder + run: python3 build/generator/build_irs_nextgen.py + + - name: Upload NextGEN IRS artifact + uses: actions/upload-artifact@v7 + with: + name: nextgen-irs + path: build/generator/out/iem_nextgen/ + if-no-files-found: error + - name: Generator reproduction vs releases/5.0.0 (informational) if: needs.prep.outputs.gen == 'true' continue-on-error: true diff --git a/build/generator/build_irs_nextgen.py b/build/generator/build_irs_nextgen.py new file mode 100644 index 0000000..ba64f4b --- /dev/null +++ b/build/generator/build_irs_nextgen.py @@ -0,0 +1,591 @@ +import os +import struct +import hashlib +import json +from pathlib import Path + +import numpy as np + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent.parent + +SRC_IRS = ROOT / "src/profiles/iem-hd/EDX_HD.irs" +OUT_DIR = HERE / "out/iem_nextgen" +OUT_IRS = OUT_DIR / "EDX_HD_NextGEN_candidate.irs" +OUT_JSON = OUT_DIR / "EDX_HD_NextGEN_candidate_irs.json" + +EXPECTED_SHA = "cf4aa68cbc005b50960d6c91392829f6fa3bbf7f385bf1c54a8f948968cc0620" + +KEEP_FRAMES = 1357 +TAIL_THRESHOLD_DB = -80.0 + +PEAK_LIMIT = 0.95 +L1_LIMIT = 1.25 + +RESPONSE_RMS_LIMIT = 0.1 +RESPONSE_MAX_LIMIT = 0.25 + +CROSSFEED_MAG_LIMIT_DB = 0.5 +CROSSFEED_PHASE_LIMIT_DEG = 0.1 + +SYMMETRY_LIMIT = 1e-7 + +NULL_THRESHOLD_REL = 1e-6 +MAX_EXCLUDED_BINS = 12 + +SAMPLE_RATE = 48000 +CHANNELS = 4 +BITS = 32 +BLOCK_ALIGN = 16 + +FREQUENCIES = np.logspace( + np.log10(20.0), + np.log10(20000.0), + 1200 +) + +BANDS = [ + (20, 100), + (100, 500), + (500, 2000), + (2000, 6000), + (6000, 10000), + (10000, 20000), +] + + +def sha256_bytes(data): + return hashlib.sha256(data).hexdigest() + + +def parse_canonical_wav(data): + if len(data) < 44: + raise ValueError("WAV too short") + + if data[0:4] != b"RIFF" or data[8:12] != b"WAVE": + raise ValueError("Invalid RIFF/WAVE") + + riff_size = struct.unpack_from(" original_frames: + raise ValueError("Candidate exceeds source frame count") + + original_payload = source_bytes[44:] + + expected_payload = original_payload[ + :KEEP_FRAMES * BLOCK_ALIGN + ] + + original = np.frombuffer( + original_payload, + dtype=" PEAK_LIMIT: + raise ValueError(f"Peak {peak:.6f} > {PEAK_LIMIT}") + + if l1 > L1_LIMIT: + raise ValueError(f"L1 {l1:.6f} > {L1_LIMIT}") + + # Tail-energy threshold, measured against the full source. + energy = np.sum( + original.astype(np.float64) ** 2, + axis=1, + ) + + total_energy = float(np.sum(energy)) + + if not np.isfinite(total_energy) or total_energy <= 0: + raise ValueError("Invalid source energy") + + remaining = np.cumsum(energy[::-1])[::-1] + + tail_db = 10.0 * np.log10( + remaining / total_energy + 1e-30 + ) + + crossings = np.flatnonzero( + tail_db < TAIL_THRESHOLD_DB + ) + + tail_frame = ( + int(crossings[0]) + if len(crossings) + else original_frames + ) + + if tail_frame >= KEEP_FRAMES: + raise ValueError( + f"Tail threshold not reached before frame {KEEP_FRAMES}" + ) + + # Composite response comparison. + H_orig = direct_dtft( + original[:, 0] + original[:, 2], + FREQUENCIES, + SAMPLE_RATE, + ) + + H_cand = direct_dtft( + candidate[:, 0] + candidate[:, 2], + FREQUENCIES, + SAMPLE_RATE, + ) + + mag_orig = np.abs(H_orig) + mag_cand = np.abs(H_cand) + + ref_peak = float(np.max(mag_orig)) + + if not np.isfinite(ref_peak) or ref_peak <= 0: + raise ValueError("Invalid reference response peak") + + null_floor = ref_peak * NULL_THRESHOLD_REL + + valid = mag_orig > null_floor + excluded_idx = np.flatnonzero(~valid) + + if len(excluded_idx) > MAX_EXCLUDED_BINS: + raise ValueError( + f"Excluded {len(excluded_idx)} bins; " + f"maximum allowed is {MAX_EXCLUDED_BINS}" + ) + + # Inspect candidate behaviour at excluded reference-null bins. + excluded_candidate_metrics = [] + + for idx in excluded_idx: + original_mag = float(mag_orig[idx]) + candidate_mag = float(mag_cand[idx]) + + if candidate_mag > null_floor * 10: + raise ValueError( + f"New energy at excluded reference-null bin " + f"{FREQUENCIES[idx]:.2f} Hz" + ) + + excluded_candidate_metrics.append({ + "frequency_hz": round(float(FREQUENCIES[idx]), 2), + "original_magnitude": original_mag, + "candidate_magnitude": candidate_mag, + "candidate_below_relative_floor": ( + candidate_mag <= null_floor + ), + }) + + if not np.any(valid): + raise ValueError("No valid response-comparison bins") + + db_diff = 20.0 * np.log10( + mag_cand[valid] / mag_orig[valid] + 1e-30 + ) + + response_rms = float( + np.sqrt(np.mean(db_diff ** 2)) + ) + + response_max = float( + np.max(np.abs(db_diff)) + ) + + if response_rms > RESPONSE_RMS_LIMIT: + raise ValueError( + f"Response RMS {response_rms:.6f} exceeds limit" + ) + + if response_max > RESPONSE_MAX_LIMIT: + raise ValueError( + f"Response max {response_max:.6f} exceeds limit" + ) + + per_band = {} + + for low, high in BANDS: + mask = ( + (FREQUENCIES >= low) + & (FREQUENCIES < high) + & valid + ) + + if np.any(mask): + band_diff = 20.0 * np.log10( + mag_cand[mask] / mag_orig[mask] + 1e-30 + ) + + per_band[f"{low}-{high}Hz"] = float( + np.sqrt(np.mean(band_diff ** 2)) + ) + + # Crossfeed magnitude and phase preservation. + cf1 = crossfeed_metrics(original, candidate, 0, 1) + cf2 = crossfeed_metrics(original, candidate, 3, 2) + + for label, metrics in ( + ("channel_1_over_0", cf1), + ("channel_2_over_3", cf2), + ): + if metrics["magnitude_delta_db"] > CROSSFEED_MAG_LIMIT_DB: + raise ValueError(f"{label}: crossfeed magnitude drift") + + if metrics["phase_delta_deg"] > CROSSFEED_PHASE_LIMIT_DEG: + raise ValueError(f"{label}: crossfeed phase drift") + + # Channel symmetry. + sym_o = channel_symmetry(original) + sym_n = channel_symmetry(candidate) + + for label, metrics in ( + ("original", sym_o), + ("candidate", sym_n), + ): + if max(metrics.values()) > SYMMETRY_LIMIT: + raise ValueError(f"{label}: channel symmetry failed") + + gates = { + key: "pass" + for key in [ + "source_hash", + "canonical_source_wav", + "canonical_candidate_wav", + "frame_alignment", + "retained_payload_byte_identical", + "finite_source_and_candidate", + "peak", + "l1_norm", + "tail_threshold", + "response_rms", + "response_max", + "crossfeed_1_magnitude", + "crossfeed_1_phase", + "crossfeed_2_magnitude", + "crossfeed_2_phase", + "original_channel_symmetry", + "candidate_channel_symmetry", + "no_new_energy_at_excluded_bins", + ] + } + + report = { + "status": "candidate_validated", + "source": { + "path": str(SRC_IRS.relative_to(ROOT)), + "sha256": EXPECTED_SHA, + }, + "candidate": { + "filename": OUT_IRS.name, + "sha256": sha256_bytes(candidate_bytes), + "frames": KEEP_FRAMES, + "duration_ms": round( + KEEP_FRAMES / SAMPLE_RATE * 1000, + 4, + ), + "bytes": len(candidate_bytes), + "reduction_pct": round( + ( + 1 - len(expected_payload) / len(original_payload) + ) * 100, + 2, + ), + }, + "format": { + "encoding": "IEEE Float32", + "channels": CHANNELS, + "sample_rate": SAMPLE_RATE, + "bits": BITS, + "block_align": BLOCK_ALIGN, + }, + "tail": { + "threshold_db": TAIL_THRESHOLD_DB, + "crossing_frame": tail_frame, + "crossing_energy_db": float(tail_db[tail_frame]), + }, + "sample_metrics": { + "peak": peak, + "l1_norm": l1, + "peak_limit": PEAK_LIMIT, + "l1_limit": L1_LIMIT, + }, + "response": { + "method": "chunked direct DTFT", + "grid": "1200 logarithmic points, 20-20000 Hz", + "null_threshold_relative": NULL_THRESHOLD_REL, + "null_floor": null_floor, + "excluded_bins": len(excluded_idx), + "excluded_frequencies_hz": [ + round(float(FREQUENCIES[i]), 2) + for i in excluded_idx + ], + "excluded_bin_details": excluded_candidate_metrics, + "maximum_excluded_bins": MAX_EXCLUDED_BINS, + "rms_difference_db": response_rms, + "rms_limit_db": RESPONSE_RMS_LIMIT, + "max_absolute_difference_db": response_max, + "max_limit_db": RESPONSE_MAX_LIMIT, + "per_band_rms_db": per_band, + }, + "crossfeed_500hz": { + "channel_1_over_0": cf1, + "channel_2_over_3": cf2, + "magnitude_delta_limit_db": CROSSFEED_MAG_LIMIT_DB, + "phase_delta_limit_deg": CROSSFEED_PHASE_LIMIT_DEG, + }, + "channel_symmetry": { + "original": sym_o, + "candidate": sym_n, + "tolerance": SYMMETRY_LIMIT, + }, + "gates": gates, + } + + OUT_DIR.mkdir(parents=True, exist_ok=True) + + tmp_irs = OUT_DIR / (OUT_IRS.name + ".tmp") + tmp_json = OUT_DIR / (OUT_JSON.name + ".tmp") + + try: + tmp_irs.write_bytes(candidate_bytes) + tmp_json.write_text( + json.dumps(report, indent=2) + "\n" + ) + + # Independently reopen and validate the written candidate. + written = tmp_irs.read_bytes() + + if sha256_bytes(written) != report["candidate"]["sha256"]: + raise ValueError("Written candidate SHA mismatch") + + written_info = parse_canonical_wav(written) + + if written_info["frames"] != KEEP_FRAMES: + raise ValueError("Written candidate frame count mismatch") + + if written[44:] != expected_payload: + raise ValueError("Written candidate payload mismatch") + + os.replace(tmp_irs, OUT_IRS) + os.replace(tmp_json, OUT_JSON) + + finally: + if tmp_irs.exists(): + tmp_irs.unlink() + + if tmp_json.exists(): + tmp_json.unlink() + + print(f"IRS candidate: {OUT_IRS}") + print(f"Frames: {KEEP_FRAMES}") + print(f"Response RMS: {response_rms:.6f} dB") + print(f"Response max: {response_max:.6f} dB") + print(f"Excluded bins: {len(excluded_idx)}") + print(f"Excluded frequencies: {report['response']['excluded_frequencies_hz']}") + print(f"Crossfeed magnitude deltas: " + f"{cf1['magnitude_delta_db']:.6f}, " + f"{cf2['magnitude_delta_db']:.6f} dB") + + +if __name__ == "__main__": + main() From 2a5fa1dab9879f76dbb9cbdb432058473222f330 Mon Sep 17 00:00:00 2001 From: nmstudioo Date: Fri, 13 Nov 2026 01:30:33 +0600 Subject: [PATCH 2/2] fix(dsp): coerce freqs to numpy array in direct_dtft (list*float TypeError) --- build/generator/build_irs_nextgen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build/generator/build_irs_nextgen.py b/build/generator/build_irs_nextgen.py index ba64f4b..2b6e650 100644 --- a/build/generator/build_irs_nextgen.py +++ b/build/generator/build_irs_nextgen.py @@ -144,6 +144,7 @@ def make_wav_header(data_size): def direct_dtft(signal, freqs, sr, chunk=32): sig = np.asarray(signal, dtype=np.float64) + freqs = np.asarray(freqs, dtype=np.float64) n = np.arange(len(sig), dtype=np.float64) out = np.empty(len(freqs), dtype=np.complex128)