From 8017954aa1b6a6b6be5c81d24ad04c6589453b22 Mon Sep 17 00:00:00 2001 From: Tyler Date: Wed, 12 Aug 2026 12:50:16 -0700 Subject: [PATCH] fix: render the spectrogram from the visible segment range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The source image was sized at one column per STFT segment with no upper bound. That is fine for short captures but impossible for long ones: a 5.7-hour file at 12 kHz yields ~478k segments, i.e. a 478450x1025 RGBA image (~2 GB) roughly 29x past the ~16k-per-dimension texture limit every GL implementation enforces. The upload failed, the texture id stayed 0, and the draw was skipped — so the spectrogram was simply blank, with no error anywhere to say why. Build the image for the segment range actually on screen instead of the whole file, capped at MAX_SPECTRO_IMAGE_WIDTH. Simply clamping the full-file width would have fixed the blank render while permanently discarding the detail zooming is meant to reveal (478k segments into 8k columns is 59:1, no matter how far in you go). Tying the range to the view keeps resolution proportional to zoom: 59 segments per column at full zoom-out, reaching 1:1 by ~1% zoom, with the image never exceeding ~33 MB. Where several segments do share a column their per-bin MAX is kept rather than a sum or mean, so a short burst lights its column instead of being diluted by quiet neighbours — the same reasoning behind a min/max waveform envelope. Amplitude normalisation is likewise scoped to the visible span, which keeps rebuild cost independent of total duration and lets the colour scale follow what is on screen instead of a loud burst hours away. Rebuilds trigger when the view leaves the cached range, which is padded by 25% so ordinary panning re-renders about once every six frames rather than every frame. The headless --render path sets no range and so still covers the whole file, capped; verified end to end on a 5.67-hour capture, which now renders 8110x1025 with visible structure where it previously produced nothing at all. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01V8ZWfr5XZyyDttvkhJUgHN --- src/render.c | 68 ++++++++++++++++++++++++++++++++++++----- src/spectrogram.c | 59 ++++++++++++++++++++++++++++++++--- src/spectrogram_types.h | 17 +++++++++++ 3 files changed, 132 insertions(+), 12 deletions(-) diff --git a/src/render.c b/src/render.c index 8a7e4c5..7b31c9e 100644 --- a/src/render.c +++ b/src/render.c @@ -191,21 +191,58 @@ void GenerateColormapTexture(void) static void ComputeSpectrogramReassignment(StftResult* stft) { if (stft->numSegments == 0) return; - int width = stft->numSegments; int height = stft->segments[0].numBins; int fftSize = (height - 1) * 2; float freqPerBin = (float)stft->sampleRate / fftSize; + // One image column per STFT segment is fine for short files, but a long + // capture has far more segments than any texture can hold: a 5.7-hour file + // at 12 kHz yields ~478k segments, i.e. a 478000x1025 RGBA image (~2 GB) + // that blows past the ~16k GPU texture limit. The upload then fails, the + // texture id stays 0, and the spectrogram silently renders as nothing. + // + // Only the segments currently on screen are rendered, and the width is + // capped. Building the whole file at once and cropping afterwards would + // either exceed the texture limit (as above) or, if globally downsampled, + // permanently throw away the detail that zooming in is supposed to reveal. + // Restricting to the visible span keeps resolution tied to the zoom level: + // the further in you go, the fewer segments share a column. + int segFirst = app.reassignSegFirst; + int segLast = app.reassignSegLast; + if (segFirst < 0) segFirst = 0; + if (segLast > stft->numSegments) segLast = stft->numSegments; + if (segLast <= segFirst) { segFirst = 0; segLast = stft->numSegments; } + int segSpan = segLast - segFirst; + + // Fold multiple segments into each column when the span still exceeds the + // cap, keeping the per-bin MAX (not a mean) so a short burst lights its + // column up instead of being averaged into the noise floor — the same + // reason the waveform scope draws min/max rather than decimated samples. + int segsPerCol = (segSpan + MAX_SPECTRO_IMAGE_WIDTH - 1) / MAX_SPECTRO_IMAGE_WIDTH; + if (segsPerCol < 1) segsPerCol = 1; + int width = (segSpan + segsPerCol - 1) / segsPerCol; + if (width < 1) width = 1; + // (Re)allocate the cached accumulation buffer for reassigned energy. free(app.reassignBuffer); - app.reassignBuffer = (float*)calloc(width * height, sizeof(float)); + app.reassignBuffer = (float*)calloc((size_t)width * height, sizeof(float)); + if (app.reassignBuffer == NULL) { + app.reassignWidth = 0; + app.reassignHeight = 0; + return; + } app.reassignWidth = width; app.reassignHeight = height; + app.reassignSegsPerCol = segsPerCol; float* accumBuffer = app.reassignBuffer; // Find max amplitude for normalization (skip NULL segments) + // Normalize against the visible span only — scanning the whole file would + // put the cost back on total duration, which is what this range-limited + // rebuild exists to avoid. It also means the colour scale adapts to what + // is on screen rather than to a loud burst somewhere else in the capture. float maxAmplitude = 0.0001f; - for (int seg = 0; seg < stft->numSegments; seg++) { + for (int seg = segFirst; seg < segLast; seg++) { if (stft->segments[seg].spectrum == NULL) continue; for (int bin = 0; bin < stft->segments[seg].numBins; bin++) if (stft->segments[seg].spectrum[bin].amplitude > maxAmplitude) @@ -215,10 +252,15 @@ static void ComputeSpectrogramReassignment(StftResult* stft) // Noise threshold: only reassign bins with significant energy float noiseThreshold = maxAmplitude * 0.01f; // 1% of max amplitude - for (int seg = 0; seg < width; seg++) { + for (int seg = segFirst; seg < segLast; seg++) { // Skip segments that haven't been computed yet (overview/high-res transition) if (stft->segments[seg].spectrum == NULL) continue; + // Column this segment lands in, relative to the start of the range. + int col = (seg - segFirst) / segsPerCol; + if (col >= width) col = width - 1; + if (col < 0) col = 0; + for (int bin = 0; bin < height; bin++) { FrequencyData* V_f = &stft->segments[seg].spectrum[bin]; FrequencyData* V_fd = &stft->segments[seg].derivativeSpectrum[bin]; @@ -264,11 +306,21 @@ static void ComputeSpectrogramReassignment(StftResult* stft) if (bin1 >= height) bin1 = height - 1; float frac = targetBinF - bin0; - int idx0 = (height - 1 - bin0) * width + seg; - int idx1 = (height - 1 - bin1) * width + seg; + int idx0 = (height - 1 - bin0) * width + col; + int idx1 = (height - 1 - bin1) * width + col; - accumBuffer[idx0] += amplitude * (1 - frac); - accumBuffer[idx1] += amplitude * frac; + // Within a column the bilinear splat accumulates as before. Across + // segments folded into one column take the max, so a brief loud + // burst isn't diluted by its quiet neighbours. + float e0 = amplitude * (1 - frac); + float e1 = amplitude * frac; + if (segsPerCol == 1) { + accumBuffer[idx0] += e0; + accumBuffer[idx1] += e1; + } else { + if (e0 > accumBuffer[idx0]) accumBuffer[idx0] = e0; + if (e1 > accumBuffer[idx1]) accumBuffer[idx1] = e1; + } } } } diff --git a/src/spectrogram.c b/src/spectrogram.c index f7a3669..b49a043 100644 --- a/src/spectrogram.c +++ b/src/spectrogram.c @@ -234,7 +234,10 @@ void ResetForNewSignal(void) app.selectedAnnotation = -1; // Indices point into the events array we just freed. app.hoverStackCount = 0; - app.hoverStackPinned = false; + // Segment range belongs to the previous file's STFT; 0/0 means "whole file" + // and lets the first rebuild pick the range for the new one. + app.reassignSegFirst = 0; + app.reassignSegLast = 0; app.autocropPending = true; // run once when this file's STFT is ready } @@ -1567,12 +1570,60 @@ int main(int argc, char* argv[]) // Draw spectrogram (background, in its own area) if (app.loaded && app.stftComputed) { + // Rebuild the source image whenever the view moves outside the + // segment range it was built for. The image covers the visible span + // (plus margin) rather than the whole file — see + // ComputeSpectrogramReassignment — so this is what keeps on-screen + // resolution tied to the zoom level instead of to total duration. + if (app.stft.numSegments > 0) { + int want0 = (int)(app.view.start * app.stft.numSegments); + int want1 = (int)ceilf(app.view.end * app.stft.numSegments); + // Margin so small pans don't re-render every frame. + int margin = (want1 - want0) / 4; + want0 -= margin; want1 += margin; + if (want0 < 0) want0 = 0; + if (want1 > app.stft.numSegments) want1 = app.stft.numSegments; + if (want1 <= want0) want1 = want0 + 1; + + bool needRebuild = app.reassignBuffer == NULL || + want0 < app.reassignSegFirst || + want1 > app.reassignSegLast; + // Also re-render once the view has zoomed in far enough that the + // cached image is being magnified — otherwise a deep zoom keeps + // stretching the same columns instead of resolving new detail. + if (!needRebuild && app.reassignSegsPerCol > 1) { + int visSegs = want1 - want0; + int visCols = visSegs / app.reassignSegsPerCol; + if (visCols < MAX_SPECTRO_IMAGE_WIDTH / 4) needRebuild = true; + } + if (needRebuild) { + app.reassignSegFirst = want0; + app.reassignSegLast = want1; + GenerateSpectrogramTexture(&app.stft, &app.spectrogramImage, + &app.spectrogramTexture); + app.visibleTextureValid = false; + } + } + int imgWidth = app.spectrogramImage.width; int imgHeight = app.spectrogramImage.height; - // Calculate visible region (time and frequency) - int visibleStartX = (int)(app.view.start * imgWidth); - int visibleEndX = (int)(app.view.end * imgWidth); + // Calculate visible region (time and frequency). X is relative to + // the segment range the image was built for, not the whole file. + float rangeStart = 0.0f, rangeEnd = 1.0f; + if (app.stft.numSegments > 0 && app.reassignSegLast > app.reassignSegFirst) { + rangeStart = (float)app.reassignSegFirst / app.stft.numSegments; + rangeEnd = (float)app.reassignSegLast / app.stft.numSegments; + } + float rangeSpan = rangeEnd - rangeStart; + if (rangeSpan <= 0.0f) rangeSpan = 1.0f; + float relStart = (app.view.start - rangeStart) / rangeSpan; + float relEnd = (app.view.end - rangeStart) / rangeSpan; + if (relStart < 0.0f) relStart = 0.0f; + if (relEnd > 1.0f) relEnd = 1.0f; + + int visibleStartX = (int)(relStart * imgWidth); + int visibleEndX = (int)(relEnd * imgWidth); int visibleWidth = visibleEndX - visibleStartX; // Frequency: 0 = bottom of image (bin 0), 1 = top of image (bin max). diff --git a/src/spectrogram_types.h b/src/spectrogram_types.h index 298607a..cc824f9 100644 --- a/src/spectrogram_types.h +++ b/src/spectrogram_types.h @@ -30,6 +30,14 @@ #define MAX_SAMPLE_RATE 48000 #define LOUDNESS_FLOOR_DB -80.0f +// Hard ceiling on the spectrogram image's width in pixels. GL implementations +// commonly cap textures at 16384 px per dimension, and a multi-hour capture +// produces far more STFT segments than that (~478k for 5.7 h at 12 kHz), so +// without a cap the texture upload fails and nothing draws at all. Segments +// beyond the cap are folded into columns; see ComputeSpectrogramReassignment. +// Kept below the common limit to leave headroom on weaker GL drivers. +#define MAX_SPECTRO_IMAGE_WIDTH 8192 + // How many overlapping annotation boxes the cursor-hit stack retains. Deeper // piles than this are counted but not listed individually (the tooltip says // "+N more"), which keeps a dense pile-up from covering the spectrogram. @@ -198,6 +206,15 @@ typedef struct { float* reassignBuffer; int reassignWidth; int reassignHeight; + // STFT segments folded into each image column (1 = one column per segment). + // >1 once the visible span has more segments than MAX_SPECTRO_IMAGE_WIDTH, + // and needed by anything converting between segment indices and image X. + int reassignSegsPerCol; + // Segment range the cached image covers, as [first, last). The image is + // built for the visible span rather than the whole file, so zooming in + // genuinely re-renders at higher resolution instead of magnifying pixels. + // A rebuild is triggered when the view leaves this range. + int reassignSegFirst, reassignSegLast; // Overlays bool showAbout; // About / help dialog