From c03d236230733cd5b91f7d7b364779771447dc04 Mon Sep 17 00:00:00 2001 From: Tyler Date: Fri, 15 May 2026 09:41:41 -0700 Subject: [PATCH] feat: implement adaptive-resolution STFT with on-demand high-res computation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For long signals (>60s), initial load computes every Nth segment at reduced resolution for a fast overview. Full resolution is computed on demand as the user zooms into specific regions, starting at the current viewport and computing 50 segments at a time. Key changes: - Add skipFactor and highResFinished fields to SpectrogramApp - ComputeSTFTInit uses calloc to NULL-initialize segments - ComputeSTFTIncremental skips non-aligned segments (skipFactor stride) - ComputeSTFTHighResRange computes full-res for a range [start, end) - GenerateSpectrogramTexture skips NULL segments for normalization - Zoom trigger computes high-res only for the visible viewport range, 50 segments at a time, staying within viewStart..viewEnd - No initial high-res block — only fills on-demand as user explores --- src/spectrogram.c | 186 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 176 insertions(+), 10 deletions(-) diff --git a/src/spectrogram.c b/src/spectrogram.c index cea1ceb..7297f68 100644 --- a/src/spectrogram.c +++ b/src/spectrogram.c @@ -162,6 +162,13 @@ typedef struct { int loadingPhase; // 0 = computing STFT, 1 = generating texture float loadingProgress; // 0.0 to 1.0 overall progress int currentSTFTSegment; // Which segment we're on for incremental processing + + // Adaptive resolution: skipFactor=1 means compute all segments, skipFactor=N + // means compute every Nth segment (faster initial load, overview-only). + // highResFinished tracks whether full-res segments have been computed for + // the current view range. + int skipFactor; + bool highResFinished; } SpectrogramApp; // ============================================================================ @@ -327,7 +334,7 @@ static void ComputeSTFTInit(AudioSignal* signal, StftResult* result, int fftSize if (numSegments <= 0) numSegments = 1; result->numSegments = numSegments; - result->segments = (StftSegment*)malloc(numSegments * sizeof(StftSegment)); + result->segments = (StftSegment*)calloc(numSegments, sizeof(StftSegment)); result->sampleRate = signal->sampleRate; result->totalSamples = signal->numSamples; result->useHannWindow = true; @@ -343,6 +350,11 @@ static bool ComputeSTFTIncremental(AudioSignal* signal, StftResult* result, int float complex* fftOutput = (float complex*)malloc(fftSize * sizeof(float complex)); for (int seg = startSegment; seg < result->numSegments; seg++) { + // Skip segments not aligned with the skip factor (overview mode) + if (seg % app.skipFactor != 0) continue; + + // Skip if already computed as high-res + if (result->segments[seg].spectrum != NULL) continue; int offset = seg * hopSize; int samplesToCopy = fftSize; if (offset + samplesToCopy > signal->numSamples) { @@ -409,6 +421,91 @@ static void FreeSTFT(StftResult* result) result->numSegments = 0; } +// ============================================================================ +// Adaptive Resolution: Skip Factor & High-Res Computation +// ============================================================================ + +// Compute an appropriate skip factor based on signal duration. +// Short signals: skipFactor=1 (full resolution, no waste). +// Long signals: higher skipFactor for fast overview. +static int ComputeSkipFactor(float signalDurationSec) +{ + if (signalDurationSec <= 60.0f) return 1; // < 1 min: full-res + if (signalDurationSec <= 300.0f) return 2; // 1-5 min: every 2nd + if (signalDurationSec <= 600.0f) return 4; // 5-10 min: every 4th + return 8; // > 10 min: every 8th +} + +// Compute full-resolution segments for the range [startSeg, endSeg). +// This replaces existing overview (skipFactor-strided) segments with +// high-resolution versions. Called when the user zooms in. +static bool ComputeSTFTHighResRange(AudioSignal* signal, StftResult* result, + int fftSize, int startSeg, int endSeg) +{ + int hopSize = fftSize / HOP_RATIO; + int numBins = fftSize / 2 + 1; + float* windowedSamples = (float*)malloc(fftSize * sizeof(float)); + float* derivWindowedSamples = (float*)malloc(fftSize * sizeof(float)); + float complex *complexInput = (float complex*)malloc(fftSize * sizeof(float complex)); + float complex* fftOutput = (float complex*)malloc(fftSize * sizeof(float complex)); + + for (int seg = startSeg; seg < endSeg && seg < result->numSegments; seg++) { + // Skip if this segment was already computed as high-res + if (result->segments[seg].spectrum != NULL) continue; + + int offset = seg * hopSize; + int samplesToCopy = fftSize; + if (offset + samplesToCopy > signal->numSamples) { + samplesToCopy = signal->numSamples - offset; + memset(windowedSamples, 0, fftSize * sizeof(float)); + memset(derivWindowedSamples, 0, fftSize * sizeof(float)); + } else { + memcpy(windowedSamples, signal->samples + offset, fftSize * sizeof(float)); + memcpy(derivWindowedSamples, signal->samples + offset, fftSize * sizeof(float)); + } + + for (int i = 0; i < fftSize; i++) { + float t = (float)i / (fftSize - 1); + float hann = 0.5f * (1.0f - cosf(2.0f * M_PI * t)); + float derivHann = M_PI * sinf(2.0f * M_PI * t); + windowedSamples[i] *= hann; + derivWindowedSamples[i] *= derivHann; + } + + // Normal STFT + for (int i = 0; i < fftSize; i++) complexInput[i] = windowedSamples[i] + 0.0f * I; + FFT(complexInput, fftOutput, fftSize, false); + + result->segments[seg].numBins = numBins; + result->segments[seg].sampleOffset = offset; + result->segments[seg].sampleCount = samplesToCopy; + result->segments[seg].spectrum = (FrequencyData*)malloc(numBins * sizeof(FrequencyData)); + + for (int bin = 0; bin < numBins; bin++) { + result->segments[seg].spectrum[bin].frequency = (float)bin * signal->sampleRate / fftSize; + result->segments[seg].spectrum[bin].amplitude = (bin == 0) ? cabsf(fftOutput[bin]) / fftSize : 2.0f * cabsf(fftOutput[bin]) / fftSize; + result->segments[seg].spectrum[bin].phase = cargf(fftOutput[bin]); + } + + // Derivative-window STFT for synchrosqueezing + result->segments[seg].derivativeSpectrum = (FrequencyData*)malloc(numBins * sizeof(FrequencyData)); + for (int i = 0; i < fftSize; i++) complexInput[i] = derivWindowedSamples[i] + 0.0f * I; + FFT(complexInput, fftOutput, fftSize, false); + + for (int bin = 0; bin < numBins; bin++) { + result->segments[seg].derivativeSpectrum[bin].frequency = (float)bin * signal->sampleRate / fftSize; + result->segments[seg].derivativeSpectrum[bin].amplitude = cabsf(fftOutput[bin]) / fftSize; + result->segments[seg].derivativeSpectrum[bin].phase = cargf(fftOutput[bin]); + } + } + + free(windowedSamples); + free(derivWindowedSamples); + free(complexInput); + free(fftOutput); + return true; +} + // ============================================================================ // Audio Loading // ============================================================================ @@ -529,16 +626,18 @@ static void GenerateSpectrogramTexture(StftResult* stft, Image* image, Texture2D int height = stft->segments[0].numBins; int fftSize = (height - 1) * 2; float freqPerBin = (float)stft->sampleRate / fftSize; - + *image = GenImageColor(width, height, BLACK); Color* pixels = (Color*)image->data; - // Find max amplitude for normalization + // Find max amplitude for normalization (skip NULL segments) float maxAmplitude = 0.0001f; - for (int seg = 0; seg < stft->numSegments; seg++) + for (int seg = 0; seg < stft->numSegments; seg++) { + if (stft->segments[seg].spectrum == NULL) continue; for (int bin = 0; bin < stft->segments[seg].numBins; bin++) - if (stft->segments[seg].spectrum[bin].amplitude > maxAmplitude) + if (stft->segments[seg].spectrum[bin].amplitude > maxAmplitude) maxAmplitude = stft->segments[seg].spectrum[bin].amplitude; + } // ===== SYNCHROSQUEEZING ===== // Reassign energy to true frequencies using derivative STFT @@ -550,6 +649,9 @@ static void GenerateSpectrogramTexture(StftResult* stft, Image* image, Texture2D float noiseThreshold = maxAmplitude * 0.01f; // 1% of max amplitude for (int seg = 0; seg < width; seg++) { + // Skip segments that haven't been computed yet (overview/high-res transition) + if (stft->segments[seg].spectrum == NULL) continue; + for (int bin = 0; bin < height; bin++) { FrequencyData* V_f = &stft->segments[seg].spectrum[bin]; FrequencyData* V_fd = &stft->segments[seg].derivativeSpectrum[bin]; @@ -841,6 +943,8 @@ static void LoadSelectedFile(void) app.loadingPhase = 0; app.loadingProgress = 0.0f; app.currentSTFTSegment = 0; + app.skipFactor = 1; + app.highResFinished = false; app.timeSelectionStart = app.viewStart = 0.0f; app.timeSelectionEnd = app.viewEnd = 1.0f; app.freqSelectionStart = 0.0f; @@ -1127,11 +1231,25 @@ static void DrawSidebar(void) Rectangle fftPlus = { x + sidebarWidth - 40 * scale, y, 30 * scale, 25 * scale }; if (CheckCollisionPointRec(GetMousePosition(), fftMinus) && IsMouseButtonPressed(MOUSE_LEFT_BUTTON)) { int newFFT = app.fftSize / 2; - if (newFFT >= FFT_SIZE_MIN) { app.fftSize = newFFT; app.stftComputed = false; app.loadingPhase = 0; needsRegen = true; } + if (newFFT >= FFT_SIZE_MIN) { + app.fftSize = newFFT; + app.stftComputed = false; + app.loadingPhase = 0; + app.skipFactor = 1; + app.highResFinished = false; + needsRegen = true; + } } if (CheckCollisionPointRec(GetMousePosition(), fftPlus) && IsMouseButtonPressed(MOUSE_LEFT_BUTTON)) { int newFFT = app.fftSize * 2; - if (newFFT <= FFT_SIZE_MAX) { app.fftSize = newFFT; app.stftComputed = false; app.loadingPhase = 0; needsRegen = true; } + if (newFFT <= FFT_SIZE_MAX) { + app.fftSize = newFFT; + app.stftComputed = false; + app.loadingPhase = 0; + app.skipFactor = 1; + app.highResFinished = false; + needsRegen = true; + } } DrawRectangleRec(fftMinus, (Color){ 50, 50, 60, 255 }); DrawRectangleLinesEx(fftMinus, 1, GRAY); @@ -1333,6 +1451,8 @@ int main(int argc, char* argv[]) app.cachedVisibleEndY = -1; app.visibleTextureValid = false; app.fftSize = FFT_SIZE_DEFAULT; + app.skipFactor = 1; + app.highResFinished = false; app.isPlaying = false; app.playbackFinished = false; @@ -1360,6 +1480,8 @@ int main(int argc, char* argv[]) app.loadingPhase = 0; app.loadingProgress = 0.0f; app.currentSTFTSegment = 0; + app.skipFactor = 1; + app.highResFinished = false; ComputeSTFTInit(&app.signal, &app.stft, app.fftSize); TraceLog(LOG_INFO, "File loaded successfully"); } @@ -1382,6 +1504,8 @@ int main(int argc, char* argv[]) app.loadingPhase = 0; app.loadingProgress = 0.0f; app.currentSTFTSegment = 0; + app.skipFactor = 1; + app.highResFinished = false; app.viewStart = 0.0f; app.viewEnd = 1.0f; ComputeSTFTInit(&app.signal, &app.stft, app.fftSize); // Invalidate visible texture cache @@ -1504,6 +1628,45 @@ int main(int argc, char* argv[]) } if (IsMouseButtonReleased(MOUSE_LEFT_BUTTON)) app.isPanning = false; + // Auto-compute high-res segments when user zooms into a new range. + // Only triggers when the view is sufficiently zoomed in (narrow range). + // This prevents reprocessing the whole signal on zoom-out. + if (app.skipFactor > 1 && app.highResFinished && app.stft.numSegments > 0) { + float viewRange = app.viewEnd - app.viewStart; + + // Only trigger when zoomed in to ~25% or less of the signal. + // When zoomed out, we keep whatever's already computed: + // high-res for visited regions, overview for the rest. + if (viewRange <= 0.25f) { + // Clamp to valid segment range + int viewStartSeg = (int)(app.viewStart * app.stft.numSegments); + int viewEndSeg = (int)(app.viewEnd * app.stft.numSegments); + if (viewStartSeg < 0) viewStartSeg = 0; + if (viewStartSeg >= app.stft.numSegments) viewStartSeg = app.stft.numSegments - 1; + if (viewEndSeg >= app.stft.numSegments) viewEndSeg = app.stft.numSegments - 1; + + // Check if we're done processing or if we just finished and need high-res + if (app.stftComputed || (app.loadingPhase >= 2)) { + // Only check segments within the current visible range + for (int seg = viewStartSeg; seg <= viewEndSeg && seg < app.stft.numSegments; seg++) { + if (app.stft.segments[seg].spectrum == NULL) { + // Compute high-res for 50 segments at a time, staying within view + int startSeg = seg; + int endSeg = seg + 50; + if (endSeg > viewEndSeg + 1) endSeg = viewEndSeg + 1; + ComputeSTFTHighResRange(&app.signal, &app.stft, app.fftSize, startSeg, endSeg); + app.visibleTextureValid = false; + app.stftComputed = false; + app.loadingPhase = 2; + app.loadingProgress = 0.0f; + TraceLog(LOG_INFO, "High-res for view range (%d to %d)", startSeg, endSeg - 1); + break; + } + } + } + } + } + // Home/End keys if (IsKeyPressed(KEY_HOME)) { app.viewStart = 0.0f; app.viewEnd = 1.0f; @@ -1698,6 +1861,8 @@ int main(int argc, char* argv[]) if (app.loadingPhase == 0) { // Initialize STFT once ComputeSTFTInit(&app.signal, &app.stft, app.fftSize); + app.skipFactor = ComputeSkipFactor(app.signal.duration); + app.highResFinished = true; // Overview loaded — ready for zoom-triggered high-res app.currentSTFTSegment = 0; app.loadingPhase = 1; } @@ -1724,7 +1889,8 @@ int main(int argc, char* argv[]) app.stftComputed = true; app.loadingPhase = -1; app.loadingProgress = 0.0f; - TraceLog(LOG_INFO, "STFT computed (%d segments)", app.stft.numSegments); + TraceLog(LOG_INFO, "STFT computed (%d segments, skipFactor=%d)", + app.stft.numSegments, app.skipFactor); } } @@ -1764,9 +1930,9 @@ int main(int argc, char* argv[]) int pctW = MeasureTextScaled(pctText, 14); DrawTextScaled(pctText, barX + barW / 2 - pctW / 2, barY + (int)(14 * scale), 14, WHITE); - // Duration estimate + // Duration estimate (account for skip factor — fewer segments to compute) int estY = barY + (int)(28 * scale); - float estSec = app.signal.duration / app.signal.sampleRate * app.stft.numSegments / 200.0f; + float estSec = app.signal.duration / app.signal.sampleRate * app.stft.numSegments / (200.0f * app.skipFactor); if (estSec > 0.5f && !isnan(estSec)) { char estText[64]; snprintf(estText, sizeof(estText), "Estimated time: %.1f sec", estSec);