diff --git a/src/spectrogram.c b/src/spectrogram.c index 6562d04..b6e1fec 100644 --- a/src/spectrogram.c +++ b/src/spectrogram.c @@ -173,7 +173,15 @@ typedef struct { // the current view range. int skipFactor; bool highResFinished; - + + // Background high-res computation state. + // After the overview (skipFactor-strided) loads, missing segments are + // filled in at full resolution in the background while the user is idle. + int bgHighResSeg; // next segment index to compute at high-res + bool bgFinished; // true when all segments are computed at high-res + int lastInteractedFrame; // frame counter when last user interaction occurred + bool isBgProcessing; // true while background task is actively computing + // Waveform scope view (underneath spectrogram viewport) ScopeView scopeView; bool showScope; // Toggle to show/hide scope view @@ -290,6 +298,115 @@ static void GenerateColormapTexture(void) UnloadImage(img); } +// ============================================================================ +// Interaction Detection +// ============================================================================ + +/** + * Returns true if the user has pressed any mouse/keyboard input this frame. + * Used to gate background processing — we only compute when the user is idle. + */ +static bool IsUserInteracting(void) +{ + if (IsMouseButtonDown(MOUSE_BUTTON_LEFT) || + IsMouseButtonDown(MOUSE_BUTTON_RIGHT) || + IsMouseButtonDown(MOUSE_BUTTON_MIDDLE)) { + return true; + } + // Check for mouse wheel + if (GetMouseWheelMove() != 0) return true; + // Check for key press (key codes are 0..512 in raylib) + for (int key = 0; key < 512; key++) { + if (IsKeyPressed(key)) return true; + } + return false; +} + +// Forward declarations for functions defined later in this file +static void FFT(float complex* input, float complex* output, int n, bool inverse); + +// ============================================================================ +// Background High-Res Computation +// ============================================================================ + +/** + * Compute high-res segments for one chunk of the signal. + * Processes from startSeg up to (but not including) endSeg, skipping + * any segment that already has spectrum data (either overview or high-res). + * Returns the next segment index to process next time (endSeg, or + * numSegments if we've reached the end). + */ +static int ComputeNextHighResChunk(AudioSignal* signal, StftResult* result, + int fftSize, int startSeg, int endSeg) +{ + int hopSize = fftSize / HOP_RATIO; + int numBins = fftSize / 2 + 1; + float* windowedSamples = (float*)malloc(fftSize * sizeof(float)); + float* derivWindowedSamples = (float*)malloc(fftSize * sizeof(float)); + float complex *complexInput = (float complex*)malloc(fftSize * sizeof(float complex)); + float complex* fftOutput = (float complex*)malloc(fftSize * sizeof(float complex)); + + for (int seg = startSeg; seg < endSeg && seg < result->numSegments; seg++) { + // Skip if already computed (overview or high-res) + if (result->segments[seg].spectrum != NULL) continue; + + int offset = seg * hopSize; + int samplesToCopy = fftSize; + if (offset + samplesToCopy > signal->numSamples) { + samplesToCopy = signal->numSamples - offset; + memset(windowedSamples, 0, fftSize * sizeof(float)); + memset(derivWindowedSamples, 0, fftSize * sizeof(float)); + } else { + memcpy(windowedSamples, signal->samples + offset, fftSize * sizeof(float)); + memcpy(derivWindowedSamples, signal->samples + offset, fftSize * sizeof(float)); + } + + // Apply Hann window and derivative window + for (int i = 0; i < fftSize; i++) { + float t = (float)i / (fftSize - 1); + float hann = 0.5f * (1.0f - cosf(2.0f * M_PI * t)); + float derivHann = M_PI * sinf(2.0f * M_PI * t); + windowedSamples[i] *= hann; + derivWindowedSamples[i] *= derivHann; + } + + // Normal STFT + for (int i = 0; i < fftSize; i++) complexInput[i] = windowedSamples[i] + 0.0f * I; + FFT(complexInput, fftOutput, fftSize, false); + + result->segments[seg].numBins = numBins; + result->segments[seg].sampleOffset = offset; + result->segments[seg].sampleCount = samplesToCopy; + result->segments[seg].spectrum = (FrequencyData*)malloc(numBins * sizeof(FrequencyData)); + + for (int bin = 0; bin < numBins; bin++) { + result->segments[seg].spectrum[bin].frequency = (float)bin * signal->sampleRate / fftSize; + result->segments[seg].spectrum[bin].amplitude = (bin == 0) ? cabsf(fftOutput[bin]) / fftSize : 2.0f * cabsf(fftOutput[bin]) / fftSize; + result->segments[seg].spectrum[bin].phase = cargf(fftOutput[bin]); + } + + // Derivative-window STFT for synchrosqueezing + result->segments[seg].derivativeSpectrum = (FrequencyData*)malloc(numBins * sizeof(FrequencyData)); + for (int i = 0; i < fftSize; i++) complexInput[i] = derivWindowedSamples[i] + 0.0f * I; + FFT(complexInput, fftOutput, fftSize, false); + + for (int bin = 0; bin < numBins; bin++) { + result->segments[seg].derivativeSpectrum[bin].frequency = (float)bin * signal->sampleRate / fftSize; + result->segments[seg].derivativeSpectrum[bin].amplitude = cabsf(fftOutput[bin]) / fftSize; + result->segments[seg].derivativeSpectrum[bin].phase = cargf(fftOutput[bin]); + } + } + + free(windowedSamples); + free(derivWindowedSamples); + free(complexInput); + free(fftOutput); + + // Return next segment to process + if (endSeg >= result->numSegments) return result->numSegments; + return endSeg; +} + // ============================================================================ // FFT Implementation // ============================================================================ @@ -959,6 +1076,9 @@ static void LoadSelectedFile(void) app.currentSTFTSegment = 0; app.skipFactor = 1; app.highResFinished = false; + app.bgHighResSeg = 0; + app.bgFinished = false; + app.isBgProcessing = false; app.timeSelectionStart = app.viewStart = 0.0f; app.timeSelectionEnd = app.viewEnd = 1.0f; app.freqSelectionStart = 0.0f; @@ -1491,24 +1611,30 @@ static void DrawSidebar(void) Rectangle fftPlus = { x + sidebarWidth - 40 * scale, y, 30 * scale, 25 * scale }; if (CheckCollisionPointRec(GetMousePosition(), fftMinus) && IsMouseButtonPressed(MOUSE_LEFT_BUTTON)) { int newFFT = app.fftSize / 2; - if (newFFT >= FFT_SIZE_MIN) { - app.fftSize = newFFT; - app.stftComputed = false; - app.loadingPhase = 0; + if (newFFT >= FFT_SIZE_MIN) { + app.fftSize = newFFT; + app.stftComputed = false; + app.loadingPhase = 0; app.skipFactor = 1; app.highResFinished = false; - needsRegen = true; + app.bgHighResSeg = 0; + app.bgFinished = false; + app.isBgProcessing = false; + needsRegen = true; } } if (CheckCollisionPointRec(GetMousePosition(), fftPlus) && IsMouseButtonPressed(MOUSE_LEFT_BUTTON)) { int newFFT = app.fftSize * 2; - if (newFFT <= FFT_SIZE_MAX) { - app.fftSize = newFFT; - app.stftComputed = false; - app.loadingPhase = 0; + if (newFFT <= FFT_SIZE_MAX) { + app.fftSize = newFFT; + app.stftComputed = false; + app.loadingPhase = 0; app.skipFactor = 1; app.highResFinished = false; - needsRegen = true; + app.bgHighResSeg = 0; + app.bgFinished = false; + app.isBgProcessing = false; + needsRegen = true; } } DrawRectangleRec(fftMinus, (Color){ 50, 50, 60, 255 }); @@ -1742,6 +1868,10 @@ int main(int argc, char* argv[]) app.fftSize = FFT_SIZE_DEFAULT; app.skipFactor = 1; app.highResFinished = false; + app.bgHighResSeg = 0; + app.bgFinished = false; + app.lastInteractedFrame = 0; + app.isBgProcessing = false; app.isPlaying = false; app.playbackFinished = false; app.showScope = true; @@ -1781,13 +1911,16 @@ int main(int argc, char* argv[]) app.currentSTFTSegment = 0; app.skipFactor = 1; app.highResFinished = false; + app.bgHighResSeg = 0; + app.bgFinished = false; + app.isBgProcessing = false; ComputeSTFTInit(&app.signal, &app.stft, app.fftSize); TraceLog(LOG_INFO, "File loaded successfully"); } } - + if (!fileLoaded) TraceLog(LOG_INFO, "Press 'O' for file browser or drag & drop WAV file"); - + while (!WindowShouldClose()) { // Drag & Drop @@ -1805,6 +1938,9 @@ int main(int argc, char* argv[]) app.currentSTFTSegment = 0; app.skipFactor = 1; app.highResFinished = false; + app.bgHighResSeg = 0; + app.bgFinished = false; + app.isBgProcessing = false; app.viewStart = 0.0f; app.viewEnd = 1.0f; ComputeSTFTInit(&app.signal, &app.stft, app.fftSize); // Invalidate visible texture cache @@ -1950,15 +2086,12 @@ int main(int argc, char* argv[]) } if (IsMouseButtonReleased(MOUSE_LEFT_BUTTON)) app.isPanning = false; - // Auto-compute high-res segments when user zooms into a new range. - // Only triggers when the view is sufficiently zoomed in (narrow range). - // This prevents reprocessing the whole signal on zoom-out. - if (app.skipFactor > 1 && app.highResFinished && app.stft.numSegments > 0) { + // Foreground high-res: when user zooms in, compute missing + // segments in the visible range immediately (responsive). + // Background task handles the rest when idle. + if (app.skipFactor > 1 && app.stft.numSegments > 0 && !app.bgFinished) { float viewRange = app.viewEnd - app.viewStart; - // Only trigger when zoomed in to ~25% or less of the signal. - // When zoomed out, we keep whatever's already computed: - // high-res for visited regions, overview for the rest. if (viewRange <= 0.25f) { // Clamp to valid segment range int viewStartSeg = (int)(app.viewStart * app.stft.numSegments); @@ -1967,28 +2100,58 @@ int main(int argc, char* argv[]) if (viewStartSeg >= app.stft.numSegments) viewStartSeg = app.stft.numSegments - 1; if (viewEndSeg >= app.stft.numSegments) viewEndSeg = app.stft.numSegments - 1; - // Check if we're done processing or if we just finished and need high-res - if (app.stftComputed || (app.loadingPhase >= 2)) { - // Only check segments within the current visible range - for (int seg = viewStartSeg; seg <= viewEndSeg && seg < app.stft.numSegments; seg++) { - if (app.stft.segments[seg].spectrum == NULL) { - // Compute high-res for 50 segments at a time, staying within view - int startSeg = seg; - int endSeg = seg + 50; - if (endSeg > viewEndSeg + 1) endSeg = viewEndSeg + 1; - ComputeSTFTHighResRange(&app.signal, &app.stft, app.fftSize, startSeg, endSeg); - app.visibleTextureValid = false; - app.stftComputed = false; - app.loadingPhase = 2; - app.loadingProgress = 0.0f; - TraceLog(LOG_INFO, "High-res for view range (%d to %d)", startSeg, endSeg - 1); - break; - } + // Find first missing segment in the visible range and compute it + for (int seg = viewStartSeg; seg <= viewEndSeg && seg < app.stft.numSegments; seg++) { + if (app.stft.segments[seg].spectrum == NULL) { + int startSeg = seg; + int endSeg = seg + 50; + if (endSeg > viewEndSeg + 1) endSeg = viewEndSeg + 1; + app.bgHighResSeg = ComputeNextHighResChunk(&app.signal, &app.stft, app.fftSize, startSeg, endSeg); + app.visibleTextureValid = false; + TraceLog(LOG_INFO, "Foreground high-res (%d to %d)", startSeg, endSeg - 1); + break; } } } } + // Background high-res: when user is idle, fill in remaining + // segments at full resolution. Pauses on any interaction. + // Also kicks in when zoomed out (no foreground trigger) to fill + // segments outside the view range. + bool isZoomedIn = (app.skipFactor > 1 && app.viewEnd - app.viewStart <= 0.25f); + if (app.isBgProcessing && !app.bgFinished && !IsUserInteracting()) { + int endSeg = app.bgHighResSeg + 50; // chunks of 50 segments + if (endSeg > app.stft.numSegments) endSeg = app.stft.numSegments; + app.bgHighResSeg = ComputeNextHighResChunk(&app.signal, &app.stft, app.fftSize, app.bgHighResSeg, endSeg); + if (app.bgHighResSeg >= app.stft.numSegments) { + // All done — generate full-res texture and mark complete + AutoScaleAmplitude(&app.stft); + GenerateSpectrogramTexture(&app.stft, &app.spectrogramImage, &app.spectrogramTexture); + app.visibleTextureValid = false; + app.bgFinished = true; + app.isBgProcessing = false; + TraceLog(LOG_INFO, "Background high-res complete (%d segments)", app.stft.numSegments); + } + } + if (app.isBgProcessing && IsUserInteracting()) { + // Pause background processing — user is interacting + app.isBgProcessing = false; + } + + // If not zoomed in, scan for missing segments to kick off processing + if (!isZoomedIn && app.isBgProcessing && !app.bgFinished && app.bgHighResSeg < app.stft.numSegments) { + bool hasMissing = false; + for (int i = app.bgHighResSeg; i < app.stft.numSegments; i++) { + if (app.stft.segments[i].spectrum == NULL) { hasMissing = true; break; } + } + if (!hasMissing) { + // No more missing segments — mark complete + app.bgFinished = true; + app.isBgProcessing = false; + } + } + // Home/End keys if (IsKeyPressed(KEY_HOME)) { app.viewStart = 0.0f; app.viewEnd = 1.0f; @@ -2235,12 +2398,14 @@ int main(int argc, char* argv[]) // Initialize STFT once ComputeSTFTInit(&app.signal, &app.stft, app.fftSize); app.skipFactor = ComputeSkipFactor(app.signal.duration); - app.highResFinished = true; // Overview loaded — ready for zoom-triggered high-res + app.bgHighResSeg = 0; + app.bgFinished = false; + app.isBgProcessing = false; app.currentSTFTSegment = 0; app.loadingPhase = 1; } if (app.loadingPhase == 1) { - // Compute STFT in chunks (process all segments this frame) + // Compute STFT in chunks (overview: skipFactor-strided) int chunksPerFrame = 200; int startSeg = app.currentSTFTSegment; int endSeg = startSeg + chunksPerFrame; @@ -2253,16 +2418,16 @@ int main(int argc, char* argv[]) } } if (app.loadingPhase == 2) { + // Overview loaded — generate texture (NULL segments render as black) + // and transition to ready state so background processing can start. AutoScaleAmplitude(&app.stft); GenerateSpectrogramTexture(&app.stft, &app.spectrogramImage, &app.spectrogramTexture); app.loadingProgress = 1.0f; - app.loadingPhase = 3; - } - if (app.loadingPhase == 3) { app.stftComputed = true; - app.loadingPhase = -1; + app.loadingPhase = 0; // Reset — background processing runs outside this block app.loadingProgress = 0.0f; - TraceLog(LOG_INFO, "STFT computed (%d segments, skipFactor=%d)", + app.isBgProcessing = true; // Kick off background high-res next frame + TraceLog(LOG_INFO, "STFT overview computed (%d segments, skipFactor=%d)", app.stft.numSegments, app.skipFactor); } }