tyler/rspektrum
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add background high-res STFT computation with idle detection

Browse Source 
After initial overview loads, compute full-res segments in the background
while the user is idle (200 segments/frame, pauses on any interaction).
Foreground high-res computes zoomed-in segments immediately for
responsiveness. Computed segments persist — zooming out never recomputes.

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
Tyler
2026-05-25 00:00:07 -07:00
parent 0e72bf4172
commit 475520b1db
1 changed files with 209 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/spectrogram.c
+209 -44
View File
@@ -173,7 +173,15 @@ typedef struct {
// the current view range.
int skipFactor;
bool highResFinished;
// Background high-res computation state.
// After the overview (skipFactor-strided) loads, missing segments are
// filled in at full resolution in the background while the user is idle.
int bgHighResSeg; // next segment index to compute at high-res
bool bgFinished; // true when all segments are computed at high-res
int lastInteractedFrame; // frame counter when last user interaction occurred
bool isBgProcessing; // true while background task is actively computing
// Waveform scope view (underneath spectrogram viewport)
ScopeView scopeView;
bool showScope; // Toggle to show/hide scope view
@@ -290,6 +298,115 @@ static void GenerateColormapTexture(void)
UnloadImage(img);
}
// ============================================================================
// Interaction Detection
// ============================================================================
/**
* Returns true if the user has pressed any mouse/keyboard input this frame.
* Used to gate background processing — we only compute when the user is idle.
*/
static bool IsUserInteracting(void)
{
if (IsMouseButtonDown(MOUSE_BUTTON_LEFT) ||
IsMouseButtonDown(MOUSE_BUTTON_RIGHT) ||
IsMouseButtonDown(MOUSE_BUTTON_MIDDLE)) {
return true;
}
// Check for mouse wheel
if (GetMouseWheelMove() != 0) return true;
// Check for key press (key codes are 0..512 in raylib)
for (int key = 0; key < 512; key++) {
if (IsKeyPressed(key)) return true;
}
return false;
}
// Forward declarations for functions defined later in this file
static void FFT(float complex* input, float complex* output, int n, bool inverse);
// ============================================================================
// Background High-Res Computation
// ============================================================================
/**
* Compute high-res segments for one chunk of the signal.
* Processes from startSeg up to (but not including) endSeg, skipping
* any segment that already has spectrum data (either overview or high-res).
* Returns the next segment index to process next time (endSeg, or
* numSegments if we've reached the end).
*/
static int ComputeNextHighResChunk(AudioSignal* signal, StftResult* result,
int fftSize, int startSeg, int endSeg)
{
int hopSize = fftSize / HOP_RATIO;
int numBins = fftSize / 2 + 1;
float* windowedSamples = (float*)malloc(fftSize * sizeof(float));
float* derivWindowedSamples = (float*)malloc(fftSize * sizeof(float));
float complex *complexInput = (float complex*)malloc(fftSize * sizeof(float complex));
float complex* fftOutput = (float complex*)malloc(fftSize * sizeof(float complex));
for (int seg = startSeg; seg < endSeg && seg < result->numSegments; seg++) {
// Skip if already computed (overview or high-res)
if (result->segments[seg].spectrum != NULL) continue;
int offset = seg * hopSize;
int samplesToCopy = fftSize;
if (offset + samplesToCopy > signal->numSamples) {
samplesToCopy = signal->numSamples - offset;
memset(windowedSamples, 0, fftSize * sizeof(float));
memset(derivWindowedSamples, 0, fftSize * sizeof(float));
} else {
memcpy(windowedSamples, signal->samples + offset, fftSize * sizeof(float));
memcpy(derivWindowedSamples, signal->samples + offset, fftSize * sizeof(float));
}
// Apply Hann window and derivative window
for (int i = 0; i < fftSize; i++) {
float t = (float)i / (fftSize - 1);
float hann = 0.5f * (1.0f - cosf(2.0f * M_PI * t));
float derivHann = M_PI * sinf(2.0f * M_PI * t);
windowedSamples[i] *= hann;
derivWindowedSamples[i] *= derivHann;
}
// Normal STFT
for (int i = 0; i < fftSize; i++) complexInput[i] = windowedSamples[i] + 0.0f * I;
FFT(complexInput, fftOutput, fftSize, false);
result->segments[seg].numBins = numBins;
result->segments[seg].sampleOffset = offset;
result->segments[seg].sampleCount = samplesToCopy;
result->segments[seg].spectrum = (FrequencyData*)malloc(numBins * sizeof(FrequencyData));
for (int bin = 0; bin < numBins; bin++) {
result->segments[seg].spectrum[bin].frequency = (float)bin * signal->sampleRate / fftSize;
result->segments[seg].spectrum[bin].amplitude = (bin == 0) ? cabsf(fftOutput[bin]) / fftSize : 2.0f * cabsf(fftOutput[bin]) / fftSize;
result->segments[seg].spectrum[bin].phase = cargf(fftOutput[bin]);
}
// Derivative-window STFT for synchrosqueezing
result->segments[seg].derivativeSpectrum = (FrequencyData*)malloc(numBins * sizeof(FrequencyData));
for (int i = 0; i < fftSize; i++) complexInput[i] = derivWindowedSamples[i] + 0.0f * I;
FFT(complexInput, fftOutput, fftSize, false);
for (int bin = 0; bin < numBins; bin++) {
result->segments[seg].derivativeSpectrum[bin].frequency = (float)bin * signal->sampleRate / fftSize;
result->segments[seg].derivativeSpectrum[bin].amplitude = cabsf(fftOutput[bin]) / fftSize;
result->segments[seg].derivativeSpectrum[bin].phase = cargf(fftOutput[bin]);
}
}
free(windowedSamples);
free(derivWindowedSamples);
free(complexInput);
free(fftOutput);
// Return next segment to process
if (endSeg >= result->numSegments) return result->numSegments;
return endSeg;
}
// ============================================================================
// FFT Implementation
// ============================================================================
@@ -959,6 +1076,9 @@ static void LoadSelectedFile(void)
app.currentSTFTSegment = 0;
app.skipFactor = 1;
app.highResFinished = false;
app.bgHighResSeg = 0;
app.bgFinished = false;
app.isBgProcessing = false;
app.timeSelectionStart = app.viewStart = 0.0f;
app.timeSelectionEnd = app.viewEnd = 1.0f;
app.freqSelectionStart = 0.0f;
@@ -1491,24 +1611,30 @@ static void DrawSidebar(void)
Rectangle fftPlus = { x + sidebarWidth - 40 * scale, y, 30 * scale, 25 * scale };
if (CheckCollisionPointRec(GetMousePosition(), fftMinus) && IsMouseButtonPressed(MOUSE_LEFT_BUTTON)) {
int newFFT = app.fftSize / 2;
if (newFFT >= FFT_SIZE_MIN) {
app.fftSize = newFFT;
app.stftComputed = false;
app.loadingPhase = 0;
if (newFFT >= FFT_SIZE_MIN) {
app.fftSize = newFFT;
app.stftComputed = false;
app.loadingPhase = 0;
app.skipFactor = 1;
app.highResFinished = false;
needsRegen = true;
app.bgHighResSeg = 0;
app.bgFinished = false;
app.isBgProcessing = false;
needsRegen = true;
}
}
if (CheckCollisionPointRec(GetMousePosition(), fftPlus) && IsMouseButtonPressed(MOUSE_LEFT_BUTTON)) {
int newFFT = app.fftSize * 2;
if (newFFT <= FFT_SIZE_MAX) {
app.fftSize = newFFT;
app.stftComputed = false;
app.loadingPhase = 0;
if (newFFT <= FFT_SIZE_MAX) {
app.fftSize = newFFT;
app.stftComputed = false;
app.loadingPhase = 0;
app.skipFactor = 1;
app.highResFinished = false;
needsRegen = true;
app.bgHighResSeg = 0;
app.bgFinished = false;
app.isBgProcessing = false;
needsRegen = true;
}
}
DrawRectangleRec(fftMinus, (Color){ 50, 50, 60, 255 });
@@ -1742,6 +1868,10 @@ int main(int argc, char* argv[])
app.fftSize = FFT_SIZE_DEFAULT;
app.skipFactor = 1;
app.highResFinished = false;
app.bgHighResSeg = 0;
app.bgFinished = false;
app.lastInteractedFrame = 0;
app.isBgProcessing = false;
app.isPlaying = false;
app.playbackFinished = false;
app.showScope = true;
@@ -1781,13 +1911,16 @@ int main(int argc, char* argv[])
app.currentSTFTSegment = 0;
app.skipFactor = 1;
app.highResFinished = false;
app.bgHighResSeg = 0;
app.bgFinished = false;
app.isBgProcessing = false;
ComputeSTFTInit(&app.signal, &app.stft, app.fftSize);
TraceLog(LOG_INFO, "File loaded successfully");
}
}
if (!fileLoaded) TraceLog(LOG_INFO, "Press 'O' for file browser or drag & drop WAV file");
while (!WindowShouldClose())
{
// Drag & Drop
@@ -1805,6 +1938,9 @@ int main(int argc, char* argv[])
app.currentSTFTSegment = 0;
app.skipFactor = 1;
app.highResFinished = false;
app.bgHighResSeg = 0;
app.bgFinished = false;
app.isBgProcessing = false;
app.viewStart = 0.0f; app.viewEnd = 1.0f;
ComputeSTFTInit(&app.signal, &app.stft, app.fftSize);
// Invalidate visible texture cache
@@ -1950,15 +2086,12 @@ int main(int argc, char* argv[])
}
if (IsMouseButtonReleased(MOUSE_LEFT_BUTTON)) app.isPanning = false;
// Auto-compute high-res segments when user zooms into a new range.
// Only triggers when the view is sufficiently zoomed in (narrow range).
// This prevents reprocessing the whole signal on zoom-out.
if (app.skipFactor > 1 && app.highResFinished && app.stft.numSegments > 0) {
// Foreground high-res: when user zooms in, compute missing
// segments in the visible range immediately (responsive).
// Background task handles the rest when idle.
if (app.skipFactor > 1 && app.stft.numSegments > 0 && !app.bgFinished) {
float viewRange = app.viewEnd - app.viewStart;
// Only trigger when zoomed in to ~25% or less of the signal.
// When zoomed out, we keep whatever's already computed:
// high-res for visited regions, overview for the rest.
if (viewRange <= 0.25f) {
// Clamp to valid segment range
int viewStartSeg = (int)(app.viewStart * app.stft.numSegments);
@@ -1967,28 +2100,58 @@ int main(int argc, char* argv[])
if (viewStartSeg >= app.stft.numSegments) viewStartSeg = app.stft.numSegments - 1;
if (viewEndSeg >= app.stft.numSegments) viewEndSeg = app.stft.numSegments - 1;
// Check if we're done processing or if we just finished and need high-res
if (app.stftComputed || (app.loadingPhase >= 2)) {
// Only check segments within the current visible range
for (int seg = viewStartSeg; seg <= viewEndSeg && seg < app.stft.numSegments; seg++) {
if (app.stft.segments[seg].spectrum == NULL) {
// Compute high-res for 50 segments at a time, staying within view
int startSeg = seg;
int endSeg = seg + 50;
if (endSeg > viewEndSeg + 1) endSeg = viewEndSeg + 1;
ComputeSTFTHighResRange(&app.signal, &app.stft, app.fftSize, startSeg, endSeg);
app.visibleTextureValid = false;
app.stftComputed = false;
app.loadingPhase = 2;
app.loadingProgress = 0.0f;
TraceLog(LOG_INFO, "High-res for view range (%d to %d)", startSeg, endSeg - 1);
break;
}
// Find first missing segment in the visible range and compute it
for (int seg = viewStartSeg; seg <= viewEndSeg && seg < app.stft.numSegments; seg++) {
if (app.stft.segments[seg].spectrum == NULL) {
int startSeg = seg;
int endSeg = seg + 50;
if (endSeg > viewEndSeg + 1) endSeg = viewEndSeg + 1;
app.bgHighResSeg = ComputeNextHighResChunk(&app.signal, &app.stft, app.fftSize, startSeg, endSeg);
app.visibleTextureValid = false;
TraceLog(LOG_INFO, "Foreground high-res (%d to %d)", startSeg, endSeg - 1);
break;
}
}
}
}
// Background high-res: when user is idle, fill in remaining
// segments at full resolution. Pauses on any interaction.
// Also kicks in when zoomed out (no foreground trigger) to fill
// segments outside the view range.
bool isZoomedIn = (app.skipFactor > 1 && app.viewEnd - app.viewStart <= 0.25f);
if (app.isBgProcessing && !app.bgFinished && !IsUserInteracting()) {
int endSeg = app.bgHighResSeg + 50; // chunks of 50 segments
if (endSeg > app.stft.numSegments) endSeg = app.stft.numSegments;
app.bgHighResSeg = ComputeNextHighResChunk(&app.signal, &app.stft, app.fftSize, app.bgHighResSeg, endSeg);
if (app.bgHighResSeg >= app.stft.numSegments) {
// All done — generate full-res texture and mark complete
AutoScaleAmplitude(&app.stft);
GenerateSpectrogramTexture(&app.stft, &app.spectrogramImage, &app.spectrogramTexture);
app.visibleTextureValid = false;
app.bgFinished = true;
app.isBgProcessing = false;
TraceLog(LOG_INFO, "Background high-res complete (%d segments)", app.stft.numSegments);
}
}
if (app.isBgProcessing && IsUserInteracting()) {
// Pause background processing — user is interacting
app.isBgProcessing = false;
}
// If not zoomed in, scan for missing segments to kick off processing
if (!isZoomedIn && app.isBgProcessing && !app.bgFinished && app.bgHighResSeg < app.stft.numSegments) {
bool hasMissing = false;
for (int i = app.bgHighResSeg; i < app.stft.numSegments; i++) {
if (app.stft.segments[i].spectrum == NULL) { hasMissing = true; break; }
}
if (!hasMissing) {
// No more missing segments — mark complete
app.bgFinished = true;
app.isBgProcessing = false;
}
}
// Home/End keys
if (IsKeyPressed(KEY_HOME)) {
app.viewStart = 0.0f; app.viewEnd = 1.0f;
@@ -2235,12 +2398,14 @@ int main(int argc, char* argv[])
// Initialize STFT once
ComputeSTFTInit(&app.signal, &app.stft, app.fftSize);
app.skipFactor = ComputeSkipFactor(app.signal.duration);
app.highResFinished = true; // Overview loaded — ready for zoom-triggered high-res
app.bgHighResSeg = 0;
app.bgFinished = false;
app.isBgProcessing = false;
app.currentSTFTSegment = 0;
app.loadingPhase = 1;
}
if (app.loadingPhase == 1) {
// Compute STFT in chunks (process all segments this frame)
// Compute STFT in chunks (overview: skipFactor-strided)
int chunksPerFrame = 200;
int startSeg = app.currentSTFTSegment;
int endSeg = startSeg + chunksPerFrame;
@@ -2253,16 +2418,16 @@ int main(int argc, char* argv[])
}
}
if (app.loadingPhase == 2) {
// Overview loaded — generate texture (NULL segments render as black)
// and transition to ready state so background processing can start.
AutoScaleAmplitude(&app.stft);
GenerateSpectrogramTexture(&app.stft, &app.spectrogramImage, &app.spectrogramTexture);
app.loadingProgress = 1.0f;
app.loadingPhase = 3;
}
if (app.loadingPhase == 3) {
app.stftComputed = true;
app.loadingPhase = -1;
app.loadingPhase = 0; // Reset — background processing runs outside this block
app.loadingProgress = 0.0f;
TraceLog(LOG_INFO, "STFT computed (%d segments, skipFactor=%d)",
app.isBgProcessing = true; // Kick off background high-res next frame
TraceLog(LOG_INFO, "STFT overview computed (%d segments, skipFactor=%d)",
app.stft.numSegments, app.skipFactor);
}
}