tyler/rspektrum
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: implement adaptive-resolution STFT with on-demand high-res computation

Browse Source 
For long signals (>60s), initial load computes every Nth segment at
reduced resolution for a fast overview. Full resolution is computed on
demand as the user zooms into specific regions, starting at the current
viewport and computing 50 segments at a time.

Key changes:
- Add skipFactor and highResFinished fields to SpectrogramApp
- ComputeSTFTInit uses calloc to NULL-initialize segments
- ComputeSTFTIncremental skips non-aligned segments (skipFactor stride)
- ComputeSTFTHighResRange computes full-res for a range [start, end)
- GenerateSpectrogramTexture skips NULL segments for normalization
- Zoom trigger computes high-res only for the visible viewport range,
  50 segments at a time, staying within viewStart..viewEnd
- No initial high-res block — only fills on-demand as user explores
This commit is contained in:
Tyler
2026-05-15 09:41:41 -07:00
parent b6942d8577
commit c03d236230
1 changed files with 176 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/spectrogram.c
+176 -10
View File
@@ -162,6 +162,13 @@ typedef struct {
int loadingPhase; // 0 = computing STFT, 1 = generating texture
float loadingProgress; // 0.0 to 1.0 overall progress
int currentSTFTSegment; // Which segment we're on for incremental processing
// Adaptive resolution: skipFactor=1 means compute all segments, skipFactor=N
// means compute every Nth segment (faster initial load, overview-only).
// highResFinished tracks whether full-res segments have been computed for
// the current view range.
int skipFactor;
bool highResFinished;
} SpectrogramApp;
// ============================================================================
@@ -327,7 +334,7 @@ static void ComputeSTFTInit(AudioSignal* signal, StftResult* result, int fftSize
if (numSegments <= 0) numSegments = 1;
result->numSegments = numSegments;
result->segments = (StftSegment*)malloc(numSegments * sizeof(StftSegment));
result->segments = (StftSegment*)calloc(numSegments, sizeof(StftSegment));
result->sampleRate = signal->sampleRate;
result->totalSamples = signal->numSamples;
result->useHannWindow = true;
@@ -343,6 +350,11 @@ static bool ComputeSTFTIncremental(AudioSignal* signal, StftResult* result, int
float complex* fftOutput = (float complex*)malloc(fftSize * sizeof(float complex));
for (int seg = startSegment; seg < result->numSegments; seg++) {
// Skip segments not aligned with the skip factor (overview mode)
if (seg % app.skipFactor != 0) continue;
// Skip if already computed as high-res
if (result->segments[seg].spectrum != NULL) continue;
int offset = seg * hopSize;
int samplesToCopy = fftSize;
if (offset + samplesToCopy > signal->numSamples) {
@@ -409,6 +421,91 @@ static void FreeSTFT(StftResult* result)
result->numSegments = 0;
}
// ============================================================================
// Adaptive Resolution: Skip Factor & High-Res Computation
// ============================================================================
// Compute an appropriate skip factor based on signal duration.
// Short signals: skipFactor=1 (full resolution, no waste).
// Long signals: higher skipFactor for fast overview.
static int ComputeSkipFactor(float signalDurationSec)
{
if (signalDurationSec <= 60.0f) return 1; // < 1 min: full-res
if (signalDurationSec <= 300.0f) return 2; // 1-5 min: every 2nd
if (signalDurationSec <= 600.0f) return 4; // 5-10 min: every 4th
return 8; // > 10 min: every 8th
}
// Compute full-resolution segments for the range [startSeg, endSeg).
// This replaces existing overview (skipFactor-strided) segments with
// high-resolution versions. Called when the user zooms in.
static bool ComputeSTFTHighResRange(AudioSignal* signal, StftResult* result,
int fftSize, int startSeg, int endSeg)
{
int hopSize = fftSize / HOP_RATIO;
int numBins = fftSize / 2 + 1;
float* windowedSamples = (float*)malloc(fftSize * sizeof(float));
float* derivWindowedSamples = (float*)malloc(fftSize * sizeof(float));
float complex *complexInput = (float complex*)malloc(fftSize * sizeof(float complex));
float complex* fftOutput = (float complex*)malloc(fftSize * sizeof(float complex));
for (int seg = startSeg; seg < endSeg && seg < result->numSegments; seg++) {
// Skip if this segment was already computed as high-res
if (result->segments[seg].spectrum != NULL) continue;
int offset = seg * hopSize;
int samplesToCopy = fftSize;
if (offset + samplesToCopy > signal->numSamples) {
samplesToCopy = signal->numSamples - offset;
memset(windowedSamples, 0, fftSize * sizeof(float));
memset(derivWindowedSamples, 0, fftSize * sizeof(float));
} else {
memcpy(windowedSamples, signal->samples + offset, fftSize * sizeof(float));
memcpy(derivWindowedSamples, signal->samples + offset, fftSize * sizeof(float));
}
for (int i = 0; i < fftSize; i++) {
float t = (float)i / (fftSize - 1);
float hann = 0.5f * (1.0f - cosf(2.0f * M_PI * t));
float derivHann = M_PI * sinf(2.0f * M_PI * t);
windowedSamples[i] *= hann;
derivWindowedSamples[i] *= derivHann;
}
// Normal STFT
for (int i = 0; i < fftSize; i++) complexInput[i] = windowedSamples[i] + 0.0f * I;
FFT(complexInput, fftOutput, fftSize, false);
result->segments[seg].numBins = numBins;
result->segments[seg].sampleOffset = offset;
result->segments[seg].sampleCount = samplesToCopy;
result->segments[seg].spectrum = (FrequencyData*)malloc(numBins * sizeof(FrequencyData));
for (int bin = 0; bin < numBins; bin++) {
result->segments[seg].spectrum[bin].frequency = (float)bin * signal->sampleRate / fftSize;
result->segments[seg].spectrum[bin].amplitude = (bin == 0) ? cabsf(fftOutput[bin]) / fftSize : 2.0f * cabsf(fftOutput[bin]) / fftSize;
result->segments[seg].spectrum[bin].phase = cargf(fftOutput[bin]);
}
// Derivative-window STFT for synchrosqueezing
result->segments[seg].derivativeSpectrum = (FrequencyData*)malloc(numBins * sizeof(FrequencyData));
for (int i = 0; i < fftSize; i++) complexInput[i] = derivWindowedSamples[i] + 0.0f * I;
FFT(complexInput, fftOutput, fftSize, false);
for (int bin = 0; bin < numBins; bin++) {
result->segments[seg].derivativeSpectrum[bin].frequency = (float)bin * signal->sampleRate / fftSize;
result->segments[seg].derivativeSpectrum[bin].amplitude = cabsf(fftOutput[bin]) / fftSize;
result->segments[seg].derivativeSpectrum[bin].phase = cargf(fftOutput[bin]);
}
}
free(windowedSamples);
free(derivWindowedSamples);
free(complexInput);
free(fftOutput);
return true;
}
// ============================================================================
// Audio Loading
// ============================================================================
@@ -529,16 +626,18 @@ static void GenerateSpectrogramTexture(StftResult* stft, Image* image, Texture2D
int height = stft->segments[0].numBins;
int fftSize = (height - 1) * 2;
float freqPerBin = (float)stft->sampleRate / fftSize;
*image = GenImageColor(width, height, BLACK);
Color* pixels = (Color*)image->data;
// Find max amplitude for normalization
// Find max amplitude for normalization (skip NULL segments)
float maxAmplitude = 0.0001f;
for (int seg = 0; seg < stft->numSegments; seg++)
for (int seg = 0; seg < stft->numSegments; seg++) {
if (stft->segments[seg].spectrum == NULL) continue;
for (int bin = 0; bin < stft->segments[seg].numBins; bin++)
if (stft->segments[seg].spectrum[bin].amplitude > maxAmplitude)
if (stft->segments[seg].spectrum[bin].amplitude > maxAmplitude)
maxAmplitude = stft->segments[seg].spectrum[bin].amplitude;
}
// ===== SYNCHROSQUEEZING =====
// Reassign energy to true frequencies using derivative STFT
@@ -550,6 +649,9 @@ static void GenerateSpectrogramTexture(StftResult* stft, Image* image, Texture2D
float noiseThreshold = maxAmplitude * 0.01f; // 1% of max amplitude
for (int seg = 0; seg < width; seg++) {
// Skip segments that haven't been computed yet (overview/high-res transition)
if (stft->segments[seg].spectrum == NULL) continue;
for (int bin = 0; bin < height; bin++) {
FrequencyData* V_f = &stft->segments[seg].spectrum[bin];
FrequencyData* V_fd = &stft->segments[seg].derivativeSpectrum[bin];
@@ -841,6 +943,8 @@ static void LoadSelectedFile(void)
app.loadingPhase = 0;
app.loadingProgress = 0.0f;
app.currentSTFTSegment = 0;
app.skipFactor = 1;
app.highResFinished = false;
app.timeSelectionStart = app.viewStart = 0.0f;
app.timeSelectionEnd = app.viewEnd = 1.0f;
app.freqSelectionStart = 0.0f;
@@ -1127,11 +1231,25 @@ static void DrawSidebar(void)
Rectangle fftPlus = { x + sidebarWidth - 40 * scale, y, 30 * scale, 25 * scale };
if (CheckCollisionPointRec(GetMousePosition(), fftMinus) && IsMouseButtonPressed(MOUSE_LEFT_BUTTON)) {
int newFFT = app.fftSize / 2;
if (newFFT >= FFT_SIZE_MIN) { app.fftSize = newFFT; app.stftComputed = false; app.loadingPhase = 0; needsRegen = true; }
if (newFFT >= FFT_SIZE_MIN) {
app.fftSize = newFFT;
app.stftComputed = false;
app.loadingPhase = 0;
app.skipFactor = 1;
app.highResFinished = false;
needsRegen = true;
}
}
if (CheckCollisionPointRec(GetMousePosition(), fftPlus) && IsMouseButtonPressed(MOUSE_LEFT_BUTTON)) {
int newFFT = app.fftSize * 2;
if (newFFT <= FFT_SIZE_MAX) { app.fftSize = newFFT; app.stftComputed = false; app.loadingPhase = 0; needsRegen = true; }
if (newFFT <= FFT_SIZE_MAX) {
app.fftSize = newFFT;
app.stftComputed = false;
app.loadingPhase = 0;
app.skipFactor = 1;
app.highResFinished = false;
needsRegen = true;
}
}
DrawRectangleRec(fftMinus, (Color){ 50, 50, 60, 255 });
DrawRectangleLinesEx(fftMinus, 1, GRAY);
@@ -1333,6 +1451,8 @@ int main(int argc, char* argv[])
app.cachedVisibleEndY = -1;
app.visibleTextureValid = false;
app.fftSize = FFT_SIZE_DEFAULT;
app.skipFactor = 1;
app.highResFinished = false;
app.isPlaying = false;
app.playbackFinished = false;
@@ -1360,6 +1480,8 @@ int main(int argc, char* argv[])
app.loadingPhase = 0;
app.loadingProgress = 0.0f;
app.currentSTFTSegment = 0;
app.skipFactor = 1;
app.highResFinished = false;
ComputeSTFTInit(&app.signal, &app.stft, app.fftSize);
TraceLog(LOG_INFO, "File loaded successfully");
}
@@ -1382,6 +1504,8 @@ int main(int argc, char* argv[])
app.loadingPhase = 0;
app.loadingProgress = 0.0f;
app.currentSTFTSegment = 0;
app.skipFactor = 1;
app.highResFinished = false;
app.viewStart = 0.0f; app.viewEnd = 1.0f;
ComputeSTFTInit(&app.signal, &app.stft, app.fftSize);
// Invalidate visible texture cache
@@ -1504,6 +1628,45 @@ int main(int argc, char* argv[])
}
if (IsMouseButtonReleased(MOUSE_LEFT_BUTTON)) app.isPanning = false;
// Auto-compute high-res segments when user zooms into a new range.
// Only triggers when the view is sufficiently zoomed in (narrow range).
// This prevents reprocessing the whole signal on zoom-out.
if (app.skipFactor > 1 && app.highResFinished && app.stft.numSegments > 0) {
float viewRange = app.viewEnd - app.viewStart;
// Only trigger when zoomed in to ~25% or less of the signal.
// When zoomed out, we keep whatever's already computed:
// high-res for visited regions, overview for the rest.
if (viewRange <= 0.25f) {
// Clamp to valid segment range
int viewStartSeg = (int)(app.viewStart * app.stft.numSegments);
int viewEndSeg = (int)(app.viewEnd * app.stft.numSegments);
if (viewStartSeg < 0) viewStartSeg = 0;
if (viewStartSeg >= app.stft.numSegments) viewStartSeg = app.stft.numSegments - 1;
if (viewEndSeg >= app.stft.numSegments) viewEndSeg = app.stft.numSegments - 1;
// Check if we're done processing or if we just finished and need high-res
if (app.stftComputed || (app.loadingPhase >= 2)) {
// Only check segments within the current visible range
for (int seg = viewStartSeg; seg <= viewEndSeg && seg < app.stft.numSegments; seg++) {
if (app.stft.segments[seg].spectrum == NULL) {
// Compute high-res for 50 segments at a time, staying within view
int startSeg = seg;
int endSeg = seg + 50;
if (endSeg > viewEndSeg + 1) endSeg = viewEndSeg + 1;
ComputeSTFTHighResRange(&app.signal, &app.stft, app.fftSize, startSeg, endSeg);
app.visibleTextureValid = false;
app.stftComputed = false;
app.loadingPhase = 2;
app.loadingProgress = 0.0f;
TraceLog(LOG_INFO, "High-res for view range (%d to %d)", startSeg, endSeg - 1);
break;
}
}
}
}
}
// Home/End keys
if (IsKeyPressed(KEY_HOME)) {
app.viewStart = 0.0f; app.viewEnd = 1.0f;
@@ -1698,6 +1861,8 @@ int main(int argc, char* argv[])
if (app.loadingPhase == 0) {
// Initialize STFT once
ComputeSTFTInit(&app.signal, &app.stft, app.fftSize);
app.skipFactor = ComputeSkipFactor(app.signal.duration);
app.highResFinished = true; // Overview loaded — ready for zoom-triggered high-res
app.currentSTFTSegment = 0;
app.loadingPhase = 1;
}
@@ -1724,7 +1889,8 @@ int main(int argc, char* argv[])
app.stftComputed = true;
app.loadingPhase = -1;
app.loadingProgress = 0.0f;
TraceLog(LOG_INFO, "STFT computed (%d segments)", app.stft.numSegments);
TraceLog(LOG_INFO, "STFT computed (%d segments, skipFactor=%d)",
app.stft.numSegments, app.skipFactor);
}
}
@@ -1764,9 +1930,9 @@ int main(int argc, char* argv[])
int pctW = MeasureTextScaled(pctText, 14);
DrawTextScaled(pctText, barX + barW / 2 - pctW / 2, barY + (int)(14 * scale), 14, WHITE);
// Duration estimate
// Duration estimate (account for skip factor — fewer segments to compute)
int estY = barY + (int)(28 * scale);
float estSec = app.signal.duration / app.signal.sampleRate * app.stft.numSegments / 200.0f;
float estSec = app.signal.duration / app.signal.sampleRate * app.stft.numSegments / (200.0f * app.skipFactor);
if (estSec > 0.5f && !isnan(estSec)) {
char estText[64];
snprintf(estText, sizeof(estText), "Estimated time: %.1f sec", estSec);