using System;
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Threading.Tasks;
namespace Spectrogram
{
public class SpectrogramGenerator
{
///
/// Number of pixel columns (FFT samples) in the spectrogram image
///
public int Width { get => FFTs.Count; }
///
/// Number of pixel rows (frequency bins) in the spectrogram image
///
public int Height { get => Settings.Height; }
///
/// Number of samples to use for each FFT (must be a power of 2)
///
public int FftSize { get => Settings.FftSize; }
///
/// Vertical resolution (frequency bin size depends on FftSize and SampleRate)
///
public double HzPerPx { get => Settings.HzPerPixel; }
///
/// Horizontal resolution (seconds per pixel depends on StepSize)
///
public double SecPerPx { get => Settings.StepLengthSec; }
///
/// Number of FFTs that remain to be processed for data which has been added but not yet analuyzed
///
public int FftsToProcess { get => (UnprocessedData.Count - Settings.FftSize) / Settings.StepSize; }
///
/// Total number of FFT steps processed
///
public int FftsProcessed { get; private set; }
///
/// Index of the pixel column which will be populated next. Location of vertical line for wrap-around displays.
///
public int NextColumnIndex { get => (FftsProcessed + rollOffset) % Width; }
///
/// This value is added to displayed frequency axis tick labels
///
public int OffsetHz { get => Settings.OffsetHz; set { Settings.OffsetHz = value; } }
///
/// Number of samples per second
///
public int SampleRate { get => Settings.SampleRate; }
///
/// Number of samples to step forward after each FFT is processed.
/// This value controls the horizontal resolution of the spectrogram.
///
public int StepSize { get => Settings.StepSize; }
///
/// The spectrogram is trimmed to cut-off frequencies below this value.
///
public double FreqMax { get => Settings.FreqMax; }
///
/// The spectrogram is trimmed to cut-off frequencies above this value.
///
public double FreqMin { get => Settings.FreqMin; }
///
/// This module contains detailed FFT/Spectrogram settings
///
private readonly Settings Settings;
///
/// This is the list of FFTs which is translated to the spectrogram image when it is requested.
/// The length of this list is the spectrogram width.
/// The length of the arrays in this list is the spectrogram height.
///
private readonly List FFTs = new List();
///
/// This list contains data values which have not yet been processed.
/// Process() processes all unprocessed data.
/// This list may not be empty after processing if there aren't enough values to fill a full FFT (FftSize).
///
private readonly List UnprocessedData;
///
/// Colormap to use when generating future FFTs.
///
public Colormap Colormap = Colormap.Viridis;
///
/// Instantiate a spectrogram generator.
/// This module calculates the FFT over a moving window as data comes in.
/// Using the Add() method to load new data and process it as it arrives.
///
/// Number of samples per second (Hz)
/// Number of samples to use for each FFT operation. This value must be a power of 2.
/// Number of samples to step forward
/// Frequency data lower than this value (Hz) will not be stored
/// Frequency data higher than this value (Hz) will not be stored
/// Spectrogram output will always be sized to this width (column count)
/// This value will be added to displayed frequency axis tick labels
/// Analyze this data immediately (alternative to calling Add() later)
public SpectrogramGenerator(
int sampleRate,
int fftSize,
int stepSize,
double minFreq = 0,
double maxFreq = double.PositiveInfinity,
int? fixedWidth = null,
int offsetHz = 0,
List initialAudioList = null)
{
Settings = new Settings(sampleRate, fftSize, stepSize, minFreq, maxFreq, offsetHz);
UnprocessedData = initialAudioList ?? new List();
if (fixedWidth.HasValue)
SetFixedWidth(fixedWidth.Value);
}
public override string ToString()
{
double processedSamples = FFTs.Count * Settings.StepSize + Settings.FftSize;
double processedSec = processedSamples / Settings.SampleRate;
string processedTime = (processedSec < 60) ? $"{processedSec:N2} sec" : $"{processedSec / 60.0:N2} min";
return $"Spectrogram ({Width}, {Height})" +
$"\n Vertical ({Height} px): " +
$"{Settings.FreqMin:N0} - {Settings.FreqMax:N0} Hz, " +
$"FFT size: {Settings.FftSize:N0} samples, " +
$"{Settings.HzPerPixel:N2} Hz/px" +
$"\n Horizontal ({Width} px): " +
$"{processedTime}, " +
$"window: {Settings.FftLengthSec:N2} sec, " +
$"step: {Settings.StepLengthSec:N2} sec, " +
$"overlap: {Settings.StepOverlapFrac * 100:N0}%";
}
[Obsolete("Assign to the Colormap field")]
///
/// Set the colormap to use for future renders
///
public void SetColormap(Colormap cmap)
{
Colormap = cmap ?? this.Colormap;
}
///
/// Load a custom window kernel to multiply against each FFT sample prior to processing.
/// Windows must be at least the length of FftSize and typically have a sum of 1.0.
///
public void SetWindow(double[] newWindow)
{
if (newWindow.Length > Settings.FftSize)
throw new ArgumentException("window length cannot exceed FFT size");
for (int i = 0; i < Settings.FftSize; i++)
Settings.Window[i] = 0;
int offset = (Settings.FftSize - newWindow.Length) / 2;
Array.Copy(newWindow, 0, Settings.Window, offset, newWindow.Length);
}
[Obsolete("use the Add() method", true)]
public void AddExtend(float[] values) { }
[Obsolete("use the Add() method", true)]
public void AddCircular(float[] values) { }
[Obsolete("use the Add() method", true)]
public void AddScroll(float[] values) { }
///
/// Load new data into the spectrogram generator
///
public void Add(IEnumerable audio, bool process = true)
{
UnprocessedData.AddRange(audio);
if (process)
Process();
}
///
/// The roll offset is used to calculate NextColumnIndex and can be set to a positive number
/// to begin adding new columns to the center of the spectrogram.
/// This can also be used to artificially move the next column index to zero even though some
/// data has already been accumulated.
///
private int rollOffset = 0;
///
/// Reset the next column index such that the next processed FFT will appear at the far left of the spectrogram.
///
///
public void RollReset(int offset = 0)
{
rollOffset = -FftsProcessed + offset;
}
///
/// Perform FFT analysis on all unprocessed data
///
public double[][] Process()
{
if (FftsToProcess < 1)
return null;
int newFftCount = FftsToProcess;
double[][] newFfts = new double[newFftCount][];
Parallel.For(0, newFftCount, newFftIndex =>
{
FftSharp.Complex[] buffer = new FftSharp.Complex[Settings.FftSize];
int sourceIndex = newFftIndex * Settings.StepSize;
for (int i = 0; i < Settings.FftSize; i++)
buffer[i].Real = UnprocessedData[sourceIndex + i] * Settings.Window[i];
FftSharp.Transform.FFT(buffer);
newFfts[newFftIndex] = new double[Settings.Height];
for (int i = 0; i < Settings.Height; i++)
newFfts[newFftIndex][i] = buffer[Settings.FftIndex1 + i].Magnitude / Settings.FftSize;
});
foreach (var newFft in newFfts)
FFTs.Add(newFft);
FftsProcessed += newFfts.Length;
UnprocessedData.RemoveRange(0, newFftCount * Settings.StepSize);
PadOrTrimForFixedWidth();
return newFfts;
}
///
/// Return a list of the mel-scaled FFTs contained in this spectrogram
///
/// Total number of output bins to use. Choose a value significantly smaller than Height.
public List GetMelFFTs(int melBinCount)
{
if (Settings.FreqMin != 0)
throw new InvalidOperationException("cannot get Mel spectrogram unless minimum frequency is 0Hz");
var fftsMel = new List();
foreach (var fft in FFTs)
fftsMel.Add(FftSharp.Transform.MelScale(fft, SampleRate, melBinCount));
return fftsMel;
}
///
/// Create and return a spectrogram bitmap from the FFTs stored in memory.
///
/// Multiply the output by a fixed value to change its brightness.
/// If true, output will be log-transformed.
/// If dB scaling is in use, this multiplier will be applied before log transformation.
/// Behavior of the spectrogram when it is full of data.
/// Roll (true) adds new columns on the left overwriting the oldest ones.
/// Scroll (false) slides the whole image to the left and adds new columns to the right.
public Bitmap GetBitmap(double intensity = 1, bool dB = false, double dBScale = 1, bool roll = false) =>
Image.GetBitmap(FFTs, Colormap, intensity, dB, dBScale, roll, NextColumnIndex);
///
/// Create a Mel-scaled spectrogram.
///
/// Total number of output bins to use. Choose a value significantly smaller than Height.
/// Multiply the output by a fixed value to change its brightness.
/// If true, output will be log-transformed.
/// If dB scaling is in use, this multiplier will be applied before log transformation.
/// Behavior of the spectrogram when it is full of data.
/// Roll (true) adds new columns on the left overwriting the oldest ones.
/// Scroll (false) slides the whole image to the left and adds new columns to the right.
public Bitmap GetBitmapMel(int melBinCount = 25, double intensity = 1, bool dB = false, double dBScale = 1, bool roll = false) =>
Image.GetBitmap(GetMelFFTs(melBinCount), Colormap, intensity, dB, dBScale, roll, NextColumnIndex);
[Obsolete("use SaveImage()", true)]
public void SaveBitmap(Bitmap bmp, string fileName) { }
///
/// Generate the spectrogram and save it as an image file.
///
/// Path of the file to save.
/// Multiply the output by a fixed value to change its brightness.
/// If true, output will be log-transformed.
/// If dB scaling is in use, this multiplier will be applied before log transformation.
/// Behavior of the spectrogram when it is full of data.
/// Roll (true) adds new columns on the left overwriting the oldest ones.
/// Scroll (false) slides the whole image to the left and adds new columns to the right.
public void SaveImage(string fileName, double intensity = 1, bool dB = false, double dBScale = 1, bool roll = false)
{
if (FFTs.Count == 0)
throw new InvalidOperationException("Spectrogram contains no data. Use Add() to add signal data.");
string extension = Path.GetExtension(fileName).ToLower();
ImageFormat fmt;
if (extension == ".bmp")
fmt = ImageFormat.Bmp;
else if (extension == ".png")
fmt = ImageFormat.Png;
else if (extension == ".jpg" || extension == ".jpeg")
fmt = ImageFormat.Jpeg;
else if (extension == ".gif")
fmt = ImageFormat.Gif;
else
throw new ArgumentException("unknown file extension");
Image.GetBitmap(FFTs, Colormap, intensity, dB, dBScale, roll, NextColumnIndex).Save(fileName, fmt);
}
///
/// Create and return a spectrogram bitmap from the FFTs stored in memory.
/// The output will be scaled-down vertically by binning according to a reduction factor and keeping the brightest pixel value in each bin.
///
/// Multiply the output by a fixed value to change its brightness.
/// If true, output will be log-transformed.
/// If dB scaling is in use, this multiplier will be applied before log transformation.
/// Behavior of the spectrogram when it is full of data.
///
public Bitmap GetBitmapMax(double intensity = 1, bool dB = false, double dBScale = 1, bool roll = false, int reduction = 4)
{
List ffts2 = new List();
for (int i = 0; i < FFTs.Count; i++)
{
double[] d1 = FFTs[i];
double[] d2 = new double[d1.Length / reduction];
for (int j = 0; j < d2.Length; j++)
for (int k = 0; k < reduction; k++)
d2[j] = Math.Max(d2[j], d1[j * reduction + k]);
ffts2.Add(d2);
}
return Image.GetBitmap(ffts2, Colormap, intensity, dB, dBScale, roll, NextColumnIndex);
}
///
/// Export spectrogram data using the Spectrogram File Format (SFF)
///
public void SaveData(string filePath, int melBinCount = 0)
{
if (!filePath.EndsWith(".sff", StringComparison.OrdinalIgnoreCase))
filePath += ".sff";
new SFF(this, melBinCount).Save(filePath);
}
///
/// Defines the total number of FFTs (spectrogram columns) to store in memory. Determines Width.
///
private int fixedWidth = 0;
///
/// Configure the Spectrogram to maintain a fixed number of pixel columns.
/// Zeros will be added to padd existing data to achieve this width, and extra columns will be deleted.
///
public void SetFixedWidth(int width)
{
fixedWidth = width;
PadOrTrimForFixedWidth();
}
private void PadOrTrimForFixedWidth()
{
if (fixedWidth > 0)
{
int overhang = Width - fixedWidth;
if (overhang > 0)
FFTs.RemoveRange(0, overhang);
while (FFTs.Count < fixedWidth)
FFTs.Insert(0, new double[Height]);
}
}
///
/// Get a vertical image containing ticks and tick labels for the frequency axis.
///
/// size (pixels)
/// number to add to each tick label
/// length of each tick mark (pixels)
/// bin size for vertical data reduction
public Bitmap GetVerticalScale(int width, int offsetHz = 0, int tickSize = 3, int reduction = 1)
{
return Scale.Vertical(width, Settings, offsetHz, tickSize, reduction);
}
///
/// Return the vertical position (pixel units) for the given frequency
///
public int PixelY(double frequency, int reduction = 1)
{
int pixelsFromZeroHz = (int)(Settings.PxPerHz * frequency / reduction);
int pixelsFromMinFreq = pixelsFromZeroHz - Settings.FftIndex1 / reduction + 1;
int pixelRow = Settings.Height / reduction - 1 - pixelsFromMinFreq;
return pixelRow - 1;
}
///
/// Return the list of FFTs in memory underlying the spectrogram.
/// This list may continue to evolve after it is returned.
///
public List GetFFTs()
{
return FFTs;
}
///
/// Return frequency and magnitude of the dominant frequency.
///
/// If true, only the latest FFT will be assessed.
public (double freqHz, double magRms) GetPeak(bool latestFft = true)
{
if (FFTs.Count == 0)
return (double.NaN, double.NaN);
if (latestFft == false)
throw new NotImplementedException("peak of mean of all FFTs not yet supported");
double[] freqs = FFTs[FFTs.Count - 1];
int peakIndex = 0;
double peakMagnitude = 0;
for (int i = 0; i < freqs.Length; i++)
{
if (freqs[i] > peakMagnitude)
{
peakMagnitude = freqs[i];
peakIndex = i;
}
}
double maxFreq = SampleRate / 2;
double peakFreqFrac = peakIndex / (double)freqs.Length;
double peakFreqHz = maxFreq * peakFreqFrac;
return (peakFreqHz, peakMagnitude);
}
}
}