|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Script to extract SSL features from the audio waveforms. |
| 3 | +
|
| 4 | +The script uses the `speechbrain.integrations.hdf5.cached_item` module to cache the features. |
| 5 | +The cached features are used in the `train_speechllm.py` script to train the SpeechLLM ASR system. |
| 6 | +
|
| 7 | +Since we do the extractions within the pipeline in the dataloader, we must place |
| 8 | +our hparams elements directly on device, and use a default bsize of 1. |
| 9 | +
|
| 10 | +Example |
| 11 | +------- |
| 12 | +python extract_ssl_feats.py hparams/extract_ssl_feats.yaml |
| 13 | + --data_folder path/to/LibriSpeech \ |
| 14 | + --output_folder path/to/feats_cache \ |
| 15 | + --ssl_hub path/to/wavlm-large \ |
| 16 | + --feats_cache_dir path/to/feats_cache |
| 17 | + ...other_hparams... |
| 18 | +
|
| 19 | +Authors |
| 20 | +------- |
| 21 | + * Adel Moumen, 2025 |
| 22 | +""" |
| 23 | + |
| 24 | +import sys |
| 25 | +from pathlib import Path |
| 26 | + |
| 27 | +import torch |
| 28 | +from hyperpyyaml import load_hyperpyyaml |
| 29 | + |
| 30 | +import speechbrain as sb |
| 31 | +from speechbrain.integrations.hdf5.cached_item import CachedHDF5DynamicItem |
| 32 | +from speechbrain.utils.distributed import run_on_main |
| 33 | +from speechbrain.utils.logger import get_logger |
| 34 | + |
| 35 | +logger = get_logger(__name__) |
| 36 | + |
| 37 | + |
| 38 | +def dataio_prepare(hparams): |
| 39 | + """This function prepares the datasets to be used in the brain class. |
| 40 | + It also defines the data processing pipeline through user-defined functions. |
| 41 | + """ |
| 42 | + data_folder = hparams["data_folder"] |
| 43 | + |
| 44 | + # 2. Define audio pipeline: |
| 45 | + @sb.utils.data_pipeline.takes("wav") |
| 46 | + @sb.utils.data_pipeline.provides("sig") |
| 47 | + def audio_pipeline(wav): |
| 48 | + sig = sb.dataio.dataio.read_audio(wav) |
| 49 | + return sig |
| 50 | + |
| 51 | + normalizer = hparams["normalize"].to(hparams["device"]).eval() |
| 52 | + ssl_encoder = hparams["ssl"].to(hparams["device"]).eval() |
| 53 | + |
| 54 | + # Base compute function used by all cached wrappers (no file bound yet) |
| 55 | + @CachedHDF5DynamicItem.cache(hparams["feats_cache_dir"], compression="gzip") |
| 56 | + @sb.utils.data_pipeline.takes("id", "sig") |
| 57 | + @sb.utils.data_pipeline.provides("feats") |
| 58 | + def compute_feats(uid, sig): |
| 59 | + sig = sig.to(hparams["device"]).unsqueeze(0) |
| 60 | + length = torch.ones(1, device=hparams["device"]) |
| 61 | + with torch.no_grad(), torch.cuda.amp.autocast(dtype=hparams["dtype"]): |
| 62 | + feats = normalizer(sig, length) |
| 63 | + feats = ssl_encoder(feats, length) |
| 64 | + return feats.squeeze(0).cpu() |
| 65 | + |
| 66 | + dynamic_items = [audio_pipeline, compute_feats] |
| 67 | + output_keys = ["id", "sig", "feats"] |
| 68 | + |
| 69 | + train_data = sb.dataio.dataset.DynamicItemDataset.from_csv( |
| 70 | + csv_path=hparams["train_csv"], |
| 71 | + replacements={"data_root": data_folder}, |
| 72 | + dynamic_items=dynamic_items, |
| 73 | + output_keys=output_keys, |
| 74 | + ) |
| 75 | + |
| 76 | + # Build valid dataset with its own cached wrapper |
| 77 | + valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv( |
| 78 | + csv_path=hparams["valid_csv"], |
| 79 | + replacements={"data_root": data_folder}, |
| 80 | + dynamic_items=dynamic_items, |
| 81 | + output_keys=output_keys, |
| 82 | + ) |
| 83 | + |
| 84 | + # test is separate |
| 85 | + test_datasets = {} |
| 86 | + for csv_file in hparams["test_csv"]: |
| 87 | + name = Path(csv_file).stem |
| 88 | + test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv( |
| 89 | + csv_path=csv_file, |
| 90 | + replacements={"data_root": data_folder}, |
| 91 | + dynamic_items=dynamic_items, |
| 92 | + output_keys=output_keys, |
| 93 | + ) |
| 94 | + |
| 95 | + datasets = {"train": train_data, "valid": valid_data} | { |
| 96 | + k: v for k, v in test_datasets.items() |
| 97 | + } |
| 98 | + |
| 99 | + for stage, dataset in datasets.items(): |
| 100 | + logger.info(f"Iterating {stage} dataset to warm the cache.") |
| 101 | + dataset.iterate_once(output_keys=["feats"]) |
| 102 | + |
| 103 | + |
| 104 | +if __name__ == "__main__": |
| 105 | + # CLI: |
| 106 | + hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:]) |
| 107 | + with open(hparams_file, encoding="utf-8") as fin: |
| 108 | + hparams = load_hyperpyyaml(fin, overrides) |
| 109 | + |
| 110 | + # create ddp_group with the right communication protocol |
| 111 | + sb.utils.distributed.ddp_init_group(run_opts) |
| 112 | + |
| 113 | + # 1. # Dataset prep (parsing Librispeech) |
| 114 | + from librispeech_prepare import prepare_librispeech # noqa |
| 115 | + |
| 116 | + # Create experiment directory |
| 117 | + sb.create_experiment_directory( |
| 118 | + experiment_directory=hparams["output_folder"], |
| 119 | + hyperparams_to_save=hparams_file, |
| 120 | + overrides=overrides, |
| 121 | + ) |
| 122 | + |
| 123 | + # multi-gpu (ddp) save data preparation |
| 124 | + run_on_main( |
| 125 | + prepare_librispeech, |
| 126 | + kwargs={ |
| 127 | + "data_folder": hparams["data_folder"], |
| 128 | + "tr_splits": hparams["train_splits"], |
| 129 | + "dev_splits": hparams["dev_splits"], |
| 130 | + "te_splits": hparams["test_splits"], |
| 131 | + "save_folder": hparams["output_folder"], |
| 132 | + "merge_lst": hparams["train_splits"], |
| 133 | + "merge_name": "train.csv", |
| 134 | + "skip_prep": hparams["skip_prep"], |
| 135 | + }, |
| 136 | + ) |
| 137 | + logger.info("Preparing data...") |
| 138 | + dataio_prepare(hparams) |
| 139 | + logger.info("Done preparing data") |
0 commit comments