Skip to content

Commit eb264df

Browse files
asumagicasumagic
committed
Try switching recipe tests to logging
For consistency and because using stdout by default is rubbish due to file syncing, in some cases nothing shows up in slurm logs until the very end. Might require some formatting fixing later to look consistent.
1 parent 6fb0ff8 commit eb264df

1 file changed

Lines changed: 56 additions & 37 deletions

File tree

‎tests/utils/recipe_tests.py‎

Lines changed: 56 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
"""
77

88
import csv
9+
import logging
910
import os
1011
import pydoc
1112
import re
@@ -56,7 +57,7 @@ def check_row_for_test(row, filters_fields, filters, test_field):
5657
test_flag = True
5758
test = test and test_flag
5859
else:
59-
print("\tError in filters_fields and filters definition.")
60+
logging.error(" Error in filters_fields and filters definition.")
6061
test = False
6162

6263
if test:
@@ -127,13 +128,13 @@ def prepare_test(
127128
test_message = {}
128129

129130
# Loop over all recipe CSVs
130-
print(f"\tfilters_fields={filters_fields} => filters={filters}")
131+
logging.info(f" filters_fields={filters_fields} => filters={filters}")
131132
for recipe_csvfile in os.listdir(recipe_folder):
132133
# skip setup scripts; consider CSV files only
133134
if recipe_csvfile in __skip_list:
134135
continue
135136

136-
print(f"Loading recipes from: {recipe_csvfile}")
137+
logging.info(f"Loading recipes from: {recipe_csvfile}")
137138
# Detect needed information for the recipe tests
138139
with open(
139140
os.path.join(recipe_folder, recipe_csvfile), newline=""
@@ -144,7 +145,7 @@ def prepare_test(
144145
if not (
145146
check_row_for_test(row, filters_fields, filters, test_field)
146147
):
147-
print(f"\tSkipped {recipe_id}")
148+
logging.info(f" Skipped {recipe_id}")
148149
continue
149150
test_script[recipe_id] = row[script_field].strip()
150151
test_hparam[recipe_id] = row[hparam_field].strip()
@@ -195,8 +196,8 @@ def check_files(
195196
for file_to_check in files_to_check:
196197
check_path = os.path.join(output_folder, file_to_check)
197198
if not os.path.exists(check_path):
198-
print(
199-
"\tERROR: The recipe %s does not contain the expected file %s"
199+
logging.error(
200+
" The recipe %s does not contain the expected file %s"
200201
% (recipe_id, check_path)
201202
)
202203
check = False
@@ -244,9 +245,10 @@ def check_performance(
244245
epoch = performance_to_check[3].strip()
245246

246247
if not os.path.exists(filename):
247-
print(
248-
"\tERROR: The file %s of recipe %s does not exist (needed for performance checks)"
249-
% (filename, recipe_id)
248+
logging.error(
249+
" The file %s of recipe %s does not exist (needed for performance checks)",
250+
filename,
251+
recipe_id,
250252
)
251253

252254
return False
@@ -271,8 +273,8 @@ def check_performance(
271273

272274
# Raising an error if there are no lines after applying the filter
273275
if len(lines_filt) == 0:
274-
print(
275-
"\tERROR: No entries %s in %s (recipe %s). See performance_check entry."
276+
logging.error(
277+
" No entries %s in %s (recipe %s). See performance_check entry."
276278
% (epoch, filename, recipe_id)
277279
)
278280
return False
@@ -283,18 +285,24 @@ def check_performance(
283285
var_value = extract_value(line, variable)
284286

285287
if var_value is None:
286-
print(
287-
"\tERROR: The file %s of recipe %s does not contain the variable %s (needed for performance checks)"
288-
% (filename, recipe_id, variable)
288+
logging.error(
289+
" The file %s of recipe %s does not contain the variable %s (needed for performance checks)",
290+
filename,
291+
recipe_id,
292+
variable,
289293
)
290294
return False
291295
var_value = float(var_value)
292296
check = check_threshold(threshold, var_value)
293297

294298
if not check:
295-
print(
296-
"\tERROR: The variable %s of file %s (recipe %s) violated the specified threshold (%s %s)"
297-
% (variable, filename, recipe_id, var_value, threshold)
299+
logging.error(
300+
" The variable %s of file %s (recipe %s) violated the specified threshold (%s %s)",
301+
variable,
302+
filename,
303+
recipe_id,
304+
var_value,
305+
threshold,
298306
)
299307

300308
break
@@ -461,9 +469,11 @@ def run_recipe_tests(
461469
-------
462470
python -c 'from speechbrain.utils.recipe_tests import run_recipe_tests; print("TEST FAILED!") if not(run_recipe_tests(filters_fields=["Dataset", "Task"], filters=[["AISHELL-1", "CommonVoice"], "SSL"])) else print("TEST PASSED")'
463471
"""
472+
logging.getLogger().setLevel(logging.INFO)
473+
464474
# Create the output folder (where the tests results will be saved)
465475
os.makedirs(output_folder, exist_ok=True)
466-
print("Test outputs will be put in %s" % (output_folder))
476+
logging.info("Test outputs will be put in %s", output_folder)
467477

468478
# Read the csv recipe file and detect which tests we have to run
469479
(
@@ -485,7 +495,9 @@ def run_recipe_tests(
485495

486496
# Early stop if there are no recipes to test
487497
if len(test_script) == 0:
488-
print("No recipes found for testing (please check recipe filters).")
498+
logging.error(
499+
"No recipes found for testing (please check recipe filters)."
500+
)
489501
return False
490502

491503
# Download all upfront
@@ -526,15 +538,17 @@ def run_recipe_tests(
526538
if len(check_str) == 0:
527539
continue
528540

529-
print(
530-
"(%i/%i) Running test for %s..."
531-
% (i + 1, len(test_script.keys()), recipe_id)
541+
logging.info(
542+
"(%i/%i) Running test for %s...",
543+
i + 1,
544+
len(test_script.keys()),
545+
recipe_id,
532546
)
533547

534548
if recipe_id in test_download:
535549
download_cmds = test_download[recipe_id].split(";")
536550
for download_cmd in download_cmds:
537-
print("\t" + download_cmd)
551+
logging.info(" " + download_cmd)
538552
eval(download_cmd)
539553

540554
# Check for setup scripts
@@ -569,13 +583,13 @@ def run_recipe_tests(
569583

570584
# Print message (if any)
571585
if recipe_id in test_message:
572-
print("\t\t" + test_message[recipe_id])
586+
logging.info(" %s", test_message[recipe_id])
573587

574588
# Running the test
575589
time_start = time()
576590
return_code = run_test_cmd(cmd, stdout_file, stderr_file)
577591
test_duration = time() - time_start
578-
print("\t... %.2fs" % test_duration)
592+
logging.info(" ... %.2fs", test_duration)
579593

580594
# Tear down
581595
td_script = os.path.join(os.path.dirname(setup_script), "tear_down")
@@ -584,15 +598,18 @@ def run_recipe_tests(
584598

585599
# Check return code
586600
if return_code != 0:
587-
print(
588-
"\tERROR: Error in %s (%s). Check %s and %s for more info."
589-
% (recipe_id, test_hparam[recipe_id], stderr_file, stdout_file)
601+
logging.error(
602+
" Error in %s (%s). Check %s and %s for more info.",
603+
recipe_id,
604+
test_hparam[recipe_id],
605+
stderr_file,
606+
stdout_file,
590607
)
591608
check = False
592609

593610
# Checks
594611
if do_checks and len(check_str) > 0:
595-
print("\t...checking files & performance...")
612+
logging.info(" ...checking files & performance...")
596613

597614
# Check if the expected files exist
598615
check &= check_files(check_str, output_fold, recipe_id)
@@ -636,9 +653,11 @@ def download_only_test(
636653
if len(check_str) == 0:
637654
continue
638655

639-
print(
640-
"(%i/%i) Collecting pretrained models for %s..."
641-
% (i + 1, len(test_script.keys()), recipe_id)
656+
logging.info(
657+
"(%i/%i) Collecting pretrained models for %s...",
658+
i + 1,
659+
len(test_script.keys()),
660+
recipe_id,
642661
)
643662

644663
output_fold = os.path.join(output_folder, recipe_id)
@@ -780,12 +799,12 @@ def load_yaml_test(
780799

781800
# Avoid files lister in avoid_list
782801
if hparam_file in avoid_list:
783-
print(
784-
f"\t({i + 1}/{len(test_script.keys())}) Skipped: {hparam_file}! (check avoid_list for details)"
802+
logging.info(
803+
f" ({i + 1}/{len(test_script.keys())}) Skipped: {hparam_file}! (check avoid_list for details)"
785804
)
786805
continue
787806

788-
print(
807+
logging.info(
789808
"(%i/%i) Checking %s..."
790809
% (i + 1, len(test_script.keys()), hparam_file)
791810
)
@@ -833,9 +852,9 @@ def load_yaml_test(
833852
try:
834853
_ = load_hyperpyyaml(fin, overrides)
835854
except Exception as e:
836-
print("\t" + str(e))
855+
logging.error(" %s", str(e))
837856
check = False
838-
print("\tERROR: cannot load %s" % (hparam_file))
857+
logging.error(" cannot load %s", hparam_file)
839858
if tag_custom_model is not None:
840859
if tag_custom_model in sys.modules:
841860
del sys.modules[tag_custom_model]

0 commit comments

Comments
 (0)