From 47e0856f6786c74e8bdae79e168599ff471b8d19 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Wed, 20 May 2026 14:43:54 -0500 Subject: [PATCH 1/2] Modified the integtests to support user-requested deletion of HDF5 files at the end of each test. --- integtest/disabled_output_test.py | 8 ++++-- integtest/hdf5_compression_test.py | 35 ++--------------------- integtest/insufficient_disk_space_test.py | 29 ++----------------- integtest/large_trigger_record_test.py | 29 ++----------------- integtest/max_file_size_test.py | 35 ++--------------------- integtest/multiple_data_writers_test.py | 8 ++++-- integtest/offline_prod_run_test.py | 8 ++++-- integtest/trmonrequestor_test.py | 8 ++++-- 8 files changed, 36 insertions(+), 124 deletions(-) diff --git a/integtest/disabled_output_test.py b/integtest/disabled_output_test.py index aa35913..def5c4c 100644 --- a/integtest/disabled_output_test.py +++ b/integtest/disabled_output_test.py @@ -7,9 +7,9 @@ import integrationtest.data_file_checks as data_file_checks import integrationtest.log_file_checks as log_file_checks -import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions as utility_functions from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir from integrationtest.verbosity_helper import IntegtestVerbosityLevels @@ -171,7 +171,7 @@ def test_dunerc_success(run_dunerc, caplog): # checks for run control success, problems during pytest setup, etc. - basic_checks.basic_checks(run_dunerc, caplog, print_test_name=True) + utility_functions.basic_checks(run_dunerc, caplog, print_test_name=True) def test_log_files(run_dunerc): @@ -220,3 +220,7 @@ def test_data_files(run_dunerc): data_file, fragment_check_list[jdx] ) assert all_ok + + +def test_cleanup(run_dunerc): + utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=False) diff --git a/integtest/hdf5_compression_test.py b/integtest/hdf5_compression_test.py index 02d3f13..034c20e 100644 --- a/integtest/hdf5_compression_test.py +++ b/integtest/hdf5_compression_test.py @@ -5,9 +5,9 @@ import integrationtest.data_file_checks as data_file_checks import integrationtest.log_file_checks as log_file_checks -import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions as utility_functions from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir from integrationtest.verbosity_helper import IntegtestVerbosityLevels @@ -241,7 +241,7 @@ def test_dunerc_success(run_dunerc, caplog): # checks for run control success, problems during pytest setup, etc. - basic_checks.basic_checks(run_dunerc, caplog, print_test_name=False) + utility_functions.basic_checks(run_dunerc, caplog, print_test_name=False) def test_log_files(run_dunerc): @@ -304,33 +304,4 @@ def test_tpstream_files(run_dunerc): def test_cleanup(run_dunerc): - pathlist_string = "" - filelist_string = "" - for data_file in run_dunerc.data_files: - filelist_string += " " + str(data_file) - if str(data_file.parent) not in pathlist_string: - pathlist_string += " " + str(data_file.parent) - for data_file in run_dunerc.tpset_files: - filelist_string += " " + str(data_file) - if str(data_file.parent) not in pathlist_string: - pathlist_string += " " + str(data_file.parent) - - if pathlist_string and filelist_string: - if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug): - print("============================================") - print("Listing the hdf5 files before deleting them:") - print("============================================") - - os.system(f"df -h {pathlist_string}") - print("--------------------") - os.system(f"ls -alF {filelist_string}") - - for data_file in run_dunerc.data_files: - data_file.unlink() - for data_file in run_dunerc.tpset_files: - data_file.unlink() - - if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug): - print("--------------------") - os.system(f"df -h {pathlist_string}") - print("============================================") + utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=True) diff --git a/integtest/insufficient_disk_space_test.py b/integtest/insufficient_disk_space_test.py index c504e66..a02c01b 100644 --- a/integtest/insufficient_disk_space_test.py +++ b/integtest/insufficient_disk_space_test.py @@ -5,9 +5,9 @@ import integrationtest.data_file_checks as data_file_checks import integrationtest.log_file_checks as log_file_checks -import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions as utility_functions from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir from integrationtest.verbosity_helper import IntegtestVerbosityLevels @@ -154,7 +154,7 @@ def test_dunerc_success(run_dunerc, caplog): # checks for run control success, problems during pytest setup, etc. - basic_checks.basic_checks(run_dunerc, caplog, print_test_name=False) + utility_functions.basic_checks(run_dunerc, caplog, print_test_name=False) def test_log_files(run_dunerc): @@ -205,27 +205,4 @@ def test_data_files(run_dunerc): def test_cleanup(run_dunerc): - pathlist_string = "" - filelist_string = "" - for data_file in run_dunerc.data_files: - filelist_string += " " + str(data_file) - if str(data_file.parent) not in pathlist_string: - pathlist_string += " " + str(data_file.parent) - - if pathlist_string and filelist_string: - if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug): - print("============================================") - print("Listing the hdf5 files before deleting them:") - print("============================================") - - os.system(f"df -h {pathlist_string}") - print("--------------------") - os.system(f"ls -alF {filelist_string}") - - for data_file in run_dunerc.data_files: - data_file.unlink() - - if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug): - print("--------------------") - os.system(f"df -h {pathlist_string}") - print("============================================") + utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=True) diff --git a/integtest/large_trigger_record_test.py b/integtest/large_trigger_record_test.py index be3ff21..38d6ff8 100644 --- a/integtest/large_trigger_record_test.py +++ b/integtest/large_trigger_record_test.py @@ -12,9 +12,9 @@ import integrationtest.data_file_checks as data_file_checks import integrationtest.log_file_checks as log_file_checks -import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions as utility_functions from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir from integrationtest.verbosity_helper import IntegtestVerbosityLevels @@ -167,7 +167,7 @@ def test_dunerc_success(run_dunerc, caplog): # checks for run control success, problems during pytest setup, etc. - basic_checks.basic_checks(run_dunerc, caplog, print_test_name=True) + utility_functions.basic_checks(run_dunerc, caplog, print_test_name=True) def test_log_files(run_dunerc): @@ -222,27 +222,4 @@ def test_data_files(run_dunerc): def test_cleanup(run_dunerc): - pathlist_string = "" - filelist_string = "" - for data_file in run_dunerc.data_files: - filelist_string += " " + str(data_file) - if str(data_file.parent) not in pathlist_string: - pathlist_string += " " + str(data_file.parent) - - if pathlist_string and filelist_string: - if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug): - print("============================================") - print("Listing the hdf5 files before deleting them:") - print("============================================") - - os.system(f"df -h {pathlist_string}") - print("--------------------") - os.system(f"ls -alF {filelist_string}") - - for data_file in run_dunerc.data_files: - data_file.unlink() - - if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug): - print("--------------------") - os.system(f"df -h {pathlist_string}") - print("============================================") + utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=True) diff --git a/integtest/max_file_size_test.py b/integtest/max_file_size_test.py index 12a072b..ce01751 100644 --- a/integtest/max_file_size_test.py +++ b/integtest/max_file_size_test.py @@ -5,9 +5,9 @@ import integrationtest.data_file_checks as data_file_checks import integrationtest.log_file_checks as log_file_checks -import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions as utility_functions from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir from integrationtest.verbosity_helper import IntegtestVerbosityLevels @@ -210,7 +210,7 @@ def test_dunerc_success(run_dunerc, caplog): # checks for run control success, problems during pytest setup, etc. - basic_checks.basic_checks(run_dunerc, caplog, print_test_name=False) + utility_functions.basic_checks(run_dunerc, caplog, print_test_name=False) def test_log_files(run_dunerc): @@ -273,33 +273,4 @@ def test_tpstream_files(run_dunerc): def test_cleanup(run_dunerc): - pathlist_string = "" - filelist_string = "" - for data_file in run_dunerc.data_files: - filelist_string += " " + str(data_file) - if str(data_file.parent) not in pathlist_string: - pathlist_string += " " + str(data_file.parent) - for data_file in run_dunerc.tpset_files: - filelist_string += " " + str(data_file) - if str(data_file.parent) not in pathlist_string: - pathlist_string += " " + str(data_file.parent) - - if pathlist_string and filelist_string: - if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug): - print("============================================") - print("Listing the hdf5 files before deleting them:") - print("============================================") - - os.system(f"df -h {pathlist_string}") - print("--------------------") - os.system(f"ls -alF {filelist_string}") - - for data_file in run_dunerc.data_files: - data_file.unlink() - for data_file in run_dunerc.tpset_files: - data_file.unlink() - - if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug): - print("--------------------") - os.system(f"df -h {pathlist_string}") - print("============================================") + utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=True) diff --git a/integtest/multiple_data_writers_test.py b/integtest/multiple_data_writers_test.py index 24ad902..01350f5 100644 --- a/integtest/multiple_data_writers_test.py +++ b/integtest/multiple_data_writers_test.py @@ -5,9 +5,9 @@ import integrationtest.data_file_checks as data_file_checks import integrationtest.log_file_checks as log_file_checks -import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions as utility_functions from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir from integrationtest.verbosity_helper import IntegtestVerbosityLevels @@ -142,7 +142,7 @@ def test_dunerc_success(run_dunerc, caplog): # checks for run control success, problems during pytest setup, etc. - basic_checks.basic_checks(run_dunerc, caplog, print_test_name=False) + utility_functions.basic_checks(run_dunerc, caplog, print_test_name=False) def test_log_files(run_dunerc): @@ -180,3 +180,7 @@ def test_data_files(run_dunerc): data_file, fragment_check_list[jdx] ) assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more raw data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" + + +def test_cleanup(run_dunerc): + utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=False) diff --git a/integtest/offline_prod_run_test.py b/integtest/offline_prod_run_test.py index 51d3b95..0c331f8 100644 --- a/integtest/offline_prod_run_test.py +++ b/integtest/offline_prod_run_test.py @@ -5,8 +5,8 @@ import integrationtest.data_file_checks as data_file_checks import integrationtest.log_file_checks as log_file_checks -import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes +import integrationtest.utility_functions as utility_functions from integrationtest.verbosity_helper import IntegtestVerbosityLevels import functools @@ -92,7 +92,7 @@ def test_dunerc_success(run_dunerc, caplog): # checks for run control success, problems during pytest setup, etc. - basic_checks.basic_checks(run_dunerc, caplog, print_test_name=False) + utility_functions.basic_checks(run_dunerc, caplog, print_test_name=False) def test_log_files(run_dunerc): @@ -127,3 +127,7 @@ def test_data_files(run_dunerc): data_file, fragment_check_list[jdx] ) assert all_ok + + +def test_cleanup(run_dunerc): + utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=False) diff --git a/integtest/trmonrequestor_test.py b/integtest/trmonrequestor_test.py index 9550074..f7c268e 100755 --- a/integtest/trmonrequestor_test.py +++ b/integtest/trmonrequestor_test.py @@ -5,8 +5,8 @@ import integrationtest.data_file_checks as data_file_checks import integrationtest.log_file_checks as log_file_checks -import integrationtest.basic_checks as basic_checks import integrationtest.data_classes as data_classes +import integrationtest.utility_functions as utility_functions from integrationtest.verbosity_helper import IntegtestVerbosityLevels import functools @@ -105,7 +105,7 @@ def make_run_command_list(runnum): # The tests themselves def test_dunerc_success(run_dunerc, caplog): # checks for run control success, problems during pytest setup, etc. - basic_checks.basic_checks(run_dunerc, caplog, print_test_name=False) + utility_functions.basic_checks(run_dunerc, caplog, print_test_name=False) def test_log_files(run_dunerc): @@ -205,3 +205,7 @@ def test_data_files(run_dunerc): assert all_ok assert trmon_ok + + +def test_cleanup(run_dunerc): + utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=False) From 2d5638ddc681cbcc0f83c6e5a8a1293ab07e7604 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Wed, 17 Jun 2026 13:11:26 -0500 Subject: [PATCH 2/2] Updated the integtests to use the fixture that removes HDF5 files at the end of the integtest. --- integtest/disabled_output_test.py | 4 ---- integtest/hdf5_compression_test.py | 5 +---- integtest/insufficient_disk_space_test.py | 5 +---- integtest/large_trigger_record_test.py | 5 +---- integtest/max_file_size_test.py | 5 +---- integtest/multiple_data_writers_test.py | 4 ---- integtest/offline_prod_run_test.py | 4 ---- integtest/trmonrequestor_test.py | 4 ---- 8 files changed, 4 insertions(+), 32 deletions(-) diff --git a/integtest/disabled_output_test.py b/integtest/disabled_output_test.py index def5c4c..175aa3e 100644 --- a/integtest/disabled_output_test.py +++ b/integtest/disabled_output_test.py @@ -220,7 +220,3 @@ def test_data_files(run_dunerc): data_file, fragment_check_list[jdx] ) assert all_ok - - -def test_cleanup(run_dunerc): - utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=False) diff --git a/integtest/hdf5_compression_test.py b/integtest/hdf5_compression_test.py index 034c20e..984e5cb 100644 --- a/integtest/hdf5_compression_test.py +++ b/integtest/hdf5_compression_test.py @@ -128,6 +128,7 @@ conf_dict.dro_map_config.det_id = 2 # det_id = 2 for kHD_PDS conf_dict.frame_file = "asset://?checksum=a8990a9eb3a505d4ded62dfdfa9e2681" # run 36012 DAPHNE data #conf_dict.frame_file = "file:///home/nfs/biery/dunedaq/12MayFDv5.3.2DevInstrUpdate/sourcecode/dfmodules/integtest/np02vdcoldbox_run035227_sample_hd_pds.bin" +conf_dict.remove_hdf5_files = True conf_dict.config_substitutions.append( data_classes.attribute_substitution( @@ -301,7 +302,3 @@ def test_tpstream_files(run_dunerc): data_file, fragment_check_list[jdx] ) assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more TP-stream data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" - - -def test_cleanup(run_dunerc): - utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=True) diff --git a/integtest/insufficient_disk_space_test.py b/integtest/insufficient_disk_space_test.py index a02c01b..7450420 100644 --- a/integtest/insufficient_disk_space_test.py +++ b/integtest/insufficient_disk_space_test.py @@ -97,6 +97,7 @@ conf_dict.tpg_enabled = False conf_dict.n_df_apps = number_of_dataflow_apps conf_dict.fake_hsi_enabled = False +conf_dict.remove_hdf5_files = True conf_dict.config_substitutions.append( data_classes.attribute_substitution( @@ -202,7 +203,3 @@ def test_data_files(run_dunerc): data_file, fragment_check_list[jdx] ) assert all_ok - - -def test_cleanup(run_dunerc): - utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=True) diff --git a/integtest/large_trigger_record_test.py b/integtest/large_trigger_record_test.py index 38d6ff8..a6dff83 100644 --- a/integtest/large_trigger_record_test.py +++ b/integtest/large_trigger_record_test.py @@ -90,6 +90,7 @@ conf_dict.config_session_name= "largerecord" conf_dict.tpg_enabled = False conf_dict.n_df_apps = number_of_dataflow_apps +conf_dict.remove_hdf5_files = True conf_dict.config_substitutions.append( data_classes.attribute_substitution( @@ -219,7 +220,3 @@ def test_data_files(run_dunerc): ) assert all_ok - - -def test_cleanup(run_dunerc): - utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=True) diff --git a/integtest/max_file_size_test.py b/integtest/max_file_size_test.py index ce01751..c9e0d96 100644 --- a/integtest/max_file_size_test.py +++ b/integtest/max_file_size_test.py @@ -110,6 +110,7 @@ conf_dict.frame_file = ( "asset://?checksum=dd156b4895f1b06a06b6ff38e37bd798" # WIBEth All Zeros ) +conf_dict.remove_hdf5_files = True conf_dict.config_substitutions.append( data_classes.attribute_substitution( @@ -270,7 +271,3 @@ def test_tpstream_files(run_dunerc): data_file, fragment_check_list[jdx] ) assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more TP-stream data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" - - -def test_cleanup(run_dunerc): - utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=True) diff --git a/integtest/multiple_data_writers_test.py b/integtest/multiple_data_writers_test.py index 01350f5..32f4681 100644 --- a/integtest/multiple_data_writers_test.py +++ b/integtest/multiple_data_writers_test.py @@ -180,7 +180,3 @@ def test_data_files(run_dunerc): data_file, fragment_check_list[jdx] ) assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more raw data file checks failed! \N{POLICE CARS REVOLVING LIGHT}" - - -def test_cleanup(run_dunerc): - utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=False) diff --git a/integtest/offline_prod_run_test.py b/integtest/offline_prod_run_test.py index 0c331f8..4180dfb 100644 --- a/integtest/offline_prod_run_test.py +++ b/integtest/offline_prod_run_test.py @@ -127,7 +127,3 @@ def test_data_files(run_dunerc): data_file, fragment_check_list[jdx] ) assert all_ok - - -def test_cleanup(run_dunerc): - utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=False) diff --git a/integtest/trmonrequestor_test.py b/integtest/trmonrequestor_test.py index f7c268e..ee5e7eb 100755 --- a/integtest/trmonrequestor_test.py +++ b/integtest/trmonrequestor_test.py @@ -205,7 +205,3 @@ def test_data_files(run_dunerc): assert all_ok assert trmon_ok - - -def test_cleanup(run_dunerc): - utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=False)