diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 2dac4ea..8abbabe 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -29,10 +29,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - - name: Unset header - # checkout@v2 adds a header that makes branch protection report errors - # because the Github action bot is not a collaborator on the repo - run: git config --local --unset http.https://github.com/.extraheader - name: Fetch tags run: git fetch --prune --unshallow - name: Disable etelemetry @@ -87,10 +83,7 @@ jobs: with: submodules: recursive fetch-depth: 0 - - name: Unset header - # checkout@v2 adds a header that makes branch protection report errors - # because the Github action bot is not a collaborator on the repo - run: git config --local --unset http.https://github.com/.extraheader + - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/extras/fileformats/extras/biosig/__init__.py b/extras/fileformats/extras/biosig/__init__.py index be5db19..f71f260 100644 --- a/extras/fileformats/extras/biosig/__init__.py +++ b/extras/fileformats/extras/biosig/__init__.py @@ -1,5 +1,4 @@ from ._version import __version__ -from . import eeg -from . import meg +from . import edf -__all__ = ["__version__", "eeg", "meg"] +__all__ = ["__version__", "edf"] diff --git a/extras/fileformats/extras/biosig/eeg.py b/extras/fileformats/extras/biosig/edf.py similarity index 54% rename from extras/fileformats/extras/biosig/eeg.py rename to extras/fileformats/extras/biosig/edf.py index 457a1d6..e322b21 100644 --- a/extras/fileformats/extras/biosig/eeg.py +++ b/extras/fileformats/extras/biosig/edf.py @@ -1,5 +1,4 @@ import os -import configparser import typing as ty import tempfile from pathlib import Path @@ -8,7 +7,7 @@ import mne.export from fileformats.core import extra_implementation, FileSet -from fileformats.biosig import Biosig, BrainVision, Edf, EdfPlus +from fileformats.biosig import Biosig, Edf, EdfPlus from .utils import mne_deidentify @@ -31,17 +30,6 @@ def edf_plus_read_metadata(edf: EdfPlus, **kwargs: ty.Any) -> ty.Mapping[str, ty } -@extra_implementation(FileSet.read_metadata) -def brain_vision_read_metadata( - bv: BrainVision, **kwargs: ty.Any -) -> ty.Mapping[str, ty.Any]: - raw = mne.io.read_raw_brainvision(bv.header_file, preload=False, verbose=False) - return { - **raw.info.to_json_dict(), - **_parse_vhdr(bv.header_file), - } - - @extra_implementation(Biosig.deidentify) def edf_deidentify( edf: Edf, @@ -58,22 +46,6 @@ def edf_deidentify( return type(edf)(deid_fspath), reid -@extra_implementation(Biosig.deidentify) -def brain_vision_deidentify( - bv: BrainVision, - spec: ty.Any = None, - out_dir: os.PathLike[str] | None = None, -) -> tuple[BrainVision, dict[str, ty.Any]]: - out_dir = Path(tempfile.mkdtemp() if out_dir is None else out_dir) - out_dir.mkdir(parents=True, exist_ok=True) - raw = mne.io.read_raw_brainvision(bv.header_file, preload=True, verbose=False) - deidentified_info, reid = mne_deidentify(raw, spec) - raw.info = deidentified_info - deid_vhdr = out_dir / "eeg.vhdr" - mne.export.export_raw(deid_vhdr, raw, fmt="brainvision", overwrite=True) - return BrainVision(out_dir / "eeg.eeg"), reid - - def _parse_edf_header(path: os.PathLike[str]) -> dict[str, ty.Any]: """ Parse EDF/EDF+ header bytes directly for patient and recording fields @@ -112,42 +84,3 @@ def _parse_edf_header(path: os.PathLike[str]) -> dict[str, ty.Any]: "edf_start_time": start_time, "edf_subtype": reserved if reserved.startswith("EDF+") else None, } - - -def _parse_vhdr(path: os.PathLike[str]) -> dict[str, ty.Any]: - """ - Parse a BrainVision .vhdr file (INI format) for fields not exposed by MNE. - Extracts acquisition settings and the free-text Comment section. - """ - parser = configparser.RawConfigParser() - # vhdr files start with a magic line before the first INI section — skip it - with open(path, encoding="utf-8-sig", errors="replace") as f: - lines = f.readlines() - ini_lines = [line for line in lines if not line.startswith("Brain Vision")] - parser.read_string("".join(ini_lines)) - - def get( - section: ty.Any, - key: str, - fallback: ty.Any = None, - ) -> ty.Any: - try: - return parser.get(section, key) - except (configparser.NoSectionError, configparser.NoOptionError): - return fallback - - # [Common Infos] uses inconsistent casing across BrainVision versions - common = next((s for s in parser.sections() if s.lower() == "common infos"), None) - - comment_lines = [] - if parser.has_section("Comment"): - comment_lines = [v for _, v in parser.items("Comment") if v.strip()] - - return { - "bv_data_format": get(common, "DataFormat") if common else None, - "bv_data_orientation": get(common, "DataOrientation") if common else None, - "bv_n_channels": get(common, "NumberOfChannels") if common else None, - "bv_sampling_interval_us": get(common, "SamplingInterval") if common else None, - "bv_binary_format": get("Binary Infos", "BinaryFormat"), - "bv_comment": "\n".join(comment_lines) if comment_lines else None, - } diff --git a/extras/fileformats/extras/biosig/meg.py b/extras/fileformats/extras/biosig/meg.py deleted file mode 100644 index 9f22346..0000000 --- a/extras/fileformats/extras/biosig/meg.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import typing as ty -from pathlib import Path -import tempfile - -import mne.io - -from fileformats.core import extra_implementation, FileSet -from fileformats.biosig import Ctf, Kit, Fif -from fileformats.biosig.base import Biosig - -from .utils import mne_deidentify - - -@extra_implementation(FileSet.read_metadata) -def ctf_read_metadata(ctf: Ctf, **kwargs: ty.Any) -> ty.Mapping[str, ty.Any]: - return mne.io.read_raw_ctf(ctf, preload=False, verbose=False).info.to_json_dict() # type: ignore[no-any-return] - - -@extra_implementation(FileSet.read_metadata) -def kit_read_metadata(kit: Kit, **kwargs: ty.Any) -> ty.Mapping[str, ty.Any]: - return mne.io.read_raw_kit(kit, mrk=kit.mark_file, verbose=False).info.to_json_dict() # type: ignore[no-any-return] - - -@extra_implementation(FileSet.read_metadata) -def fif_read_metadata(fif: Fif, **kwargs: ty.Any) -> ty.Mapping[str, ty.Any]: - return mne.io.read_raw_fif(fif, preload=False, verbose=False).info.to_json_dict() # type: ignore[no-any-return] - - -@extra_implementation(Biosig.deidentify) -def fif_deidentify( - fif: Fif, - spec: ty.Any = None, - out_dir: os.PathLike[str] | None = None, -) -> tuple[Fif, dict[str, ty.Any]]: - out_dir = Path(tempfile.mkdtemp() if out_dir is None else out_dir) - out_dir.mkdir(parents=True, exist_ok=True) - raw = mne.io.read_raw_fif(fif, preload=True, verbose=False) - deidentified_info, reid = mne_deidentify(raw, spec) - raw.info = deidentified_info - deid_fspath = out_dir / "meg-signals.fif" - raw.save(deid_fspath, overwrite=True) - return Fif(deid_fspath), reid diff --git a/extras/fileformats/extras/biosig/tests/test_eeg_extras.py b/extras/fileformats/extras/biosig/tests/test_edf_extras.py similarity index 56% rename from extras/fileformats/extras/biosig/tests/test_eeg_extras.py rename to extras/fileformats/extras/biosig/tests/test_edf_extras.py index 1d9716d..920e348 100644 --- a/extras/fileformats/extras/biosig/tests/test_eeg_extras.py +++ b/extras/fileformats/extras/biosig/tests/test_edf_extras.py @@ -10,10 +10,7 @@ - miaocao@swin.edu.au """ -from fileformats.biosig import ( - BrainVision, - EdfPlus, -) +from fileformats.biosig import EdfPlus # ------------------------------ # EEG: EDF @@ -25,15 +22,3 @@ def test_edf_plus_read_metadata(edf_plus_path): assert isinstance(metadata, dict) assert metadata["sfreq"] is not None assert "edf_patient_code" in metadata - - -# ------------------------------ -# EEG: BrainVision -# ------------------------------ - - -def test_brainvision_read_metadata(bv_vhdr_path): - metadata = BrainVision(bv_vhdr_path.with_suffix(".eeg")).metadata - assert isinstance(metadata, dict) - assert metadata["sfreq"] is not None - assert "bv_n_channels" in metadata diff --git a/extras/fileformats/extras/biosig/tests/test_meg_extras.py b/extras/fileformats/extras/biosig/tests/test_meg_extras.py deleted file mode 100644 index c3a2f22..0000000 --- a/extras/fileformats/extras/biosig/tests/test_meg_extras.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Pytest tests for EEG/MEG file format validation and metadata reading. - -Test data is downloaded via MNE's dataset utilities and cached for the session. - -Authors: -- Miao Cao - -Email: -- miaocao@swin.edu.au -""" - -from fileformats.biosig import Ctf, Kit, Fif - -# ------------------------------ -# MEG: CTF -# ------------------------------ - - -def test_ctf_read_metadata(ctf_ds_path): - metadata = Ctf(ctf_ds_path).metadata - assert isinstance(metadata, dict) - assert metadata["sfreq"] is not None - - -# ------------------------------ -# MEG: FIF -# ------------------------------ - - -def test_fif_read_metadata(fif_path): - metadata = Fif(fif_path).metadata - assert isinstance(metadata, dict) - assert metadata["sfreq"] is not None - - -def test_fif_deidentify(fif_path, tmp_path): - fif = Fif(fif_path) - orig_metadata = fif.metadata - - deid_fif, reid = fif.deidentify(out_dir=tmp_path) - - assert isinstance(deid_fif, Fif) - assert isinstance(reid, dict) - assert reid, "expected at least one field to be stripped or changed" - - deid_metadata = deid_fif.metadata - - # Every field recorded in reid should differ between original and deidentified - for key in reid: - assert orig_metadata.get(key) != deid_metadata.get( - key - ), f"reid claims '{key}' changed but original and deidentified values match" - - # Subject identifying fields should be absent or cleared in the deidentified file - deid_subject_info = deid_metadata.get("subject_info") or {} - for pii_field in ("last_name", "first_name", "birthday"): - assert not deid_subject_info.get( - pii_field - ), f"subject_info.{pii_field} was not cleared by deidentification" - - -# ------------------------------ -# MEG: KIT -# ------------------------------ - - -def test_kit_read_metadata(kit_sqd_path): - metadata = Kit(kit_sqd_path).metadata - assert isinstance(metadata, dict) - assert metadata["sfreq"] is not None diff --git a/fileformats/biosig/__init__.py b/fileformats/biosig/__init__.py index cb2baa8..b111148 100644 --- a/fileformats/biosig/__init__.py +++ b/fileformats/biosig/__init__.py @@ -10,26 +10,10 @@ - miaocao@swin.edu.au """ -from .base import Biosig -from .eeg import ( - Eeg, +from .base import Biosig, Eeg, Meg +from .edf import ( Edf, EdfPlus, - BrainVisionHeader, - BrainVisionMarker, - BrainVision, -) -from .meg import ( - Meg, - CtfMeg4, - CtfRes4, - CtfInfo, - Ctf, - Fif, - KitMark, - KitHeadPosition, - KitSensorInfo, - Kit, ) from ._version import __version__ @@ -38,20 +22,7 @@ "__version__", "Biosig", "Eeg", - "Fif", - "FifGz", "Edf", "EdfPlus", - "BrainVisionHeader", - "BrainVisionMarker", - "BrainVision", "Meg", - "CtfMeg4", - "CtfRes4", - "CtfInfo", - "Ctf", - "KitMark", - "KitHeadPosition", - "KitSensorInfo", - "Kit", ] diff --git a/fileformats/biosig/base.py b/fileformats/biosig/base.py index 2250e9b..15e89c2 100644 --- a/fileformats/biosig/base.py +++ b/fileformats/biosig/base.py @@ -35,3 +35,19 @@ def deidentify( were stripped/modified during the deidentification process. """ raise NotImplementedError + + +class Eeg(Biosig): + """Base class for all Electroencephalography recordings""" + + pass + + +# ------------------------------ +# Base MEG Type (Abstract Class) +# ------------------------------ +class Meg(Biosig): + """ + Base class for MEG data formats + All specific MEG formats inherit from this class with unified validation logic + """ diff --git a/fileformats/biosig/eeg.py b/fileformats/biosig/edf.py similarity index 58% rename from fileformats/biosig/eeg.py rename to fileformats/biosig/edf.py index e9f092c..f039f24 100644 --- a/fileformats/biosig/eeg.py +++ b/fileformats/biosig/edf.py @@ -11,19 +11,12 @@ from fileformats.core import validated_property, mtime_cached_property from fileformats.core.exceptions import FormatMismatchError -from fileformats.generic import BinaryFile, UnicodeFile -from fileformats.core.mixin import WithMagicNumber, WithAdjacentFiles - +from fileformats.generic import BinaryFile +from fileformats.core.mixin import WithMagicNumber from .base import Biosig -class Eeg(Biosig): - """Base class for all Electroencephalography recordings""" - - pass - - -class Edf(WithMagicNumber, BinaryFile, Eeg): +class Edf(WithMagicNumber, BinaryFile, Biosig): """ EDF format EEG (European Data Format) — binary file with fixed-width ASCII header followed by binary signal data. @@ -93,51 +86,3 @@ def edf_type(self) -> str: f"for EDF+, found: {self._edf_type!r}" ) return self._edf_type - - -class WithBrainVisionMagic: - """Checks the 'Brain Vision' magic string in text mode, handling optional BOM via utf-8-sig.""" - - @validated_property - def brain_vision_magic(self) -> None: - first_line = self.fspath.read_text(encoding="utf-8-sig").split("\n")[0] # type: ignore[attr-defined] - if not first_line.startswith("Brain Vision"): - raise FormatMismatchError( - f"File does not start with 'Brain Vision' magic string: {self}" - ) - - -class BrainVisionHeader(WithBrainVisionMagic, UnicodeFile, Biosig): - """ - BrainVision header file (.vhdr) — plain-text INI file describing channel - configuration, sampling rate, amplifier settings, and references to the - data and marker files. - """ - - ext = ".vhdr" - - -class BrainVisionMarker(WithBrainVisionMagic, UnicodeFile, Biosig): - """ - BrainVision marker file (.vmrk) — plain-text INI file containing event - markers and annotations time-stamped to samples in the data file. - """ - - ext = ".vmrk" - - -class BrainVision(WithAdjacentFiles, BinaryFile, Biosig): - """ - BrainVision binary data file (.eeg) — raw multiplexed sample data, - format described by the accompanying .vhdr header file. No magic number. - """ - - ext = ".eeg" - - @validated_property - def header_file(self) -> BrainVisionHeader: - return BrainVisionHeader(self.select_by_ext(BrainVisionHeader)) - - @validated_property - def marker_file(self) -> BrainVisionMarker: - return BrainVisionMarker(self.select_by_ext(BrainVisionMarker)) diff --git a/fileformats/biosig/meg.py b/fileformats/biosig/meg.py deleted file mode 100644 index 0184bb7..0000000 --- a/fileformats/biosig/meg.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -MEG file formats for XNAT Ingest workflows. -Defines and validates common MEG file formats (FIF, CTF, KIT) for use in XNAT Ingest pipelines. - -Authors: -- Miao Cao - -Email: -- miaocao@swin.edu.au -""" - -import struct -from fileformats.core import validated_property -from fileformats.core.mixin import WithAdjacentFiles, WithMagicNumber -from fileformats.core.exceptions import FormatMismatchError -from fileformats.generic import Directory, File, BinaryFile, UnicodeFile -from fileformats.application import Xml - -from .base import Biosig - - -# ------------------------------ -# Base MEG Type (Abstract Class) -# ------------------------------ -class Meg(Biosig): - """ - Base class for MEG data formats - All specific MEG formats inherit from this class with unified validation logic - """ - - -# ------------------------------ -# Implementation of Specific EEG Formats -# ------------------------------ -class Fif(BinaryFile, Meg): - """ - MNE FIF format (standard format for NeuroMag/MEGIN MEG/EEG devices) - Most commonly used binary format, supports compression (.fif.gz) - """ - - ext = ".fif" - - @validated_property - def fiff_header(self) -> None: - # FIFF files begin with a tag stream; the first tag must be FIFF_FILE_ID (kind=100) - # with data type FIFFT_ID_STRUCT (dtype=31), encoded as big-endian uint32 pairs. - data = self.read_contents(8) - if len(data) < 8: - raise FormatMismatchError(f"File too short to be a valid FIFF file: {self}") - kind, dtype = struct.unpack(">II", data) - if kind != 100 or dtype != 31: - raise FormatMismatchError( - f"First FIFF tag has kind={kind}, dtype={dtype}; expected kind=100 " - f"(FIFF_FILE_ID) and dtype=31 (FIFFT_ID_STRUCT) in {self}" - ) - - -class CtfMeg4(WithMagicNumber, BinaryFile, Meg): - """ - CTF MEG4 binary data file (.meg4) — raw sensor data in CTF's proprietary format. - The resource file (.res4) in the same .ds directory describes the channel layout. - """ - - ext = ".meg4" - # First 8 bytes: "MEG41CP\0" (CTF MEG4 format version identifier) - magic_number = b"MEG41CP\x00" - - -class CtfRes4(WithMagicNumber, BinaryFile, Meg): - """ - CTF resource file (.res4) — binary header describing channel layout, sampling - rate, sensor positions, and filter settings for the accompanying .meg4 data. - """ - - ext = ".res4" - # First 8 bytes: "MEG41RS\0" (CTF resource file version identifier) - magic_number = b"MEG42RS\x00" - - -class CtfInfo(Xml, Meg): - """ - CTF dataset info file (.infods) — XML file containing dataset-level metadata - such as subject info, acquisition date, and operator notes. - """ - - ext = ".infods" - - -class Ctf(Directory, Meg): - """ - CTF format MEG (directory-based, proprietary format for CTF MEG devices) - Core files include *.meg4/*.res4/*.infods under .ds directory - """ - - ext = ".ds" - - content_types = (CtfMeg4, CtfRes4, CtfInfo) - - -class KitMark(BinaryFile): - """Marker""" - - ext = ".mrk" - - -class KitHeadPosition(UnicodeFile): - ext = ".elp" - - -class KitSensorInfo(File): - ext = ".hsj" - - -class Kit(WithAdjacentFiles, Meg, BinaryFile): - """ - KIT/RIKEN (Ricon) MEG format (directory-based) - Required files: - - Main data file (.sqd or .con) - Optional files: .mrk (marker/coregistration), .elp (head position), .hsj (sensor info) - """ - - ext = ".sqd" - alternate_exts = (".con",) - - marker_generic_names = ("marker.mrk", "markers.mrk", "kit.mrk") - - @property - def mark_file(self) -> KitMark | None: - try: - mrk_path = self.select_by_ext(KitMark) - return KitMark(mrk_path) - except FormatMismatchError: - pass - for cand in self.marker_generic_names: - mrk_path = self.parent / cand - if mrk_path.exists(): - return KitMark(mrk_path) - return None - - @property - def head_position_file(self) -> KitHeadPosition | None: - try: - return KitHeadPosition(self.select_by_ext(KitHeadPosition)) - except FormatMismatchError as e: - if e.args[0].startswith("No matching files"): - return None - raise - - @property - def sensor_info_file(self) -> KitSensorInfo | None: - try: - return KitSensorInfo(self.select_by_ext(KitSensorInfo)) - except FormatMismatchError as e: - if e.args[0].startswith("No matching files"): - return None - raise diff --git a/fileformats/biosig/tests/test_edf.py b/fileformats/biosig/tests/test_edf.py new file mode 100644 index 0000000..60e1dff --- /dev/null +++ b/fileformats/biosig/tests/test_edf.py @@ -0,0 +1,23 @@ +""" +Pytest tests for EEG/MEG file format validation and metadata reading. + +Test data is downloaded via MNE's dataset utilities and cached for the session. + +Authors: +- Miao Cao + +Email: +- miaocao@swin.edu.au +""" + +from fileformats.biosig import ( + EdfPlus, +) + +# ------------------------------ +# EEG: EDF +# ------------------------------ + + +def test_edf_plus_instantiate(edf_plus_path): + EdfPlus(edf_plus_path) diff --git a/fileformats/biosig/tests/test_eeg.py b/fileformats/biosig/tests/test_eeg.py deleted file mode 100644 index 5c98f23..0000000 --- a/fileformats/biosig/tests/test_eeg.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Pytest tests for EEG/MEG file format validation and metadata reading. - -Test data is downloaded via MNE's dataset utilities and cached for the session. - -Authors: -- Miao Cao - -Email: -- miaocao@swin.edu.au -""" - -from fileformats.biosig import ( - BrainVision, - BrainVisionHeader, - BrainVisionMarker, - EdfPlus, -) - -# ------------------------------ -# EEG: FIF -# ------------------------------ - -# ------------------------------ -# EEG: EDF -# ------------------------------ - - -def test_edf_plus_instantiate(edf_plus_path): - EdfPlus(edf_plus_path) - - -# ------------------------------ -# EEG: BrainVision -# ------------------------------ - - -def test_brainvision_header_instantiate(bv_vhdr_path): - BrainVisionHeader(bv_vhdr_path) - - -def test_brainvision_marker_instantiate(bv_vhdr_path): - BrainVisionMarker(bv_vhdr_path.with_suffix(".vmrk")) - - -def test_brainvision_data_instantiate(bv_vhdr_path): - BrainVision(bv_vhdr_path.with_suffix(".eeg")) diff --git a/fileformats/biosig/tests/test_meg.py b/fileformats/biosig/tests/test_meg.py deleted file mode 100644 index 465ad66..0000000 --- a/fileformats/biosig/tests/test_meg.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -Pytest tests for EEG/MEG file format validation and metadata reading. - -Test data is downloaded via MNE's dataset utilities and cached for the session. - -Authors: -- Miao Cao - -Email: -- miaocao@swin.edu.au -""" - -from fileformats.biosig import Ctf, CtfInfo, CtfMeg4, CtfRes4, Kit, Fif - -# ------------------------------ -# MEG: CTF -# ------------------------------ - - -def test_ctf_instantiate(ctf_ds_path): - Ctf(ctf_ds_path) - - -def test_fif_instantiate(fif_path): - Fif(fif_path) - - -def test_ctf_meg4_instantiate(ctf_ds_path): - meg4_files = list(ctf_ds_path.glob("*.meg4")) - assert meg4_files, "No .meg4 file found in CTF .ds directory" - CtfMeg4(meg4_files[0]) - - -def test_ctf_res4_instantiate(ctf_ds_path): - res4_files = list(ctf_ds_path.glob("*.res4")) - assert res4_files, "No .res4 file found in CTF .ds directory" - CtfRes4(res4_files[0]) - - -def test_ctf_infods_instantiate(ctf_ds_path): - infods_files = list(ctf_ds_path.glob("*.infods")) - assert infods_files, "No .infods file found in CTF .ds directory" - CtfInfo(infods_files[0]) - - -# ------------------------------ -# MEG: KIT -# ------------------------------ - - -def test_kit_instantiate(kit_sqd_path): - Kit(kit_sqd_path)