None
Test: ingestor|loki|file_found_by_scicat_is_consistent_with_manual|
None
None
None
instrument = 'loki', coda_proposal_id = '999999'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
> raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
tests/ingestor/ingestor_test.py:28:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
instrument = 'LoKI', proposal_id = '999999', n = 2, inspect = 40
def get_latest_nexus_paths(
instrument: str, proposal_id: str | None, n: int = 1, inspect: int = 20
) -> list[Path]:
"""
Get the latest file paths manually by sorting the files in the directory.
Parameters
----------
instrument:
The instrument name.
proposal_id:
SciCat proposal ID for CODA.
If ``None``, the environment variable ``OVERRIDE_RAW_DATA_DIR`` must be set.
n:
The number of files to return.
inspect:
The number of files to inspect before returning.
Returns
-------
:
Paths to the n latest files for the given instrument and proposal.
"""
instrument = NEXUS_INSTRUMENT_NAME[instrument]
path = _coda_raw_data_path(proposal_id)
files_found = []
logging.info(
"Scanning the %d latest files in %s for instrument %s",
inspect,
path,
instrument,
)
recent_files = sorted(path.glob("*.hdf"))[-inspect::][::-1]
logging.info("Recent files: %s", recent_files)
for file in recent_files:
logging.info("Inspecting file: %s", file)
try:
with snx.File(file, "r", locking=False) as f:
instrument_in_file = f["/entry/instrument/name"][()]
if instrument_in_file == instrument:
logging.info(
"File %s matches requested instrument %s. Adding to list. "
"Last modified: %s",
file,
instrument,
datetime.fromtimestamp(
os.stat(file).st_mtime, tz=local_timezone()
),
)
files_found.append(file)
if len(files_found) == n:
logging.info("Found %d files. Returning.", n)
return files_found
else:
logging.info(
"File instrument is %s, but %s was requested. Skipping.",
instrument_in_file,
instrument,
)
except OSError as e:
logging.warning(
"File %s could not be opened. Reason: %s",
file,
e,
)
continue
logging.error("No files found for instrument %s", instrument)
> raise RuntimeError(f"No files found for instrument {instrument}")
E RuntimeError: No files found for instrument LoKI
.tox/ingestor/lib/python3.12/site-packages/dmsc_nightly/nexusfiles/coda.py:94: RuntimeError
None
None
None
None
None
None
None
None
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
> raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
tests/ingestor/ingestor_test.py:28:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
instrument = 'LoKI', proposal_id = '977695', n = 2, inspect = 40
def get_latest_nexus_paths(
instrument: str, proposal_id: str | None, n: int = 1, inspect: int = 20
) -> list[Path]:
"""
Get the latest file paths manually by sorting the files in the directory.
Parameters
----------
instrument:
The instrument name.
proposal_id:
SciCat proposal ID for CODA.
If ``None``, the environment variable ``OVERRIDE_RAW_DATA_DIR`` must be set.
n:
The number of files to return.
inspect:
The number of files to inspect before returning.
Returns
-------
:
Paths to the n latest files for the given instrument and proposal.
"""
instrument = NEXUS_INSTRUMENT_NAME[instrument]
path = _coda_raw_data_path(proposal_id)
files_found = []
logging.info(
"Scanning the %d latest files in %s for instrument %s",
inspect,
path,
instrument,
)
recent_files = sorted(path.glob("*.hdf"))[-inspect::][::-1]
logging.info("Recent files: %s", recent_files)
for file in recent_files:
logging.info("Inspecting file: %s", file)
try:
with snx.File(file, "r", locking=False) as f:
instrument_in_file = f["/entry/instrument/name"][()]
if instrument_in_file == instrument:
logging.info(
"File %s matches requested instrument %s. Adding to list. "
"Last modified: %s",
file,
instrument,
datetime.fromtimestamp(
os.stat(file).st_mtime, tz=local_timezone()
),
)
files_found.append(file)
if len(files_found) == n:
logging.info("Found %d files. Returning.", n)
return files_found
else:
logging.info(
"File instrument is %s, but %s was requested. Skipping.",
instrument_in_file,
instrument,
)
except OSError as e:
logging.warning(
"File %s could not be opened. Reason: %s",
file,
e,
)
continue
logging.error("No files found for instrument %s", instrument)
> raise RuntimeError(f"No files found for instrument {instrument}")
E RuntimeError: No files found for instrument LoKI
.tox/ingestor/lib/python3.12/site-packages/dmsc_nightly/nexusfiles/coda.py:94: RuntimeError
None
None
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00082281.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00082293.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00082287.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
None
None
None
None
None
None
None
None
None
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080295.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080871.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080865.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080295.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080739.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080733.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080295.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080601.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080595.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080295.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080451.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080445.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080295.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080313.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00080307.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
> raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
tests/ingestor/ingestor_test.py:28:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
instrument = 'LoKI', proposal_id = '977695', n = 2, inspect = 40
def get_latest_nexus_paths(
instrument: str, proposal_id: str | None, n: int = 1, inspect: int = 20
) -> list[Path]:
"""
Get the latest file paths manually by sorting the files in the directory.
Parameters
----------
instrument:
The instrument name.
proposal_id:
SciCat proposal ID for CODA.
If ``None``, the environment variable ``OVERRIDE_RAW_DATA_DIR`` must be set.
n:
The number of files to return.
inspect:
The number of files to inspect before returning.
Returns
-------
:
Paths to the n latest files for the given instrument and proposal.
"""
instrument = NEXUS_INSTRUMENT_NAME[instrument]
path = _coda_raw_data_path(proposal_id)
files_found = []
logging.info(
"Scanning the %d latest files in %s for instrument %s",
inspect,
path,
instrument,
)
recent_files = sorted(path.glob("*.hdf"))[-inspect::][::-1]
logging.info("Recent files: %s", recent_files)
for file in recent_files:
logging.info("Inspecting file: %s", file)
try:
with snx.File(file, "r", locking=False) as f:
instrument_in_file = f["/entry/instrument/name"][()]
if instrument_in_file == instrument:
logging.info(
"File %s matches requested instrument %s. Adding to list. "
"Last modified: %s",
file,
instrument,
datetime.fromtimestamp(
os.stat(file).st_mtime, tz=local_timezone()
),
)
files_found.append(file)
if len(files_found) == n:
logging.info("Found %d files. Returning.", n)
return files_found
else:
logging.info(
"File instrument is %s, but %s was requested. Skipping.",
instrument_in_file,
instrument,
)
except OSError as e:
logging.warning(
"File %s could not be opened. Reason: %s",
file,
e,
)
continue
logging.error("No files found for instrument %s", instrument)
> raise RuntimeError(f"No files found for instrument {instrument}")
E RuntimeError: No files found for instrument LoKI
.tox/ingestor/lib/python3.12/site-packages/dmsc_nightly/nexusfiles/coda.py:94: RuntimeError
None
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079239.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079883.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079876.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079239.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079743.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079736.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079239.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079610.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079603.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079239.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079470.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079463.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079239.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079330.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079323.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078805.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079204.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079190.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078805.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079064.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00079057.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078805.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078917.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078910.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078455.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078770.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078763.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078455.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078630.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078609.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
None
None
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078147.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078210.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00078196.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
# Resolve symlinks because the raw folder is symlinked to `/ess/data`
# and we don't care whether we access files through links or regular paths.
scicat_path = scicat_path.resolve()
raw_paths = [path.resolve() for path in raw_paths]
> assert scicat_path in raw_paths
E AssertionError: assert PosixPath('/ess/raw/coda/2025/977695/raw/977695_00077262.hdf') in [PosixPath('/ess/raw/coda/2025/977695/raw/977695_00077892.hdf'), PosixPath('/ess/raw/coda/2025/977695/raw/977695_00077886.hdf')]
tests/ingestor/ingestor_test.py:37: AssertionError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
> raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
tests/ingestor/ingestor_test.py:28:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
instrument = 'LoKI', proposal_id = '977695', n = 2, inspect = 40
def get_latest_nexus_paths(
instrument: str, proposal_id: str | None, n: int = 1, inspect: int = 20
) -> list[Path]:
"""
Get the latest file paths manually by sorting the files in the directory.
Parameters
----------
instrument:
The instrument name.
proposal_id:
SciCat proposal ID for CODA.
If ``None``, the environment variable ``OVERRIDE_RAW_DATA_DIR`` must be set.
n:
The number of files to return.
inspect:
The number of files to inspect before returning.
Returns
-------
:
Paths to the n latest files for the given instrument and proposal.
"""
instrument = NEXUS_INSTRUMENT_NAME[instrument]
path = _coda_raw_data_path(proposal_id)
files_found = []
logging.info(
"Scanning the %d latest files in %s for instrument %s",
inspect,
path,
instrument,
)
recent_files = sorted(path.glob("*.hdf"))[-inspect::][::-1]
logging.info("Recent files: %s", recent_files)
for file in recent_files:
logging.info("Inspecting file: %s", file)
try:
with snx.File(file, "r", locking=False) as f:
instrument_in_file = f["/entry/instrument/name"][()]
if instrument_in_file == instrument:
logging.info(
"File %s matches requested instrument %s. Adding to list. "
"Last modified: %s",
file,
instrument,
datetime.fromtimestamp(
os.stat(file).st_mtime, tz=local_timezone()
),
)
files_found.append(file)
if len(files_found) == n:
logging.info("Found %d files. Returning.", n)
return files_found
else:
logging.info(
"File instrument is %s, but %s was requested. Skipping.",
instrument_in_file,
instrument,
)
except OSError as e:
logging.warning(
"File %s could not be opened. Reason: %s",
file,
e,
)
continue
logging.error("No files found for instrument %s", instrument)
> raise RuntimeError(f"No files found for instrument {instrument}")
E RuntimeError: No files found for instrument LoKI
.tox/ingestor/lib/python3.12/site-packages/dmsc_nightly/nexusfiles/coda.py:94: RuntimeError
instrument = 'loki', coda_proposal_id = '977695'
def test_file_found_by_scicat_is_consistent_with_manual(
instrument: str,
coda_proposal_id: str,
) -> None:
scicat_path = scicat.get_latest_coda_nexus_path(
proposal_id=coda_proposal_id, instrument=instrument
)
> raw_paths = coda.get_latest_nexus_paths(
proposal_id=coda_proposal_id, instrument=instrument, n=2, inspect=40
)
tests/ingestor/ingestor_test.py:28:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
instrument = 'LoKI', proposal_id = '977695', n = 2, inspect = 40
def get_latest_nexus_paths(
instrument: str, proposal_id: str | None, n: int = 1, inspect: int = 20
) -> list[Path]:
"""
Get the latest file paths manually by sorting the files in the directory.
Parameters
----------
instrument:
The instrument name.
proposal_id:
SciCat proposal ID for CODA.
If ``None``, the environment variable ``OVERRIDE_RAW_DATA_DIR`` must be set.
n:
The number of files to return.
inspect:
The number of files to inspect before returning.
Returns
-------
:
Paths to the n latest files for the given instrument and proposal.
"""
instrument = NEXUS_INSTRUMENT_NAME[instrument]
path = _coda_raw_data_path(proposal_id)
files_found = []
logging.info(
"Scanning the %d latest files in %s for instrument %s",
inspect,
path,
instrument,
)
recent_files = sorted(path.glob("*.hdf"))[-inspect::][::-1]
logging.info("Recent files: %s", recent_files)
for file in recent_files:
logging.info("Inspecting file: %s", file)
try:
with snx.File(file, "r", locking=False) as f:
instrument_in_file = f["/entry/instrument/name"][()]
if instrument_in_file == instrument:
logging.info(
"File %s matches requested instrument %s. Adding to list. "
"Last modified: %s",
file,
instrument,
datetime.fromtimestamp(
os.stat(file).st_mtime, tz=local_timezone()
),
)
files_found.append(file)
if len(files_found) == n:
logging.info("Found %d files. Returning.", n)
return files_found
else:
logging.info(
"File instrument is %s, but %s was requested. Skipping.",
instrument_in_file,
instrument,
)
except OSError as e:
logging.warning(
"File %s could not be opened. Reason: %s",
file,
e,
)
continue
logging.error("No files found for instrument %s", instrument)
> raise RuntimeError(f"No files found for instrument {instrument}")
E RuntimeError: No files found for instrument LoKI
.tox/ingestor/lib/python3.12/site-packages/dmsc_nightly/nexusfiles/coda.py:94: RuntimeError