TBL data reduction#

[1]:

import plopp as pp
from ess.reduce import time_of_flight
from ess import tbl
import ess.tbl.data  # noqa: F401
from ess.imaging.types import *

Create and configure the workflow#

[2]:

wf = tbl.TblWorkflow()

wf[Filename[SampleRun]] = tbl.data.tutorial_sample_data()
wf[TimeOfFlightLookupTableFilename] = tbl.data.tbl_tof_lookup_table_no_choppers()

Downloading file 'tbl_sample_data_2025-03.hdf' from 'https://public.esss.dk/groups/scipp/ess/tbl/2/tbl_sample_data_2025-03.hdf' to '/home/runner/.cache/ess/tbl'.
Downloading file 'TBL-tof-lookup-table-no-choppers.h5' from 'https://public.esss.dk/groups/scipp/ess/tbl/2/TBL-tof-lookup-table-no-choppers.h5' to '/home/runner/.cache/ess/tbl'.

nGEM detector data#

[3]:

wf[NeXusDetectorName] = "ngem_detector"

wf.visualize(RawDetector[SampleRun], graph_attr={"rankdir": "LR"})

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/backend/execute.py:76, in run_check(cmd, input_lines, encoding, quiet, **kwargs)
     75         kwargs['stdout'] = kwargs['stderr'] = subprocess.PIPE
---> 76     proc = _run_input_lines(cmd, input_lines, kwargs=kwargs)
     77 else:

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/backend/execute.py:96, in _run_input_lines(cmd, input_lines, kwargs)
     95 def _run_input_lines(cmd, input_lines, *, kwargs):
---> 96     popen = subprocess.Popen(cmd, stdin=subprocess.PIPE, **kwargs)
     98     stdin_write = popen.stdin.write

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/subprocess.py:1026, in Popen.__init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, user, group, extra_groups, encoding, errors, text, umask, pipesize, process_group)
   1023             self.stderr = io.TextIOWrapper(self.stderr,
   1024                     encoding=encoding, errors=errors)
-> 1026     self._execute_child(args, executable, preexec_fn, close_fds,
   1027                         pass_fds, cwd, env,
   1028                         startupinfo, creationflags, shell,
   1029                         p2cread, p2cwrite,
   1030                         c2pread, c2pwrite,
   1031                         errread, errwrite,
   1032                         restore_signals,
   1033                         gid, gids, uid, umask,
   1034                         start_new_session, process_group)
   1035 except:
   1036     # Cleanup if the child failed starting.

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/subprocess.py:1955, in Popen._execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, gid, gids, uid, umask, start_new_session, process_group)
   1954 if err_filename is not None:
-> 1955     raise child_exception_type(errno_num, err_msg, err_filename)
   1956 else:

FileNotFoundError: [Errno 2] No such file or directory: PosixPath('dot')

The above exception was the direct cause of the following exception:

ExecutableNotFound                        Traceback (most recent call last)
File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/IPython/core/formatters.py:1036, in MimeBundleFormatter.__call__(self, obj, include, exclude)
   1033     method = get_real_method(obj, self.print_method)
   1035     if method is not None:
-> 1036         return method(include=include, exclude=exclude)
   1037     return None
   1038 else:

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/jupyter_integration.py:98, in JupyterIntegration._repr_mimebundle_(self, include, exclude, **_)
     96 include = set(include) if include is not None else {self._jupyter_mimetype}
     97 include -= set(exclude or [])
---> 98 return {mimetype: getattr(self, method_name)()
     99         for mimetype, method_name in MIME_TYPES.items()
    100         if mimetype in include}

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/jupyter_integration.py:98, in <dictcomp>(.0)
     96 include = set(include) if include is not None else {self._jupyter_mimetype}
     97 include -= set(exclude or [])
---> 98 return {mimetype: getattr(self, method_name)()
     99         for mimetype, method_name in MIME_TYPES.items()
    100         if mimetype in include}

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/jupyter_integration.py:112, in JupyterIntegration._repr_image_svg_xml(self)
    110 def _repr_image_svg_xml(self) -> str:
    111     """Return the rendered graph as SVG string."""
--> 112     return self.pipe(format='svg', encoding=SVG_ENCODING)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/piping.py:104, in Pipe.pipe(self, format, renderer, formatter, neato_no_op, quiet, engine, encoding)
     55 def pipe(self,
     56          format: typing.Optional[str] = None,
     57          renderer: typing.Optional[str] = None,
   (...)     61          engine: typing.Optional[str] = None,
     62          encoding: typing.Optional[str] = None) -> typing.Union[bytes, str]:
     63     """Return the source piped through the Graphviz layout command.
     64
     65     Args:
   (...)    102         '<?xml version='
    103     """
--> 104     return self._pipe_legacy(format,
    105                              renderer=renderer,
    106                              formatter=formatter,
    107                              neato_no_op=neato_no_op,
    108                              quiet=quiet,
    109                              engine=engine,
    110                              encoding=encoding)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/_tools.py:185, in deprecate_positional_args.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
    177     wanted = ', '.join(f'{name}={value!r}'
    178                        for name, value in deprecated.items())
    179     warnings.warn(f'The signature of {func_name} will be reduced'
    180                   f' to {supported_number} positional arg{s_}{qualification}'
    181                   f' {list(supported)}: pass {wanted} as keyword arg{s_}',
    182                   stacklevel=stacklevel,
    183                   category=category)
--> 185 return func(*args, **kwargs)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/piping.py:121, in Pipe._pipe_legacy(self, format, renderer, formatter, neato_no_op, quiet, engine, encoding)
    112 @_tools.deprecate_positional_args(supported_number=1, ignore_arg='self')
    113 def _pipe_legacy(self,
    114                  format: typing.Optional[str] = None,
   (...)    119                  engine: typing.Optional[str] = None,
    120                  encoding: typing.Optional[str] = None) -> typing.Union[bytes, str]:
--> 121     return self._pipe_future(format,
    122                              renderer=renderer,
    123                              formatter=formatter,
    124                              neato_no_op=neato_no_op,
    125                              quiet=quiet,
    126                              engine=engine,
    127                              encoding=encoding)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/piping.py:149, in Pipe._pipe_future(self, format, renderer, formatter, neato_no_op, quiet, engine, encoding)
    146 if encoding is not None:
    147     if codecs.lookup(encoding) is codecs.lookup(self.encoding):
    148         # common case: both stdin and stdout need the same encoding
--> 149         return self._pipe_lines_string(*args, encoding=encoding, **kwargs)
    150     try:
    151         raw = self._pipe_lines(*args, input_encoding=self.encoding, **kwargs)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/backend/piping.py:212, in pipe_lines_string(engine, format, input_lines, encoding, renderer, formatter, neato_no_op, quiet)
    206 cmd = dot_command.command(engine, format,
    207                           renderer=renderer,
    208                           formatter=formatter,
    209                           neato_no_op=neato_no_op)
    210 kwargs = {'input_lines': input_lines, 'encoding': encoding}
--> 212 proc = execute.run_check(cmd, capture_output=True, quiet=quiet, **kwargs)
    213 return proc.stdout

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/backend/execute.py:81, in run_check(cmd, input_lines, encoding, quiet, **kwargs)
     79 except OSError as e:
     80     if e.errno == errno.ENOENT:
---> 81         raise ExecutableNotFound(cmd) from e
     82     raise
     84 if not quiet and proc.stderr:

ExecutableNotFound: failed to execute PosixPath('dot'), make sure the Graphviz executables are on your systems' PATH

[3]:

<graphviz.graphs.Digraph at 0x7fedb80e8b50>

[4]:

ngem = wf.compute(RawDetector[SampleRun])
ngem

[5]:

ngem['dim_0', 0].hist().plot(aspect='equal', title='First detection') + ngem['dim_0', 1].hist().plot(aspect='equal', title='Last detection')

[5]:

../_images/tbl_tbl-data-reduction_7_0.svg

Time-of-flight#

[6]:

wf.visualize(TofDetector[SampleRun], graph_attr={"rankdir": "LR"})

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/backend/execute.py:76, in run_check(cmd, input_lines, encoding, quiet, **kwargs)
     75         kwargs['stdout'] = kwargs['stderr'] = subprocess.PIPE
---> 76     proc = _run_input_lines(cmd, input_lines, kwargs=kwargs)
     77 else:

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/backend/execute.py:96, in _run_input_lines(cmd, input_lines, kwargs)
     95 def _run_input_lines(cmd, input_lines, *, kwargs):
---> 96     popen = subprocess.Popen(cmd, stdin=subprocess.PIPE, **kwargs)
     98     stdin_write = popen.stdin.write

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/subprocess.py:1026, in Popen.__init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, user, group, extra_groups, encoding, errors, text, umask, pipesize, process_group)
   1023             self.stderr = io.TextIOWrapper(self.stderr,
   1024                     encoding=encoding, errors=errors)
-> 1026     self._execute_child(args, executable, preexec_fn, close_fds,
   1027                         pass_fds, cwd, env,
   1028                         startupinfo, creationflags, shell,
   1029                         p2cread, p2cwrite,
   1030                         c2pread, c2pwrite,
   1031                         errread, errwrite,
   1032                         restore_signals,
   1033                         gid, gids, uid, umask,
   1034                         start_new_session, process_group)
   1035 except:
   1036     # Cleanup if the child failed starting.

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/subprocess.py:1955, in Popen._execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, gid, gids, uid, umask, start_new_session, process_group)
   1954 if err_filename is not None:
-> 1955     raise child_exception_type(errno_num, err_msg, err_filename)
   1956 else:

FileNotFoundError: [Errno 2] No such file or directory: PosixPath('dot')

The above exception was the direct cause of the following exception:

ExecutableNotFound                        Traceback (most recent call last)
File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/IPython/core/formatters.py:1036, in MimeBundleFormatter.__call__(self, obj, include, exclude)
   1033     method = get_real_method(obj, self.print_method)
   1035     if method is not None:
-> 1036         return method(include=include, exclude=exclude)
   1037     return None
   1038 else:

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/jupyter_integration.py:98, in JupyterIntegration._repr_mimebundle_(self, include, exclude, **_)
     96 include = set(include) if include is not None else {self._jupyter_mimetype}
     97 include -= set(exclude or [])
---> 98 return {mimetype: getattr(self, method_name)()
     99         for mimetype, method_name in MIME_TYPES.items()
    100         if mimetype in include}

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/jupyter_integration.py:98, in <dictcomp>(.0)
     96 include = set(include) if include is not None else {self._jupyter_mimetype}
     97 include -= set(exclude or [])
---> 98 return {mimetype: getattr(self, method_name)()
     99         for mimetype, method_name in MIME_TYPES.items()
    100         if mimetype in include}

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/jupyter_integration.py:112, in JupyterIntegration._repr_image_svg_xml(self)
    110 def _repr_image_svg_xml(self) -> str:
    111     """Return the rendered graph as SVG string."""
--> 112     return self.pipe(format='svg', encoding=SVG_ENCODING)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/piping.py:104, in Pipe.pipe(self, format, renderer, formatter, neato_no_op, quiet, engine, encoding)
     55 def pipe(self,
     56          format: typing.Optional[str] = None,
     57          renderer: typing.Optional[str] = None,
   (...)     61          engine: typing.Optional[str] = None,
     62          encoding: typing.Optional[str] = None) -> typing.Union[bytes, str]:
     63     """Return the source piped through the Graphviz layout command.
     64
     65     Args:
   (...)    102         '<?xml version='
    103     """
--> 104     return self._pipe_legacy(format,
    105                              renderer=renderer,
    106                              formatter=formatter,
    107                              neato_no_op=neato_no_op,
    108                              quiet=quiet,
    109                              engine=engine,
    110                              encoding=encoding)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/_tools.py:185, in deprecate_positional_args.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
    177     wanted = ', '.join(f'{name}={value!r}'
    178                        for name, value in deprecated.items())
    179     warnings.warn(f'The signature of {func_name} will be reduced'
    180                   f' to {supported_number} positional arg{s_}{qualification}'
    181                   f' {list(supported)}: pass {wanted} as keyword arg{s_}',
    182                   stacklevel=stacklevel,
    183                   category=category)
--> 185 return func(*args, **kwargs)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/piping.py:121, in Pipe._pipe_legacy(self, format, renderer, formatter, neato_no_op, quiet, engine, encoding)
    112 @_tools.deprecate_positional_args(supported_number=1, ignore_arg='self')
    113 def _pipe_legacy(self,
    114                  format: typing.Optional[str] = None,
   (...)    119                  engine: typing.Optional[str] = None,
    120                  encoding: typing.Optional[str] = None) -> typing.Union[bytes, str]:
--> 121     return self._pipe_future(format,
    122                              renderer=renderer,
    123                              formatter=formatter,
    124                              neato_no_op=neato_no_op,
    125                              quiet=quiet,
    126                              engine=engine,
    127                              encoding=encoding)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/piping.py:149, in Pipe._pipe_future(self, format, renderer, formatter, neato_no_op, quiet, engine, encoding)
    146 if encoding is not None:
    147     if codecs.lookup(encoding) is codecs.lookup(self.encoding):
    148         # common case: both stdin and stdout need the same encoding
--> 149         return self._pipe_lines_string(*args, encoding=encoding, **kwargs)
    150     try:
    151         raw = self._pipe_lines(*args, input_encoding=self.encoding, **kwargs)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/backend/piping.py:212, in pipe_lines_string(engine, format, input_lines, encoding, renderer, formatter, neato_no_op, quiet)
    206 cmd = dot_command.command(engine, format,
    207                           renderer=renderer,
    208                           formatter=formatter,
    209                           neato_no_op=neato_no_op)
    210 kwargs = {'input_lines': input_lines, 'encoding': encoding}
--> 212 proc = execute.run_check(cmd, capture_output=True, quiet=quiet, **kwargs)
    213 return proc.stdout

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/graphviz/backend/execute.py:81, in run_check(cmd, input_lines, encoding, quiet, **kwargs)
     79 except OSError as e:
     80     if e.errno == errno.ENOENT:
---> 81         raise ExecutableNotFound(cmd) from e
     82     raise
     84 if not quiet and proc.stderr:

ExecutableNotFound: failed to execute PosixPath('dot'), make sure the Graphviz executables are on your systems' PATH

[6]:

<graphviz.graphs.Digraph at 0x7feda47c23d0>

Visualize the time-of-flight lookup table#

[7]:

table = wf.compute(time_of_flight.TimeOfFlightLookupTable)
table.plot()

[7]:

../_images/tbl_tbl-data-reduction_11_0.svg

Results#

[8]:

ngem_tofs = wf.compute(TofDetector[SampleRun])
ngem_wavs = wf.compute(WavelengthDetector[SampleRun])

ngem_tofs.bins.concat().hist(tof=100).plot() + ngem_wavs.bins.concat().hist(wavelength=100).plot()

---------------------------------------------------------------------------
UnsatisfiedRequirement                    Traceback (most recent call last)
Cell In[8], line 1
----> 1 ngem_tofs = wf.compute(TofDetector[SampleRun])
      2 ngem_wavs = wf.compute(WavelengthDetector[SampleRun])
      4 ngem_tofs.bins.concat().hist(tof=100).plot() + ngem_wavs.bins.concat().hist(wavelength=100).plot()

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/sciline/pipeline.py:191, in Pipeline.compute(self, tp, reporter, **kwargs)
    170 def compute(
    171     self,
    172     tp: type | Iterable[type] | "UnionType",  # noqa: UP037 (needed by Sphinx)
    173     reporter: Reporter | None = None,
    174     **kwargs: Any,
    175 ) -> Any:
    176     """
    177     Compute result for the given keys.
    178
   (...)    189         Keyword arguments passed to the ``.get()`` method.
    190     """
--> 191     return self.get(tp, **kwargs).compute(reporter=reporter)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/sciline/pipeline.py:281, in Pipeline.get(self, keys, scheduler, handler, max_depth)
    278         info = f'{e} Requested node not in graph. Did you mean one of: {nodes}?'
    279     # Not raising `from e` because that includes noisy traceback of internals,
    280     # which are not relevant to the user.
--> 281     raise type(e)(f'{info}\n\n') from None
    282 return TaskGraph(
    283     graph=graph,
    284     targets=targets if multi else keys,  # type: ignore[arg-type]
    285     scheduler=scheduler,
    286 )

UnsatisfiedRequirement: Missing input node 'LookupTableRelativeErrorThreshold'. Affects requested targets (via providers given in parentheses):
1. LookupTableRelativeErrorThreshold → (ess.reduce.time_of_flight.eto_to_tof.mask_large_uncertainty_in_lut_detector) → ErrorLimitedTofLookupTable[NXdetector] → (ess.reduce.time_of_flight.eto_to_tof.detector_time_of_flight_data) → TofDetector[SampleRun]

He3 detector data#

[9]:

he3_wf = wf.copy()

he3 = {}
for bank in ('he3_detector_bank0', 'he3_detector_bank1'):
    he3_wf[NeXusDetectorName] = bank
    he3[bank] = he3_wf.compute(RawDetector[SampleRun])

[10]:

lims = {'vmin': 0, 'vmax': 4500}

he3['he3_detector_bank0'].hist().plot(title='he3_bank0', **lims) + he3['he3_detector_bank1'].hist().plot(title='he3_bank1', **lims)

[10]:

../_images/tbl_tbl-data-reduction_16_0.svg

Time-of-flight#

[11]:

he3_tofs = {}
he3_wavs = {}

for bank in ('he3_detector_bank0', 'he3_detector_bank1'):
    he3_wf[NeXusDetectorName] = bank
    he3_tofs[bank] = he3_wf.compute(TofDetector[SampleRun]).bins.concat().hist(tof=100)
    he3_wavs[bank] = he3_wf.compute(WavelengthDetector[SampleRun]).bins.concat().hist(wavelength=100)

pp.plot(he3_tofs) + pp.plot(he3_wavs)

---------------------------------------------------------------------------
UnsatisfiedRequirement                    Traceback (most recent call last)
Cell In[11], line 6
      4 for bank in ('he3_detector_bank0', 'he3_detector_bank1'):
      5     he3_wf[NeXusDetectorName] = bank
----> 6     he3_tofs[bank] = he3_wf.compute(TofDetector[SampleRun]).bins.concat().hist(tof=100)
      7     he3_wavs[bank] = he3_wf.compute(WavelengthDetector[SampleRun]).bins.concat().hist(wavelength=100)
      9 pp.plot(he3_tofs) + pp.plot(he3_wavs)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/sciline/pipeline.py:191, in Pipeline.compute(self, tp, reporter, **kwargs)
    170 def compute(
    171     self,
    172     tp: type | Iterable[type] | "UnionType",  # noqa: UP037 (needed by Sphinx)
    173     reporter: Reporter | None = None,
    174     **kwargs: Any,
    175 ) -> Any:
    176     """
    177     Compute result for the given keys.
    178
   (...)    189         Keyword arguments passed to the ``.get()`` method.
    190     """
--> 191     return self.get(tp, **kwargs).compute(reporter=reporter)

File ~/work/ess/ess/.pixi/envs/docs-essimaging/lib/python3.11/site-packages/sciline/pipeline.py:281, in Pipeline.get(self, keys, scheduler, handler, max_depth)
    278         info = f'{e} Requested node not in graph. Did you mean one of: {nodes}?'
    279     # Not raising `from e` because that includes noisy traceback of internals,
    280     # which are not relevant to the user.
--> 281     raise type(e)(f'{info}\n\n') from None
    282 return TaskGraph(
    283     graph=graph,
    284     targets=targets if multi else keys,  # type: ignore[arg-type]
    285     scheduler=scheduler,
    286 )

UnsatisfiedRequirement: Missing input node 'LookupTableRelativeErrorThreshold'. Affects requested targets (via providers given in parentheses):
1. LookupTableRelativeErrorThreshold → (ess.reduce.time_of_flight.eto_to_tof.mask_large_uncertainty_in_lut_detector) → ErrorLimitedTofLookupTable[NXdetector] → (ess.reduce.time_of_flight.eto_to_tof.detector_time_of_flight_data) → TofDetector[SampleRun]