DMSC Integration Testing

self = <nbconvert.preprocessors.execute.ExecutePreprocessor object at 0x7f20861b0aa0>
msg_id = 'e5dcb01d-64ff41356c44ff95915e9870_1333069_10'
cell = {'cell_type': 'code', 'execution_count': None, 'id': '4', 'metadata': {'execution': {}}, 'outputs': [], 'source': 'results'}
timeout = 120
task_poll_output_msg = <Task pending name='Task-438' coro=<NotebookClient._async_poll_output_msg() running at /home/gitlab-runner/builds/sr_s.../3/dmsc-nightly/dmsc-nightly/.tox/codeshelf-loki/lib/python3.12/site-packages/zmq/_future.py:416, Task.task_wakeup()]>>
task_poll_kernel_alive = <Task cancelled name='Task-437' coro=<NotebookClient._async_poll_kernel_alive() done, defined at /home/gitlab-runner/builds/sr_srNBdt/3/dmsc-nightly/dmsc-nightly/.tox/codeshelf-loki/lib/python3.12/site-packages/nbclient/client.py:821>>

    async def _async_poll_for_reply(
        self,
        msg_id: str,
        cell: NotebookNode,
        timeout: int | None,
        task_poll_output_msg: asyncio.Future[t.Any],
        task_poll_kernel_alive: asyncio.Future[t.Any],
    ) -> dict[str, t.Any]:
        msg: dict[str, t.Any]
        assert self.kc is not None
        new_timeout: float | None = None
        if timeout is not None:
            deadline = monotonic() + timeout
            new_timeout = float(timeout)
        error_on_timeout_execute_reply = None
        while True:
            try:
                if error_on_timeout_execute_reply:
                    msg = error_on_timeout_execute_reply
                    msg["parent_header"] = {"msg_id": msg_id}
                else:
>                   msg = await ensure_async(self.kc.shell_channel.get_msg(timeout=new_timeout))
                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.tox/codeshelf-loki/lib/python3.12/site-packages/nbclient/client.py:782: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.tox/codeshelf-loki/lib/python3.12/site-packages/jupyter_core/utils/__init__.py:214: in ensure_async
    result = await obj
             ^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <jupyter_client.channels.AsyncZMQSocketChannel object at 0x7f2086283ef0>
timeout = 120.0

    async def get_msg(  # type:ignore[override]
        self, timeout: float | None = None
    ) -> t.Dict[str, t.Any]:
        """Gets a message if there is one that is ready."""
        assert self.socket is not None
        timeout_ms = None if timeout is None else int(timeout * 1000)  # seconds to ms
        ready = await self.socket.poll(timeout_ms)
        if ready:
            res = await self._recv()
            return res
        else:
>           raise Empty
E           _queue.Empty

.tox/codeshelf-loki/lib/python3.12/site-packages/jupyter_client/channels.py:330: Empty

During handling of the above exception, another exception occurred:

notebook = PosixPath('/tmp/pytest-of-gitlab-runner/pytest-450/codeshelf/src/loki/notebooks/workflow-widget-loki.ipynb')
run_codeshelf_notebook = <function _run_codeshelf_notebook at 0x7f20868e80e0>
tmp_path = PosixPath('/tmp/pytest-of-gitlab-runner/pytest-450/test_run_notebooks_workflow_wi0')

    def test_run_notebooks(
        notebook: Path, run_codeshelf_notebook: Callable, tmp_path: Path
    ):
>       run_codeshelf_notebook(notebook_path=notebook, run_path=tmp_path)

tests/codeshelf/codeshelf_test.py:10: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/codeshelf/conftest.py:131: in _run_codeshelf_notebook
    ep.preprocess(nb, {"metadata": {"path": run_path}})
.tox/codeshelf-loki/lib/python3.12/site-packages/nbconvert/preprocessors/execute.py:103: in preprocess
    self.preprocess_cell(cell, resources, index)
.tox/codeshelf-loki/lib/python3.12/site-packages/nbconvert/preprocessors/execute.py:124: in preprocess_cell
    cell = self.execute_cell(cell, index, store_history=True)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.tox/codeshelf-loki/lib/python3.12/site-packages/jupyter_core/utils/__init__.py:165: in wrapped
    return loop.run_until_complete(inner)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
/opt/miniforge/lib/python3.12/asyncio/base_events.py:687: in run_until_complete
    return future.result()
           ^^^^^^^^^^^^^^^
.tox/codeshelf-loki/lib/python3.12/site-packages/nbclient/client.py:1005: in async_execute_cell
    exec_reply = await self.task_poll_for_reply
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.tox/codeshelf-loki/lib/python3.12/site-packages/nbclient/client.py:806: in _async_poll_for_reply
    error_on_timeout_execute_reply = await self._async_handle_timeout(timeout, cell)
                                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <nbconvert.preprocessors.execute.ExecutePreprocessor object at 0x7f20861b0aa0>
timeout = 120
cell = {'cell_type': 'code', 'execution_count': None, 'id': '4', 'metadata': {'execution': {}}, 'outputs': [], 'source': 'results'}

    async def _async_handle_timeout(
        self, timeout: int, cell: NotebookNode | None = None
    ) -> None | dict[str, t.Any]:
        self.log.error("Timeout waiting for execute reply (%is)." % timeout)
        if self.interrupt_on_timeout:
            self.log.error("Interrupting kernel")
            assert self.km is not None
            await ensure_async(self.km.interrupt_kernel())
            if self.error_on_timeout:
                execute_reply = {"content": {**self.error_on_timeout, "status": "error"}}
                return execute_reply
            return None
        else:
            assert cell is not None
>           raise CellTimeoutError.error_from_timeout_and_cell(
                "Cell execution timed out", timeout, cell
            )
E           nbclient.exceptions.CellTimeoutError: A cell timed out while it was being executed, after 120 seconds.
E           The message was: Cell execution timed out.
E           Here is a preview of the cell contents:
E           -------------------
E           results
E           -------------------

.tox/codeshelf-loki/lib/python3.12/site-packages/nbclient/client.py:856: CellTimeoutError

self = <nbconvert.preprocessors.execute.ExecutePreprocessor object at 0x7f7bf3871bb0>
msg_id = 'b6a22c1f-d8a2dbca615ce7f98e1b852a_887965_9'
cell = {'cell_type': 'code', 'execution_count': None, 'id': '4', 'metadata': {'execution': {}}, 'outputs': [], 'source': 'results'}
timeout = 600
task_poll_output_msg = <Task pending name='Task-313' coro=<NotebookClient._async_poll_output_msg() running at /home/gitlab-runner/builds/sr_s.../4/dmsc-nightly/dmsc-nightly/.tox/codeshelf-loki/lib/python3.12/site-packages/zmq/_future.py:416, Task.task_wakeup()]>>
task_poll_kernel_alive = <Task cancelled name='Task-312' coro=<NotebookClient._async_poll_kernel_alive() done, defined at /home/gitlab-runner/builds/sr_srNBdt/4/dmsc-nightly/dmsc-nightly/.tox/codeshelf-loki/lib/python3.12/site-packages/nbclient/client.py:821>>

    async def _async_poll_for_reply(
        self,
        msg_id: str,
        cell: NotebookNode,
        timeout: int | None,
        task_poll_output_msg: asyncio.Future[t.Any],
        task_poll_kernel_alive: asyncio.Future[t.Any],
    ) -> dict[str, t.Any]:
        msg: dict[str, t.Any]
        assert self.kc is not None
        new_timeout: float | None = None
        if timeout is not None:
            deadline = monotonic() + timeout
            new_timeout = float(timeout)
        error_on_timeout_execute_reply = None
        while True:
            try:
                if error_on_timeout_execute_reply:
                    msg = error_on_timeout_execute_reply
                    msg["parent_header"] = {"msg_id": msg_id}
                else:
>                   msg = await ensure_async(self.kc.shell_channel.get_msg(timeout=new_timeout))
                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.tox/codeshelf-loki/lib/python3.12/site-packages/nbclient/client.py:782: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.tox/codeshelf-loki/lib/python3.12/site-packages/jupyter_core/utils/__init__.py:214: in ensure_async
    result = await obj
             ^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <jupyter_client.channels.AsyncZMQSocketChannel object at 0x7f7bf3870ec0>
timeout = 600.0

    async def get_msg(  # type:ignore[override]
        self, timeout: float | None = None
    ) -> t.Dict[str, t.Any]:
        """Gets a message if there is one that is ready."""
        assert self.socket is not None
        timeout_ms = None if timeout is None else int(timeout * 1000)  # seconds to ms
        ready = await self.socket.poll(timeout_ms)
        if ready:
            res = await self._recv()
            return res
        else:
>           raise Empty
E           _queue.Empty

.tox/codeshelf-loki/lib/python3.12/site-packages/jupyter_client/channels.py:330: Empty

During handling of the above exception, another exception occurred:

notebook = PosixPath('/tmp/pytest-of-gitlab-runner/pytest-369/codeshelf/src/loki/notebooks/workflow-widget-loki.ipynb')
run_codeshelf_notebook = <function _run_codeshelf_notebook at 0x7f7bf3ea00e0>
tmp_path = PosixPath('/tmp/pytest-of-gitlab-runner/pytest-369/test_run_notebooks_workflow_wi0')

    def test_run_notebooks(
        notebook: Path, run_codeshelf_notebook: Callable, tmp_path: Path
    ):
>       run_codeshelf_notebook(notebook_path=notebook, run_path=tmp_path)

tests/codeshelf/codeshelf_test.py:10: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/codeshelf/conftest.py:104: in _run_codeshelf_notebook
    ep.preprocess(nb, {"metadata": {"path": run_path}})
.tox/codeshelf-loki/lib/python3.12/site-packages/nbconvert/preprocessors/execute.py:103: in preprocess
    self.preprocess_cell(cell, resources, index)
.tox/codeshelf-loki/lib/python3.12/site-packages/nbconvert/preprocessors/execute.py:124: in preprocess_cell
    cell = self.execute_cell(cell, index, store_history=True)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.tox/codeshelf-loki/lib/python3.12/site-packages/jupyter_core/utils/__init__.py:165: in wrapped
    return loop.run_until_complete(inner)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
/opt/miniforge/lib/python3.12/asyncio/base_events.py:687: in run_until_complete
    return future.result()
           ^^^^^^^^^^^^^^^
.tox/codeshelf-loki/lib/python3.12/site-packages/nbclient/client.py:1005: in async_execute_cell
    exec_reply = await self.task_poll_for_reply
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.tox/codeshelf-loki/lib/python3.12/site-packages/nbclient/client.py:806: in _async_poll_for_reply
    error_on_timeout_execute_reply = await self._async_handle_timeout(timeout, cell)
                                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <nbconvert.preprocessors.execute.ExecutePreprocessor object at 0x7f7bf3871bb0>
timeout = 600
cell = {'cell_type': 'code', 'execution_count': None, 'id': '4', 'metadata': {'execution': {}}, 'outputs': [], 'source': 'results'}

    async def _async_handle_timeout(
        self, timeout: int, cell: NotebookNode | None = None
    ) -> None | dict[str, t.Any]:
        self.log.error("Timeout waiting for execute reply (%is)." % timeout)
        if self.interrupt_on_timeout:
            self.log.error("Interrupting kernel")
            assert self.km is not None
            await ensure_async(self.km.interrupt_kernel())
            if self.error_on_timeout:
                execute_reply = {"content": {**self.error_on_timeout, "status": "error"}}
                return execute_reply
            return None
        else:
            assert cell is not None
>           raise CellTimeoutError.error_from_timeout_and_cell(
                "Cell execution timed out", timeout, cell
            )
E           nbclient.exceptions.CellTimeoutError: A cell timed out while it was being executed, after 600 seconds.
E           The message was: Cell execution timed out.
E           Here is a preview of the cell contents:
E           -------------------
E           results
E           -------------------

.tox/codeshelf-loki/lib/python3.12/site-packages/nbclient/client.py:856: CellTimeoutError