Multi-dimensional datasets

Contents

Multi-dimensional datasets¶

This is the continuation of 1-D datasets and tables.

Interaction with `numpy`¶

Variable in a dataset are exposed in a numpy-compatible buffer format, so we can directly hand them to numpy functions:

[16]:

d['alice'].values = np.sin(d['alice'].values)

In contrast to the 1-D case considered earlier, the values are now a multi-dimensional array:

[17]:

d['alice'].values

[17]:

array([[[ 3.08490327e-01,  7.02519644e-01, -4.52374880e-01,
          1.70762909e-01],
        [ 7.49131714e-01,  9.36680954e-01, -6.65382860e-01,
          3.64167184e-01],
        [ 6.11633173e-01,  8.69946836e-01,  5.78441009e-02,
         -1.66644464e-01],
        [ 8.26355193e-01,  8.75810439e-01,  2.26318101e-01,
         -2.68547034e-01],
        [ 8.81485850e-01,  9.02852715e-01, -6.81611384e-01,
         -5.77536910e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [-9.53398021e-02,  8.99343532e-01, -2.29668932e-01,
         -7.36849538e-01],
        [ 9.16549358e-01, -2.95331029e-02,  1.66710243e-02,
          3.84957553e-01],
        [ 7.22395028e-01,  5.26357518e-01,  2.96304755e-01,
          7.36135115e-01],
        [-3.15964344e-02,  4.50732846e-01, -3.75324182e-01,
         -7.52612490e-02]],

       [[-9.91530411e-01,  5.78558379e-01, -5.91478341e-01,
         -5.08281701e-02],
        [-9.82014625e-01, -1.89092465e-01, -5.70541437e-01,
          8.49708644e-01],
        [-6.00048814e-01, -6.11322673e-01, -5.77333484e-01,
          7.22574829e-01],
        [-9.27357264e-01,  3.82168826e-01, -4.80365950e-01,
          7.88888199e-01],
        [-6.56820262e-01, -2.98444888e-01, -1.14175391e-02,
          5.69071821e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [-9.06951493e-02, -6.47693367e-01, -5.31366787e-01,
          7.68856036e-01],
        [-4.19162032e-01, -4.11194882e-01, -6.73895377e-01,
          3.98070807e-01],
        [-6.53123667e-01, -2.45431749e-01, -9.99978512e-01,
          5.58383495e-01],
        [-5.28385347e-01, -5.16193941e-01, -5.21583904e-01,
          9.63051539e-01]],

       [[-9.06190453e-01, -8.22525232e-02,  3.39899882e-01,
          2.85943408e-01],
        [-8.55814688e-01, -5.58263958e-01, -3.44647306e-01,
          6.47185384e-02],
        [-8.77022208e-01,  6.72344455e-02, -1.49061222e-01,
          2.23769876e-01],
        [-9.23658410e-01,  1.61628808e-01, -8.12749139e-02,
          3.36981570e-01],
        [-9.84269133e-01,  7.54172992e-01, -3.81341028e-01,
          5.67162986e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [-7.14314818e-01, -2.60343709e-01, -6.32039548e-01,
          5.50629294e-01],
        [-7.56509163e-01,  4.62091144e-01,  1.29720439e-01,
          5.65959048e-01],
        [-4.65908020e-01, -4.28161079e-01, -1.97721469e-02,
          2.76550505e-02],
        [-9.95032918e-01,  3.63704047e-01,  4.05360278e-01,
          5.03057184e-01]],

       [[-4.68880527e-02,  7.08914494e-01,  1.86436880e-01,
          7.22467340e-01],
        [ 6.02881842e-01,  7.28501410e-01,  9.49519219e-01,
         -4.86401030e-01],
        [-4.19816787e-02,  3.33067038e-01,  8.16869797e-02,
         -6.15027057e-02],
        [-2.41512145e-01,  3.78285578e-01,  3.91599907e-01,
         -1.36710498e-01],
        [ 4.60413450e-01,  5.55866095e-01,  5.63275420e-01,
          1.75291201e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [ 7.44930899e-02,  6.59792217e-01,  5.32495078e-01,
          2.04425910e-01],
        [ 2.80914128e-01,  7.88742836e-01,  9.26023262e-01,
          5.96421334e-02],
        [ 1.72389963e-01,  5.03348052e-01, -2.39564055e-01,
          1.46924823e-01],
        [ 7.58768461e-01,  6.65119823e-01, -2.26863112e-01,
          1.33323533e-01]],

       [[-7.19963425e-02,  9.97884205e-01,  5.67534438e-04,
         -8.88972335e-01],
        [-4.78989409e-01,  9.94448894e-01,  3.91130245e-01,
         -8.95416042e-01],
        [-8.34030779e-01,  3.78694787e-01,  9.27589105e-02,
         -1.43308016e-01],
        [-5.47896526e-01,  3.82163856e-01,  3.70078672e-02,
          1.47218437e-01],
        [-8.94835295e-01,  2.46613181e-01,  2.48329359e-01,
         -6.67987709e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [-5.25382254e-01,  8.37150471e-01,  6.18927901e-01,
          3.70979794e-02],
        [-1.76150188e-01,  5.30545574e-01,  8.50976296e-01,
         -9.07417695e-01],
        [-3.21998263e-01,  5.13486456e-01,  4.43071797e-01,
         -5.77203465e-01],
        [-3.22707614e-01,  3.39763460e-01,  3.49329852e-01,
         -9.64575229e-01]],

       [[-7.42373264e-01,  5.93060160e-01, -5.84737688e-01,
         -4.01584338e-01],
        [ 1.60036750e-02,  6.92148866e-01, -3.93327081e-01,
         -2.27534728e-01],
        [-6.15788923e-01,  7.90097187e-01,  1.96804614e-01,
          6.57630278e-01],
        [-5.56417498e-01,  3.79067561e-01, -4.94784348e-01,
         -1.81818920e-01],
        [-8.79366571e-01,  2.14637787e-01,  7.05140493e-02,
          4.43341127e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [-6.49129309e-01,  1.69340685e-01,  7.81024592e-04,
          6.16028658e-01],
        [ 1.25808967e-01,  1.94138170e-01, -1.25988402e-01,
          1.85447914e-01],
        [ 3.29068260e-01,  2.09407320e-01, -6.49694893e-01,
         -3.27042337e-01],
        [-1.14991160e-01, -9.97517170e-02,  9.85614734e-02,
          3.72073928e-01]],

       [[ 4.48214756e-01,  4.38584374e-01, -7.88344643e-01,
          9.44011395e-01],
        [ 1.63764586e-01, -2.47625162e-01, -1.88204935e-01,
          8.14997439e-02],
        [-1.91289220e-01, -1.92612963e-01,  4.02854655e-02,
          4.92035995e-01],
        [ 1.72061602e-01, -2.44519266e-02,  6.08588811e-01,
          9.18478386e-01],
        [ 6.84009185e-03, -1.34917858e-01, -1.55377931e-01,
          8.70477211e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [ 7.17241071e-03, -7.32273383e-01, -1.50537765e-01,
          7.10460833e-01],
        [ 8.11413666e-01,  2.50624029e-01, -1.30140752e-02,
          8.58369110e-01],
        [ 9.82606661e-01, -1.79636765e-01,  3.69760750e-01,
         -2.14259171e-01],
        [ 6.87124247e-01, -3.88261986e-01,  5.93504734e-02,
          3.76272393e-01]],

       [[-4.15478614e-02, -8.20944149e-01, -2.19263038e-01,
         -5.04310997e-02],
        [ 4.63633703e-01, -4.15801945e-01, -5.55472085e-01,
         -5.69924994e-01],
        [-4.26796073e-01,  4.63768578e-01,  3.73872731e-01,
         -5.55247654e-01],
        [-3.87729975e-01, -5.23964734e-01,  5.45651601e-01,
         -5.53981038e-01],
        [-5.10749829e-01,  6.41321392e-01, -7.57991113e-01,
          7.35010795e-02],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [-7.73684103e-02,  2.55356205e-01,  1.71041421e-01,
         -7.30862466e-01],
        [-3.13479306e-01, -8.52237153e-02, -3.33599842e-01,
         -6.16707887e-01],
        [ 2.48409511e-01, -8.07043174e-01, -4.33374046e-01,
         -5.84228675e-01],
        [-6.10782212e-01,  4.38621463e-01, -1.47608022e-01,
         -8.46892740e-01]]])

Exercise 3¶

Use sc.mean to compute the mean of the data for Alice along the Z dimension.
Do the same with numpy, what are the complications you encounter, that are not present when using the dataset?

Solution 3¶

[18]:

help(sc.mean)

Help on function mean in module scipp.core.reduction:

mean(x: 'VariableLike', dim: 'Optional[str]' = None, *, out: 'Optional[VariableLike]' = None) -> 'VariableLike'
    Element-wise mean over the specified dimension.

    If the input has variances, the variances stored in the output are based on
    the "standard deviation of the mean", i.e.,
    :math:`\sigma_{mean} = \sigma / \sqrt{N}`.
    :math:`N` is the length of the input dimension.
    :math:`\sigma` is estimated as the average of the standard deviations of
    the input elements along that dimension.

    See :py:func:`scipp.sum` on how rounding errors for float32 inputs are handled.

    :param x: Input data.
    :param dim: Dimension along which to calculate the mean. If not
                given, the mean over all dimensions is calculated.
    :param out: Optional output buffer.
    :raises: If the dimension does not exist, or the dtype cannot be summed,
             e.g., if it is a string.
    :return: The mean of the input values.
    :seealso: :py:func:`scipp.nanmean`.

[19]:

mean = sc.mean(d['alice'], 'z')

When using numpy to compute the mean: - We must remember (or lookup) which dimension corresponds to the Z dimensions. - We need a separate call for values and variances. - We need to manually scale the variance with the inverse of the number of data points to get the variance of the mean (standard deviation of the mean scales with 1/sqrt(N)).

[20]:

np_values = np.mean(d['alice'].values, axis=0)
np_variances = np.mean(d['alice'].variances, axis=0)
np_variances /= d['alice'].shape[0]

[21]:

print(mean.values)
print(mean.variances)
print(np_values)
print(np_variances)

[[-0.25547766  0.38954057 -0.26366179  0.09142114]
 [-0.0401754   0.24262457 -0.17211578 -0.10239784]
 [-0.37191557  0.26235915  0.01460726  0.14616352]
 [-0.32326938  0.25133855  0.09409263  0.13131364]
 [-0.32216271  0.36026268 -0.13820252  0.1816651 ]
 [ 0.          0.          0.          0.        ]
 [-0.25882053  0.14758408 -0.02754595  0.17747334]
 [ 0.05867318  0.21252376  0.09711167  0.11604012]
 [ 0.12672993  0.01154082 -0.15415579 -0.0292044 ]
 [-0.14470037  0.15671675 -0.04484714  0.05763117]]
[[0.04952562 0.04686    0.04886557 0.04937315]
 [0.04814335 0.04827578 0.04796968 0.04753866]
 [0.04839807 0.04659424 0.04849753 0.05204315]
 [0.05153999 0.05129722 0.04937667 0.04658262]
 [0.04752207 0.0476515  0.0459198  0.04983319]
 [0.04954154 0.04607951 0.05040432 0.04745453]
 [0.05065706 0.04596775 0.04846066 0.05139884]
 [0.04755006 0.04805108 0.04761291 0.04959621]
 [0.05034251 0.04847695 0.04749352 0.04757468]
 [0.04912227 0.049137   0.051303   0.05050632]]
[[-0.25547766  0.38954057 -0.26366179  0.09142114]
 [-0.0401754   0.24262457 -0.17211578 -0.10239784]
 [-0.37191557  0.26235915  0.01460726  0.14616352]
 [-0.32326938  0.25133855  0.09409263  0.13131364]
 [-0.32216271  0.36026268 -0.13820252  0.1816651 ]
 [ 0.          0.          0.          0.        ]
 [-0.25882053  0.14758408 -0.02754595  0.17747334]
 [ 0.05867318  0.21252376  0.09711167  0.11604012]
 [ 0.12672993  0.01154082 -0.15415579 -0.0292044 ]
 [-0.14470037  0.15671675 -0.04484714  0.05763117]]
[[0.04952562 0.04686    0.04886557 0.04937315]
 [0.04814335 0.04827578 0.04796968 0.04753866]
 [0.04839807 0.04659424 0.04849753 0.05204315]
 [0.05153999 0.05129722 0.04937667 0.04658262]
 [0.04752207 0.0476515  0.0459198  0.04983319]
 [0.04954154 0.04607951 0.05040432 0.04745453]
 [0.05065706 0.04596775 0.04846066 0.05139884]
 [0.04755006 0.04805108 0.04761291 0.04959621]
 [0.05034251 0.04847695 0.04749352 0.04757468]
 [0.04912227 0.049137   0.051303   0.05050632]]

[ ]:

previous

1-D datasets and tables

next

About scipp