Multi-dimensional datasets¶

This is the continuation of 1-D datasets and tables.

Interaction with `numpy`¶

Variable in a dataset are exposed in a numpy-compatible buffer format, so we can directly hand them to numpy functions:

[16]:

d['alice'].values = np.sin(d['alice'].values)

In contrast to the 1-D case considered earlier, the values are now a multi-dimensional array:

[17]:

d['alice'].values

[17]:

array([[[-4.10748400e-01,  5.67818436e-01,  9.33123202e-01,
          7.97447909e-01],
        [-7.56822750e-01,  2.04653860e-01,  6.64802903e-01,
         -2.50455769e-01],
        [ 3.59708472e-01, -9.26153131e-01,  9.98009065e-01,
          6.78331248e-02],
        [-4.86200083e-01, -1.34605248e-02,  7.84141968e-01,
         -1.46765811e-01],
        [ 1.57151069e-01,  1.48637987e-01,  9.65001501e-01,
          5.16718160e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [-7.49256294e-01,  4.78946889e-02,  6.81911143e-01,
          6.18767073e-01],
        [ 1.03277466e-01,  1.39589665e-01,  6.70065873e-01,
         -1.34927074e-01],
        [-7.47665420e-01,  2.44379277e-01,  5.84147284e-01,
          4.85776936e-01],
        [-1.08605728e-01,  3.67895642e-01,  9.95634341e-01,
          2.30915687e-02]],

       [[ 5.29617292e-01,  3.76334896e-01,  1.82067827e-01,
         -1.55069093e-01],
        [ 2.48424660e-01,  4.69156789e-01, -3.89413048e-01,
         -4.77183159e-01],
        [ 2.06701515e-01,  4.80900807e-02,  5.62771731e-01,
         -4.07313962e-01],
        [ 1.28903966e-01,  8.63334929e-01,  2.68074690e-01,
          2.54911250e-02],
        [-3.77768057e-01, -1.34764349e-01,  6.61528852e-02,
          3.61056280e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [ 8.43078973e-02,  8.50459031e-01,  2.07589511e-01,
         -3.32863765e-01],
        [ 1.15142407e-03,  1.71560187e-01,  4.50548364e-02,
         -6.26511999e-02],
        [ 7.80724099e-01,  7.61743069e-01, -1.64332320e-01,
         -4.77695865e-01],
        [ 9.84972885e-02,  5.40347312e-01, -3.91634328e-01,
         -4.95002161e-01]],

       [[-3.86195310e-01, -2.80250813e-01,  6.48377829e-01,
          4.18168880e-01],
        [-5.78113870e-01,  3.84871720e-01, -4.87218272e-02,
          5.85819606e-01],
        [-4.97036192e-01, -1.69247895e-01,  7.49693201e-02,
          2.99674184e-01],
        [-2.97217184e-01,  4.95764162e-01,  3.87399947e-01,
          6.73168664e-01],
        [-1.12098004e-01,  4.50198764e-01,  8.03781393e-01,
          4.68487747e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [ 1.35821620e-01,  1.49104614e-01,  1.01671180e-01,
          5.00927839e-01],
        [ 5.03269843e-01, -2.88086744e-01,  3.25004237e-02,
          3.32980702e-01],
        [-2.31418753e-01, -7.02801066e-01,  3.74928138e-01,
          3.72468648e-01],
        [ 2.95604063e-01,  1.61622122e-01, -1.92820667e-01,
         -3.96563257e-01]],

       [[-1.15630131e-01,  1.81282557e-01, -5.80787086e-02,
         -2.23880280e-01],
        [-2.42982104e-01,  1.47493454e-02, -4.60497192e-01,
         -5.21382334e-01],
        [-4.23697927e-01,  4.41542665e-01, -5.42853425e-01,
          3.02530871e-01],
        [ 2.06309718e-02,  6.49447209e-01,  4.34740875e-01,
          3.16329990e-01],
        [ 1.61480434e-01, -4.46146343e-01,  5.18222587e-01,
         -2.71847706e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [-4.26447248e-01,  3.50181114e-01,  8.48633984e-01,
          7.76049632e-01],
        [-2.20898323e-01, -4.36530061e-01,  6.36243522e-01,
          2.17618026e-01],
        [ 2.17101007e-01,  1.46937182e-01,  3.43673716e-01,
         -1.41815047e-02],
        [ 8.60256405e-02, -2.04041665e-01,  5.55356142e-01,
          9.89141124e-02]],

       [[ 4.39273097e-01, -6.41754044e-02, -4.87951531e-01,
          7.98038669e-01],
        [ 4.99640418e-01, -8.06201007e-02, -6.00881516e-01,
          4.38322194e-01],
        [-4.06330297e-01,  6.16770601e-01,  1.65544367e-02,
          9.53952135e-01],
        [-2.42544015e-01,  6.11237102e-01, -3.45924443e-01,
          7.34664149e-01],
        [ 2.30466644e-01,  1.88986752e-01,  6.62998478e-01,
          7.56297095e-02],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [ 1.58626256e-01,  1.04155670e-01,  2.04360240e-02,
          3.39388458e-01],
        [-9.08425471e-02,  5.18550283e-01, -1.36492622e-02,
          6.87634014e-01],
        [ 5.65012811e-01,  3.79559534e-01,  7.28737130e-01,
          2.18359252e-01],
        [-5.68817272e-01, -3.09848035e-01,  5.51208168e-01,
          5.32854565e-01]],

       [[-1.23906612e-01,  4.18687976e-01,  4.00690874e-01,
         -4.74532902e-01],
        [-4.06547124e-01,  5.67062540e-01, -1.96562552e-01,
         -2.69855592e-01],
        [ 6.29188998e-02, -3.03627000e-01, -7.16460413e-01,
         -4.95097786e-01],
        [ 1.37743446e-01,  3.67690596e-01, -7.80686337e-01,
          4.64058164e-01],
        [-4.61271524e-01,  6.60768345e-01,  2.40203096e-01,
         -3.87310505e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [-7.07306864e-01,  5.11757084e-02,  5.64126046e-01,
         -6.67303329e-02],
        [-6.52516871e-02,  6.14318381e-01, -5.41895845e-01,
         -2.37372403e-01],
        [ 1.84270277e-01,  2.03350501e-01,  2.62092127e-01,
         -5.18162706e-01],
        [-2.57384904e-01,  2.62887520e-01, -5.26981497e-01,
         -6.95852314e-01]],

       [[-4.46051213e-01, -4.72782926e-01, -6.55350837e-01,
          1.75770638e-01],
        [-2.07668267e-01, -7.29552102e-01,  3.74671990e-02,
          1.37415279e-01],
        [-4.45408549e-01, -9.52175917e-01, -9.52304158e-01,
         -1.04392070e-01],
        [-9.59088458e-01, -6.83786972e-01, -6.89687258e-01,
         -7.40005806e-03],
        [-6.99437112e-01, -1.07596223e-01, -2.22638801e-01,
          2.64748623e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [-4.70157064e-01, -3.77056739e-01, -8.85228333e-01,
         -3.07443563e-01],
        [-6.29529492e-02, -3.41153159e-01, -5.45065818e-01,
          5.52480251e-01],
        [-1.78971376e-01, -7.41232748e-01, -8.11892373e-01,
         -6.25864461e-01],
        [-4.18347392e-01, -4.81233102e-01, -4.34315101e-01,
          2.96165552e-01]],

       [[-9.64551914e-01,  6.37052790e-02, -8.20789151e-01,
         -6.74028474e-01],
        [-8.43836390e-01,  3.27555904e-04, -5.37965852e-01,
         -6.34040631e-01],
        [-3.66969884e-01, -3.55046119e-01, -3.70990197e-01,
         -6.84828531e-01],
        [-7.35663249e-01, -5.91669798e-01, -7.71253488e-01,
         -6.49356438e-01],
        [-9.86576824e-01, -2.96094649e-01,  1.04059056e-01,
         -3.04088095e-01],
        [ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,
          0.00000000e+00],
        [-9.67196192e-01, -4.65891346e-01, -3.42734222e-02,
         -8.08460169e-01],
        [-8.74840700e-01, -4.93356884e-01, -1.66316506e-01,
         -1.35478616e-01],
        [-2.05782438e-01, -2.41153273e-01, -7.55693427e-01,
         -2.96838383e-01],
        [-2.19025860e-01, -6.77901113e-01, -6.32138948e-01,
          3.18034914e-01]]])

Exercise 3¶

Use sc.mean to compute the mean of the data for Alice along the Z dimension.
Do the same with numpy, what are the complications you encounter, that are not present when using the dataset?

Solution 3¶

[18]:

help(sc.mean)

Help on function mean in module scipp.core.reduction:

mean(x: 'VariableLike', dim: 'Optional[str]' = None, *, out: 'Optional[VariableLike]' = None) -> 'VariableLike'
    Element-wise mean over the specified dimension.

    If the input has variances, the variances stored in the output are based on
    the "standard deviation of the mean", i.e.,
    :math:`\sigma_{mean} = \sigma / \sqrt{N}`.
    :math:`N` is the length of the input dimension.
    :math:`\sigma` is estimated as the average of the standard deviations of
    the input elements along that dimension.

    See :py:func:`scipp.sum` on how rounding errors for float32 inputs are handled.

    :param x: Input data.
    :param dim: Dimension along which to calculate the mean. If not
                given, the mean over all dimensions is calculated.
    :param out: Optional output buffer.
    :raises: If the dimension does not exist, or the dtype cannot be summed,
             e.g., if it is a string.
    :return: The mean of the input values.
    :seealso: :py:func:`scipp.nanmean`.

[19]:

mean = sc.mean(d['alice'], 'z')

When using numpy to compute the mean: - We must remember (or lookup) which dimension corresponds to the Z dimensions. - We need a separate call for values and variances. - We need to manually scale the variance with the inverse of the number of data points to get the variance of the mean (standard deviation of the mean scales with 1/sqrt(N)).

[20]:

np_values = np.mean(d['alice'].values, axis=0)
np_variances = np.mean(d['alice'].variances, axis=0)
np_variances /= d['alice'].shape[0]

[21]:

print(mean.values)
print(mean.variances)
print(np_values)
print(np_variances)

[[-0.18477415  0.0988275   0.01776119  0.08273942]
 [-0.28598818  0.1038312  -0.19147149 -0.12392005]
 [-0.18876425 -0.19998084 -0.11628796 -0.00845525]
 [-0.30417933  0.21231959 -0.08914926  0.17627372]
 [-0.26100667  0.05799879  0.39222252  0.09042428]
 [ 0.          0.          0.          0.        ]
 [-0.36770099  0.08875284  0.18810827  0.0899544 ]
 [-0.08838593 -0.01438854  0.01461715  0.15253546]
 [ 0.04790878  0.00634781  0.07020753 -0.10701726]
 [-0.13650677 -0.04253391 -0.00946149 -0.03979463]]
[[0.05106701 0.05942471 0.03819642 0.04812455]
 [0.05041095 0.05951739 0.03927906 0.04701604]
 [0.05158939 0.06000347 0.04275997 0.04794954]
 [0.04935181 0.06044867 0.0431661  0.04794473]
 [0.05191789 0.06368351 0.04294115 0.04730904]
 [0.05199373 0.06467688 0.04338447 0.04891914]
 [0.04964161 0.06031764 0.04324904 0.04795721]
 [0.04958878 0.06015519 0.04158757 0.04785204]
 [0.05142629 0.06140492 0.0410294  0.04800964]
 [0.05130711 0.06184661 0.04406647 0.05017599]]
[[-0.18477415  0.0988275   0.01776119  0.08273942]
 [-0.28598818  0.1038312  -0.19147149 -0.12392005]
 [-0.18876425 -0.19998084 -0.11628796 -0.00845525]
 [-0.30417933  0.21231959 -0.08914926  0.17627372]
 [-0.26100667  0.05799879  0.39222252  0.09042428]
 [ 0.          0.          0.          0.        ]
 [-0.36770099  0.08875284  0.18810827  0.0899544 ]
 [-0.08838593 -0.01438854  0.01461715  0.15253546]
 [ 0.04790878  0.00634781  0.07020753 -0.10701726]
 [-0.13650677 -0.04253391 -0.00946149 -0.03979463]]
[[0.05106701 0.05942471 0.03819642 0.04812455]
 [0.05041095 0.05951739 0.03927906 0.04701604]
 [0.05158939 0.06000347 0.04275997 0.04794954]
 [0.04935181 0.06044867 0.0431661  0.04794473]
 [0.05191789 0.06368351 0.04294115 0.04730904]
 [0.05199373 0.06467688 0.04338447 0.04891914]
 [0.04964161 0.06031764 0.04324904 0.04795721]
 [0.04958878 0.06015519 0.04158757 0.04785204]
 [0.05142629 0.06140492 0.0410294  0.04800964]
 [0.05130711 0.06184661 0.04406647 0.05017599]]

[ ]:

previous

1-D datasets and tables

next

About scipp