I finally found another workaround without testing the function directly. Instead, I just wrote an assertion that tests the state to not have been changed.
it('does not toggle sound, when space key is clicked', () => {
const preloadedState = {
sound: {
isMuted: true,
},
};
const { getByLabelText, store } = renderWithProviders(<SoundBtn />, { preloadedState });
const button = getByLabelText('click to play sound');
fireEvent.keyDown(button, { code: 'Space', key: ' ' });
expect(store.getState().sound.isMuted).toBe(true);
});