After some digging, I finally got it to work.

import importlib
import sys

# 1. manually create a namespace package specification
spec = importlib.machinery.ModuleSpec("xxx", None)
spec.submodule_search_locations = ["/path/xxx"]

# 2. import module from the spec
xxx = importlib.util.module_from_spec(spec)

# 3. add it to global cache
sys.modules["xxx"] = xxx