As a workaround for now I went with installing the dependencies via an init script on the cluster.
resources:
pipelines:
bronze:
name: ${var.stage_name}_bronze
clusters:
- label: default
autoscale: ${var.default_dlt_cluster.autoscale}
spark_conf: ${var.default_dlt_cluster.spark_conf}
init_scripts:
- workspace:
destination: ${workspace.file_path}/resources/init_scripts/cluster_dependencies.sh
[...]