I ended up finding the solution by using test and testContext.

idNumber: Yup.object({
        number: Yup.number(),
        label: Yup.string()
    })
        .test("", "ID is required", (value, testContext) => {
            let unknown = testContext.parent.name.isNameUnknown

            if (!unknown) {
                testContext.schema.fields.number.required()
                testContext.schema.fields.label.required()
            }
            
            return unknown || (!unknown && value.number && value.label)
        })