Well, so far I have resolved by avoiding stripping the full source path and having a parallel path for the object:

BUILDDIR = ./build
EXT_SRC := $(wildcard ext/lib1/src/*.c) \
          $(wildcard ext/lib2/*.c) \
          $(wildcard ext/lib3/xyz/*.c)
EXT_OBJ := $(EXT_SRC:%.c=$(BUILDDIR)/%.o)

$(BUILDDIR)/ext/%.o: ext/%.c
    mkdir -p $(dir $@)
    $(CC) $(CFLAGS) -c $^ -o $@

.PHONY: clean
clean:
    rm -rf $(BUILDDIR)