Try like this.

$builder->when(! $builder->getQuery()->unions
               && is_null($builder->getQuery()->aggregate),
    function ($query) {
        $query->orderBy('priority');
    }
);