This will make ./gradlew testFlavor1UnitTest exclude all classes from the common test folder while keeping everything else exactly the same.

afterEvaluate {
    tasks.withType(Test).matching { it.name.contains('Flavor') }.all {
        exclude '**/test/**'
    }
}