Refer - https://medium.com/@pradipsudo/explore-nullable-property-of-columns-in-a-spark-data-frame-1d1b7b042adb

val cols = for {
    f <- df.schema.fields
} yield {
    import org.apache.spark.sql.catalyst.expressions.object
    new Column(AssertNotNull(col(f.name).expr)).alias(f.name)
}

df.select(cols._*)