Reports

You cannot use fill for ArrayType and StructType columns: "The replacement value must be an int, float, boolean, or string."

Try withColumn with when/otherwise:

df = df.fillna(
    {
        "responseStatus": "SUCCESS",
    }
)
df = df.withColumn("data", F.when(df.data.isNull(), F.array()).otherwise(df.data))
df = df.withColumn(
    "responseDetails",
    F.when(
        df.responseDetails.isNull(),
        F.struct(
            F.lit(0).alias("pagesize"),
            F.lit(0).alias("pageoffset"),
            F.lit(0).alias("size"),
            F.lit(0).alias("total"),
        ),
    ).otherwise(df.responseDetails),
)
df.show()

79267538