You can create a new column with the square value and then do the groupby:

data = pd.DataFrame({"day": [0,0,0,0,0,0,1,1,1,1,1,1], "cat1": ["A", "A", "A", "B", "B", "B", "A", "A", "B", "B", "B", "B"], "cat2":["1", "1", "2", "1", "2", "2", "1", "2", "1", "1", "2", "2"], "value": [10, 230, 32,12, 12, 65, 12, 34, 97, 0, 12,1]})
data["value2"] = data["value"] * data["value"]
display(data)

gb = data.groupby(["day", "cat1", "cat2"])["value2"].mean()
display(gb)