Turns out you can just stick a b in front of each string to convert it to bytes and everything works great!

class DownloadHandler(web.RequestHandler): 
    def get(self): 
        files_as_text = b"" 
        for file in os.listdir("files"): 
            files_as_text += file.encode("utf-8")+b"---title_split---"+open("files/"+file, "rb").read()+b"---file_split---" 
        self.write(files_as_text)