Using the example found at this link provided by @margusl, I was able to get this working. https://rstudio.github.io/websocket/#websocket-proxy

library(httpuv)
library(httr)

s <- startServer("0.0.0.0", 8080,
                 list(
                   call = function(req) {
                     
                     res <- GET(paste0("http://127.0.0.1:4040", req$PATH_INFO))
                     
                     content_type <- if (grepl("\\.css$", req$PATH_INFO)) {
                       "text/css"
                     } else if (grepl("\\.js$", req$PATH_INFO)) {
                       "application/javascript"
                     } else {
                       "text/html"
                     }
                     
                     list(
                       status = 200L,
                       headers = list(
                         'Content-Type' = content_type,
                         'X-Forwarded-Host' = req$HTTP_HOST,
                         'X-Forwarded-For' = req$REMOTE_ADDR
                       ),
                       body = content(res, type = "raw")
                     )
                   },
                   onWSOpen = function(clientWS) {
                     serverWS <- websocket::WebSocket$new("ws://127.0.0.1:4040")
                     
                     msg_from_client_buffer <- list()
                     # Flush the queued messages from the client
                     flush_msg_from_client_buffer <- function() {
                       for (msg in msg_from_client_buffer) {
                         serverWS$send(msg)
                       }
                       msg_from_client_buffer <<- list()
                     }
                     
                     clientWS$onMessage(function(binary, msgFromClient) {
                       if (serverWS$readyState() == 0) {
                         msg_from_client_buffer[length(msg_from_client_buffer) + 1] <<- msgFromClient
                       } else {
                         serverWS$send(msgFromClient)
                       }
                     })
                     
                     serverWS$onOpen(function(event) {
                       serverWS$onMessage(function(msgFromServer) {
                         clientWS$send(msgFromServer$data)
                       })
                       flush_msg_from_client_buffer()
                     })
                   }
                 )
)