This will be a long answer. So I made some of my custom configurations to fix this, also took help from the answer by @ShehanLakshita.

Firstly I wrapped PrivateRoute and Route with memo to prevent unnecessary re renders,

const MemoizedRoute = React.memo(({ isPrivate, component: Component, ...rest }) => {
if (isPrivate) {
  return (
    <PrivateRoute
      {...rest}
      component={Component} 
    />
  );
}
return (
  <Route
    {...rest}
    render={routeProps => <Component {...routeProps} {...rest} />}
  />
);
});

I modified my loadable component to cache the LazyComponent correctly as suggested by @Shehan,

const cachedComponentsMap = new Map();

const useCachedLazy = (importFunc, namedExport) => {
  try {
    if (!cachedComponentsMap.has(importFunc)) {
      const LazyComponent = lazy(() =>
        importFunc().then(module => ({
          default: namedExport ? module[namedExport] : module.default,
        }))
      );
      cachedComponentsMap.set(importFunc, LazyComponent);
    }
    return cachedComponentsMap.get(importFunc);
  } catch (error) {
    console.error('Error loading component:', error);
    throw error; 
  }
};


const loadable = (importFunc, { namedExport = null } = {}) => {
  const cachedKey = `${importFunc}-${namedExport}`;

  if (!cachedComponentsMap.has(cachedKey)) {
    const MemoizedLazyComponent = React.memo(props => {
      const LazyComponent = useCachedLazy(importFunc, namedExport);
      return <LazyComponent {...props} />;
    });

    cachedComponentsMap.set(cachedKey, MemoizedLazyComponent);
  }

  return cachedComponentsMap.get(cachedKey);
};

Now the components do not unmount and remount unnecessary and I also got lazy working and the performance is improved.