library(MASS)

custom_fun <- function(R, portfolio, rob_method = "mcd"){
    out <- list()
    out$sigma <- cov.rob(R, method = rob_method)
    return(out)

# Passing the rob_method argument to custom_fun
optimize.portfolio(R, portfolio, momentFUN = custom_fun, 
                     rob_method = "mcd")
optimize.portfolio(R, portfolio, momentFUN = custom_fun,  
                     rob_method = "mve")