vignette atime data.t.Rmd

---
title: "vignette atime"
author: "Doris Amoakohene"
date: "`r Sys.Date()`"
output: html_document
---


Modify atime compare-data.table-tidyverse vignette, and analyze efficiency of packages such as polars, arrow, collapse, spark.
To modify the atime vignette in the compare-data.table-tidyverse package and analyze the efficiency of packages such as polars, arrow, collapse, and spark, you would need to perform the following steps:

Perform Comparison with 
data.table
arrow

adding:

collapse 
spark


The purpose of this vignette is to make figures which show the efficiency of data.table.

fwrite: Fast csv writer

```{r}
library(data.table)
library(reshape2)
library(readr)
library(arrow)
library(ggplot2)
library(collapse)
#library(sparklyr)
library(polars)
library(dplyr)
library(plyr)
library(tidyr)
library(stats)
library(readxl)
library(summarytools)
library(psych)


#library(sparklyr)

#spark_install()


#library(sparklyr)

#sc <- spark_connect(master = "local")

#options(sparklyr.log.console = TRUE)
#sc <- spark_connect(master = "local")

#sc <- spark_connect(master = "local")
```

```{r}
write.colors <- c(
  "readr::write_csv"="#9970AB",
  "data.table::fwrite"="#D6604D",
  "write_csv_arrow"="#BF812D",
  "polars::DataFrame_write_csv"="#F1EB90",
  #"sparklyr::spark_write_csv" = "#722f37",
  "utils::write.csv"="deepskyblue")

n.rows <- 100
seconds.limit <- 1


atime.write.vary.cols <- atime::atime(
  N=as.integer(10^seq(2, 6, by=0.5)),
  setup={
    set.seed(1)
    input.vec <- rnorm(n.rows*N)
    input.mat <- matrix(input.vec, n.rows, N)
    input.df <- data.frame(input.mat)
    #df <- copy_to(sc, input.df)
    
  },
  seconds.limit = seconds.limit,
  "data.table::fwrite"={
    data.table::fwrite(input.df, tempfile(), showProgress = FALSE)
  },
  "write_csv_arrow"={
    arrow::write_csv_arrow(input.df, tempfile())
  },
  "readr::write_csv"={
    readr::write_csv(input.df, tempfile(), progress = FALSE)
  },
  "polars::DataFrame_write_csv"={
    
    pl$DataFrame(input.df)$write_csv(tempfile(fileext = ".csv"))
  },
  #"write_csv_spark"={
    #sparklyr::spark_write_csv(df, tempfile(fileext = ".csv"))
  #},
  "utils::write.csv"= {
    utils::write.csv(input.df, tempfile())
  }
)
```



```{r}
refs.write.vary.cols <- atime::references_best(atime.write.vary.cols)
pred.write.vary.cols <- predict(refs.write.vary.cols)

gg.write <- plot(pred.write.vary.cols)+
  theme(text=element_text(size=20))+
  ggtitle(sprintf("Write real numbers to CSV, %d x N", n.rows))+
  scale_x_log10("N = number of columns to write")+
  scale_y_log10("Computation time (seconds)
median line, min/max band
over 10 timings")+
  facet_null()+
  scale_fill_manual(values=write.colors)+
  scale_color_manual(values=write.colors)



```

```{r}

print(gg.write)
ggsave("gg.write.png", gg.write, width = 10, height = 6, units = "in", dpi = 300)
dev.off()


gg.write
```

fread: fast CSV reader

```{r}
read.colors <- c(
  "readr::read_csv\n(lazy=TRUE)"="#9970AB",
  "data.table::fread"="#D6604D",
  "read_csv_arrow"="#BF812D",
  #"sparklyr::spark_read_csv"="#722f37",
  "polars::pl$read_csv"="#F1EB90",
  "utils::read.csv" = "deepskyblue")


atime.read.vary.cols <- atime::atime(
  N=as.integer(10^seq(2, 6, by=0.5)),
  setup={
    set.seed(1)
    input.vec <- rnorm(n.rows*N)
    input.mat <- matrix(input.vec, n.rows, N)
    input.df <- data.frame(input.mat)
    input.csv <- tempfile()
    fwrite(input.df, input.csv)
  },
  seconds.limit = seconds.limit,
  "data.table::fread"={
    data.table::fread(input.csv, showProgress = FALSE)
  },
  "read_csv_arrow"={
    arrow::read_csv_arrow(input.csv)
  },
  "readr::read_csv\n(lazy=TRUE)"={
    readr::read_csv(input.csv, progress = FALSE, show_col_types = FALSE, lazy=TRUE)
  },
  "polars::pl$read_csv" = {
    polars::pl$read_csv(input.csv)
  },
  #"sparklyr::spark_read_csv"= sparklyr::spark_read_csv(input.csv)
  #},
  "utils::read.csv"=utils::read.csv(input.csv))
```


```{r}
refs.read.vary.cols <- atime::references_best(atime.read.vary.cols)
pred.read.vary.cols <- predict(refs.read.vary.cols)

gg.read <- plot(pred.read.vary.cols)+
  theme(text=element_text(size=20))+
  ggtitle(sprintf("Read real numbers from CSV, %d x N", n.rows))+
  scale_x_log10("N = number of columns to read")+
  scale_y_log10("Computation time (seconds)
median line, min/max band
over 10 timings")+
  facet_null()+
  scale_fill_manual(values=read.colors)+
  scale_color_manual(values=read.colors)
```

```{r}

print(gg.read)
ggsave("gg.read.png", gg.read, width = 10, height = 6, units = "in", dpi = 300)
dev.off()

gg.read
```


Summarize by group



```{r}
ml.colors <- c(
  "dplyr::summarise"="#00FF00",
  "[.data.table"="#D6604D",
  "stats::aggregate"="#4DB3E6",
  "summarytools::descr"="#00BFC4",
  "psych::describe"="#FFA500",
  "plyr::ddply"="#FF0000FF")
  
options(dplyr.summarise.inform=FALSE)
n.folds <- 10
ml.atime <- atime::atime(
  N=as.integer(10^seq(2, 7, by=0.5)),
  setup={
    loss.dt <- data.table(
      name="loss", 
      fold=rep(1:n.folds, each=2*N),
      loss=rnorm(2*N*n.folds),
      set=rep(c("subtrain","validation"),each=N),
      epoch=1:N,
      key=c("set","epoch","fold"))
  },
  seconds.limit=seconds.limit,
  "[.data.table"={
    loss.dt[, .(
      loss_length=.N,
      loss_mean=mean(loss),
      loss_sd=sd(loss)
    ), by=.(set, epoch)]
  },
  "stats::aggregate"={
    res <- stats::aggregate(
      loss ~ set + epoch, 
      loss.dt, 
      function(values)list(c(
        loss_length=length(values),
        loss_mean=mean(values), 
        loss_sd=sd(values))))
    data.frame(
      subset(res, select=-loss), 
      do.call(rbind, res$loss))
  },
  "plyr::ddply"={
    ddply(loss.dt, c("set", "epoch"), summarize,
          loss_length = length(loss),
          loss_mean = mean(loss),
          loss_sd = sd(loss))
  },
  "dplyr::summarise"={
    loss.dt |> 
      dplyr::group_by(set, epoch) |> 
      dplyr::summarise(
        loss_length=length(loss),
        loss_mean=mean(loss), 
        loss_sd=sd(loss)
        )
    })

```


```{r}
ml.refs <- atime::references_best(ml.atime)
ml.pred <- predict(ml.refs)
ml.gg <- plot(ml.pred)+
  theme(text=element_text(size=20))+
  ggtitle(sprintf("Mean,SD,Length over %d real numbers, N times", n.folds))+
  scale_x_log10("N = number of Mean,SD,Length to compute")+
  scale_y_log10("Computation time (seconds)
median line, min/max band
over 10 timings")+
  facet_null()+
  scale_fill_manual(values=ml.colors)+
  scale_color_manual(values=ml.colors)

```


```{r}

print(ml.gg)
ggsave("ml.gg.png", ml.gg, width = 10, height = 6, units = "in", dpi = 300)
dev.off()

ml.gg


```

Summarize by group, expanded
```{r}
options(dplyr.summarise.inform=FALSE)
n.folds <- 10
ml.exp.atime <- atime::atime(
  N=as.integer(10^seq(2, 7, by=0.5)),
  setup={
    loss.dt <- data.table(
      name="loss", 
      fold=rep(1:n.folds, each=2*N),
      loss=rnorm(2*N*n.folds),
      set=rep(c("subtrain","validation"),each=N),
      epoch=1:N,
      key=c("set","epoch","fold"))
    },
  seconds.limit=seconds.limit,
  "[.data.table"={
    loss.dt[, .(
      loss_length=.N,
      loss_mean=mean(loss),
      loss_sd=sd(loss)
    ), by=.(set, epoch)]
  },
  "stats::aggregate"={
    res <- stats::aggregate(
      loss ~ set + epoch, 
      loss.dt, 
      function(values)list(c(
        loss_length=length(values),
        loss_mean=mean(values), 
        loss_sd=sd(values))))
    data.frame(
      subset(res, select=-loss), 
      do.call(rbind, res$loss))
  },
  "dplyr::summarise"={
    loss.dt |> 
      dplyr::group_by(set, epoch) |> 
      dplyr::summarise(
        loss_length=length(loss),
        loss_mean=mean(loss), 
        loss_sd=sd(loss))
  },
  "collapse::fsummarise"={
    loss.dt |> 
      collapse::fgroup_by(set, epoch) |> 
      collapse::fsummarise(
        loss_length=length(loss),
        loss_mean=mean(loss), 
        loss_sd=sd(loss))
  })

```

```{r}
ml.exp.refs <- atime::references_best(ml.exp.atime)
ml.exp.pred <- predict(ml.exp.refs)
ml.exp.colors <- c(
  "collapse::fsummarise"="#5AAE61",
  "dplyr::summarise"="#00FF00",
  "[.data.table(key)"="#D6604D",
  "[.data.table(no key)"="#B6604D",
  "stats::aggregate"="#4DB3E6")
ml.exp.gg <- plot(ml.exp.pred)+
  theme(text=element_text(size=20))+
  ggtitle(sprintf("Mean,SD,Length over %d real numbers, N times", n.folds))+
  scale_x_log10("N = number of Mean,SD,Length to compute")+
  scale_y_log10("Computation time (seconds)
median line, min/max band
over 10 timings")+
  facet_null()+
  scale_fill_manual(values=ml.exp.colors)+
  scale_color_manual(values=ml.exp.colors)
```

```{r}

print(ml.exp.gg)
ggsave("ml.exp.gg.png", ml.exp.gg, width = 10, height = 6, units = "in", dpi = 300)
dev.off()

ml.exp.gg
```


Reshaping wide to long 
```{r}
ml.colors <- c(
  "tidyr::tidyr_pivot_longer"="#D95F02",
  "data.table::melt"="#D6604D",
  "stats::reshape"="#4DB3E6")

ml.reshape.atime <- atime::atime(
  N=as.integer(10^seq(2, 15, by=0.5)),
  setup={
    df <- data.frame(
      id = rep(1:N, each = 2),
      category = rep(c("A", "B"), N),
      value = rnorm(2 * N)
      )
    },
  seconds.limit= seconds.limit,
  "stats::reshape"={
    stats::reshape(df, direction = "long",idvar = "state", varying = list(names(df)), v.names = "value", timevar = "category", times = names(df))
    },
    "tidyr::tidyr_pivot_longer" = {
      tidyr::pivot_longer(df, cols = starts_with("value"), names_to = "category", values_to = "value", names_repair = "unique")
    },
    "data.table::melt" = {
      data.table::melt(data.table(df), id.vars = "id", variable.name = "category", value.name = "value")
    }
  )

```

```{r}
ml.reshape.refs <- atime::references_best(ml.reshape.atime)
ml.reshape.pred <- predict(ml.reshape.refs)
ml.reshape <- plot(ml.reshape.pred)+
  theme(text=element_text(size=17))+
  ggtitle(sprintf("Reshaping from wide to long over %dreal numbers, N times", n.folds))+
  scale_x_log10("N = number of Mean,SD,Length to compute")+
  scale_y_log10("Computation time (seconds)
median line, min/max band
over 10 timings")+
  facet_null()+
  scale_fill_manual(values=ml.colors)+
  scale_color_manual(values=ml.colors)
```

```{r}
print(ml.reshape)
ggsave("ml.reshape.png", ml.reshape, width = 10, height = 6, units = "in", dpi = 300)
dev.off()

ml.reshape
```




Reshaping Long to wide 



```{r}

ml.colors <- c(
  "tidyr_pivot_wide"="#D95F02",
  "data_table_dcast"="#D6604D",
  "stats_reshape"="#4DB3E6")

ml.wide.atime <- atime::atime(
  N=as.integer(10^seq(2, 7, by=0.5)),
  
  setup={
    df <- data.frame(
      id = rep(1:N, each = 2),
      category = rep(c("A", "B"), N),
      value = rnorm(2 * N)
      )
    },
  seconds.limit= seconds.limit,
  "stats_reshape"={
    reshape(df, direction = "wide", idvar = "id", timevar = "category")
    },
    "tidyr_pivot_wide" = {
      tidyr::pivot_wider(df, names_from = "category", values_from = value)
    },
    data_table_dcast = {
      data.table::dcast(data.table(df), id ~ category, value.var = "value")
    }
  )

```

```{r}
ml.wide.refs <- atime::references_best(ml.wide.atime)
ml.wide.pred <- predict(ml.wide.refs)
ml.wide <- plot(ml.wide.pred)+
  theme(text=element_text(size=20))+
  ggtitle(sprintf("Reshaping from long to wide over %dreal numbers, N times", n.folds))+
  scale_x_log10("N = number of Mean,SD,Length to compute")+
  scale_y_log10("Computation time (seconds)
median line, min/max band
over 10 timings")+
  facet_null()+
  scale_fill_manual(values=ml.colors)+
  scale_color_manual(values=ml.colors)
```

```{r}
print(ml.wide)
ggsave("ml.wide.png", ml.wide, width = 10, height = 6, units = "in", dpi = 300)
dev.off()

ml.wide
```



creating data frame

```{r}
library(data.table)
library(dplyr)
library(atime)

ml.colors <- c(
  "data_table_create" = "#D6604D",
  "dplyr_create" = "#00FF00",
  "base_create" = "#7570B3"
)
```


```{r}
ml.create.atime <- atime::atime(
  N = as.integer(10^seq(2, 7, by = 0.5)),
  
  data_table_create = {
    dt <- data.table(
      id = rep(1:N, each = 2),
      category = rep(c("A", "B"), N),
      value = rnorm(2 * N)
    )
  },
  
  dplyr_create = {
    df <- tibble(
      id = rep(1:N, each = 2),
      category = rep(c("A", "B"), N),
      value = rnorm(2 * N)
    )
  },
  
  base_create = {
    df <- data.frame(
      id = rep(1:N, each = 2),
      category = rep(c("A", "B"), N),
      value = rnorm(2 * N)
    )
  }
)
```


```{r}
ml.create.refs <- atime::references_best(ml.create.atime)
ml.create.pred <- predict(ml.create.refs)
ml.create <- plot(ml.create.pred) +
  theme(text = element_text(size = 20)) +
  ggtitle(sprintf("Creating data frames over %d rows", n.folds)) +
  scale_x_log10("N = number of rows") +
  scale_y_log10("Computation time (seconds)") +
  facet_null() +
  scale_fill_manual(values = ml.colors) +
  scale_color_manual(values = ml.colors)
```


```{r}
print(ml.create)
ggsave("ml.create.png", ml.create, width = 10, height = 6, units = "in", dpi = 300)

ml.create
```