We could probably parallelise dbWriteTable, in particular value conversions. Also on the query side. Because every column is essentially independent. Actually thats a great idea, will probably add this at some point.
importing ten large tables on ten cores is 7x as fast with a monetdb session that allows multiple connections. until monetdblite can compete with multicore imports, monetdb server control code should not be deprecated.
library(DBI)
library(MonetDBLite)
library(MonetDB.R)
# create ten csv files with 1,000,000 records each
for( j in 1:10 ){
tf <- tempfile()
write.csv( mtcars[ rep( 1:32 , 31250 ) , ] , tf , row.names = FALSE )
assign( paste0( "tf" , j ) , tf )
}
# # # # # monetdblite import of 10,000,000 records
db <- dbConnect( MonetDBLite() )
system.time( {
for( j in 1:10 ) {
tablename <- basename( get( paste0( 'tf' , j ) ) )
dbWriteTable( db , tablename , mtcars[ 0 , ] )
dbSendUpdate(db, paste0("COPY OFFSET 2 INTO ", tablename, " FROM '", get( paste0( 'tf' , j ) ) , "' using delimiters ',','\\n','\"' NULL AS ''" ) )
}
} )
# # # # # external mserver import of 10,000,000 records
batfile <-
monetdb.server.setup(
database.directory = paste0( tempdir() , "/MonetDB" ) ,
monetdb.program.path =
ifelse(
.Platform$OS.type == "windows" ,
"C:/Program Files/MonetDB/MonetDB5" ,
""
) ,
dbname = "mydb" ,
dbport = 50000
)
dbname <- "mydb"
dbport <- 50000
monetdb.server.start( batfile )
monet.url <- paste0( "monetdb://localhost:" , dbport , "/" , dbname )
mydb <- dbConnect( MonetDB.R() , monet.url , wait = TRUE )
pid <- as.integer( dbGetQuery( mydb , "SELECT value FROM env() WHERE name = 'monet_pid'" )[[1]] )
library(snow)
cl<-makeCluster(10,type="SOCK")
myfun <-
function( myfile , murl ){
library(DBI)
library(MonetDBLite)
library(MonetDB.R)
tablename <- basename( myfile )
con <- dbConnect( MonetDB.R() , murl , wait = TRUE )
dbSendUpdate(con, paste0("COPY OFFSET 2 INTO ", tablename, " FROM '", myfile , "' using delimiters ',','\\n','\"' NULL AS ''" ) )
TRUE
}
system.time({
for( j in 1:10 ) {
tablename <- basename( get( paste0( 'tf' , j ) ) )
dbWriteTable( mydb , tablename , mtcars[ 0 , ] )
}
clusterApply(cl,sapply( paste0( 'tf' , 1:10 ) , get ) ,myfun , murl = monet.url)
})
stopCluster(cl)
monetdb.server.stop( pid )
# monetdblite time:
# user system elapsed
#58.28 2.87 98.41
# external mserver time:
# user system elapsed
#0.04 0.02 14.12