using Pkg
#Pkg.@pkg_str("add MemPool#master")
using JuliaDB
using CSV
#using Tables
using DelimitedFiles
using TextParse
using DataFrames
]st
fileToBeRead="C:\\temp\\test0.csv"
bindir="c:\\temp\\bindata"
mt=rand(5_000,5);
df=DataFrame(mt)
df[:,3]=Int.(trunc.(Int,100*mt[:,3]));
df[:,4]=Int.(trunc.(Int,10000*mt[:,4]));
isfile(fileToBeRead)&&rm(fileToBeRead)
CSV.write(fileToBeRead,df)
#read file with CSV
df_read=CSV.read(fileToBeRead,types=[Float64,Float64,Int64,Int64,Float64]);
sum(df_read[1])
@assert Int==eltype(df_read[3]) #ok
#read file with JuliaDB
@time csvfiles = glob(fileToBeRead);
!isdir(bindir) && mkdir(bindir)
@time loadndsparse(csvfiles, output=bindir,
header_exists=true,
chunks=80,
colparsers=Dict(1=>Float64, 2=>Float64, 3=>Int64,4=>Int64,5=>Float64),
datacols=[1,2,3,4,5])
ERROR: UndefRefError: access to undefined reference
getproperty(::Any, ::Symbol) at .\sysimg.jl:18
get_wrkrips() at C:\Users\bernhard.konig\.julia\packages\MemPool\PUncN\src\datastore.jl:65
run_work_thunk(::typeof(MemPool.get_wrkrips), ::Bool) at C:\cygwin\home\Administrator\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.0\Distributed\src\process_messages.jl:56
#remotecall_fetch#148(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::Function, ::Distributed.LocalProcess) at C:\cygwin\home\Administrator\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.0\Distributed\src\remotecall.jl:364
remotecall_fetch(::Function, ::Distributed.LocalProcess) at C:\cygwin\home\Administrator\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.0\Distributed\src\remotecall.jl:364
#remotecall_fetch#152(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::Function, ::Int64) at C:\cygwin\home\Administrator\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.0\Distributed\src\remotecall.jl:406
remotecall_fetch at C:\cygwin\home\Administrator\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.0\Distributed\src\remotecall.jl:406 [inlined]
get_workers_at(::Sockets.IPv4) at C:\Users\bernhard.konig\.julia\packages\MemPool\PUncN\src\datastore.jl:95
affinity(::MemPool.FileRef) at C:\Users\bernhard.konig\.julia\packages\Dagger\sdZXi\src\chunks.jl:84
affinity(::Dagger.Chunk{Any,MemPool.FileRef}) at C:\Users\bernhard.konig\.julia\packages\Dagger\sdZXi\src\chunks.jl:50
affinity(::Dagger.Thunk) at C:\Users\bernhard.konig\.julia\packages\Dagger\sdZXi\src\thunk.jl:52
pop_with_affinity!(::Dagger.Context, ::Array{Dagger.Thunk,1}, ::Dagger.OSProc, ::Bool) at C:\Users\bernhard.konig\.julia\packages\Dagger\sdZXi\src\scheduler.jl:97
compute_dag(::Dagger.Context, ::Dagger.Thunk) at C:\Users\bernhard.konig\.julia\packages\Dagger\sdZXi\src\scheduler.jl:36
compute(::Dagger.Context, ::Dagger.Thunk) at C:\Users\bernhard.konig\.julia\packages\Dagger\sdZXi\src\compute.jl:25
#fromchunks#47(::Nothing, ::Int64, ::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::Array{Dagger.Thunk,1}) at C:\Users\bernhard.konig\.julia\packages\JuliaDB\PRJbx\src\table.jl:148
fromchunks(::Array{Dagger.Thunk,1}) at C:\Users\bernhard.konig\.julia\packages\JuliaDB\PRJbx\src\table.jl:129
offset_index!(::JuliaDB.DNDSparse{Tuple{Int64},NamedTuple{(:x1, :x2, :x3, :x4, :x5),Tuple{Float64,Float64,Int64,Int64,Float64}}}, ::Int64) at C:\Users\bernhard.konig\.julia\packages\JuliaDB\PRJbx\src\io.jl:28
#_loadtable#188(::Int64, ::String, ::Bool, ::Array{Any,1}, ::Bool, ::Bool, ::Base.Iterators.Pairs{Symbol,Any,Tuple{Symbol,Symbol,Symbol},NamedTuple{(:header_exists, :colparsers, :datacols),Tuple{Bool,Dict{Int64,DataType},Array{Int64,1}}}}, ::Function, ::Type, ::Array{String,1}) at C:\Users\bernhard.konig\.julia\packages\JuliaDB\PRJbx\src\io.jl:153
#_loadtable at .\none:0 [inlined]
#loadndsparse#187 at C:\Users\bernhard.konig\.julia\packages\JuliaDB\PRJbx\src\io.jl:82 [inlined]
(::getfield(JuliaDB, Symbol("#kw##loadndsparse")))(::NamedTuple{(:output, :header_exists, :chunks, :colparsers, :datacols),Tuple{String,Bool,Int64,Dict{Int64,DataType},Array{Int64,1}}}, ::typeof(loadndsparse), ::Array{String,1}) at .\none:0
top-level scope at util.jl:156
eval(::Module, ::Any) at .\boot.jl:319
eval_user_input(::Any, ::REPL.REPLBackend) at C:\cygwin\home\Administrator\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.0\REPL\src\REPL.jl:85
macro expansion at C:\cygwin\home\Administrator\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.0\REPL\src\REPL.jl:117 [inlined]
(::getfield(REPL, Symbol("##28#29")){REPL.REPLBackend})() at .\task.jl:259
Stacktrace:
[1] #remotecall_fetch#148(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::Function, ::Distributed.LocalProcess) at C:\cygwin\home\Administrator\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.0\Distributed\src\remotecall.jl:365
[2] remotecall_fetch(::Function, ::Distributed.LocalProcess) at C:\cygwin\home\Administrator\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.0\Distributed\src\remotecall.jl:364
[3] #remotecall_fetch#152(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::Function, ::Int64) at C:\cygwin\home\Administrator\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.0\Distributed\src\remotecall.jl:406
[4] remotecall_fetch at C:\cygwin\home\Administrator\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.0\Distributed\src\remotecall.jl:406 [inlined]
[5] get_workers_at(::Sockets.IPv4) at C:\Users\bernhard.konig\.julia\packages\MemPool\PUncN\src\datastore.jl:95
[6] affinity(::MemPool.FileRef) at C:\Users\bernhard.konig\.julia\packages\Dagger\sdZXi\src\chunks.jl:84
[7] affinity(::Dagger.Chunk{Any,MemPool.FileRef}) at C:\Users\bernhard.konig\.julia\packages\Dagger\sdZXi\src\chunks.jl:50
[8] affinity(::Dagger.Thunk) at C:\Users\bernhard.konig\.julia\packages\Dagger\sdZXi\src\thunk.jl:52
[9] pop_with_affinity!(::Dagger.Context, ::Array{Dagger.Thunk,1}, ::Dagger.OSProc, ::Bool) at C:\Users\bernhard.konig\.julia\packages\Dagger\sdZXi\src\scheduler.jl:97
[10] compute_dag(::Dagger.Context, ::Dagger.Thunk) at C:\Users\bernhard.konig\.julia\packages\Dagger\sdZXi\src\scheduler.jl:36
[11] compute(::Dagger.Context, ::Dagger.Thunk) at C:\Users\bernhard.konig\.julia\packages\Dagger\sdZXi\src\compute.jl:25
[12] #fromchunks#47(::Nothing, ::Int64, ::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::Array{Dagger.Thunk,1}) at C:\Users\bernhard.konig\.julia\packages\JuliaDB\PRJbx\src\table.jl:148
[13] fromchunks(::Array{Dagger.Thunk,1}) at C:\Users\bernhard.konig\.julia\packages\JuliaDB\PRJbx\src\table.jl:129
[14] offset_index!(::JuliaDB.DNDSparse{Tuple{Int64},NamedTuple{(:x1, :x2, :x3, :x4, :x5),Tuple{Float64,Float64,Int64,Int64,Float64}}}, ::Int64) at C:\Users\bernhard.konig\.julia\packages\JuliaDB\PRJbx\src\io.jl:28
[15] #_loadtable#188(::Int64, ::String, ::Bool, ::Array{Any,1}, ::Bool, ::Bool, ::Base.Iterators.Pairs{Symbol,Any,Tuple{Symbol,Symbol,Symbol},NamedTuple{(:header_exists, :colparsers, :datacols),Tuple{Bool,Dict{Int64,DataType},Array{Int64,1}}}}, ::Function, ::Type, ::Array{String,1}) at C:\Users\bernhard.konig\.julia\packages\JuliaDB\PRJbx\src\io.jl:153
[16] #_loadtable at .\none:0 [inlined]
[17] #loadndsparse#187 at C:\Users\bernhard.konig\.julia\packages\JuliaDB\PRJbx\src\io.jl:82 [inlined]
[18] (::getfield(JuliaDB, Symbol("#kw##loadndsparse")))(::NamedTuple{(:output, :header_exists, :chunks, :colparsers, :datacols),Tuple{String,Bool,Int64,Dict{Int64,DataType},Array{Int64,1}}}, ::typeof(loadndsparse), ::Array{String,1}) at .\none:0
[19] top-level scope at util.jl:156