Comments (19)
wanghaisheng
commented on June 2, 2024
1
https://github.com/mittagessen/kraken-models
-
- against arabic-beirut-200.clstm
root@4ef282b20886:/work/clstm-based-japan# ./run-xps-tests
>>>>>>> export seed=0.222
>>>>>>> seed=0.222
>>>>>>> scons -s -c
>>>>>>> rm -f test-lstm.o '*.a'
>>>>>>> scons -j 4 gpu=0 debug=0 options= clstmocrtrain clstmfiltertrain clstmfilter clstmocr test-lstm
scons: Reading SConscript files ...
scons: done reading SConscript files.
scons: Building targets ...
g++ --std=c++11 -Wno-unused-result -o clstmocrtrain.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstmocrtrain.cc
g++ --std=c++11 -Wno-unused-result -o clstm.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm.cc
g++ --std=c++11 -Wno-unused-result -o ctc.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 ctc.cc
protoc(["clstm", "clstm.pb.cc", "clstm.pb.h"], ["clstm.proto"])
g++ --std=c++11 -Wno-unused-result -o clstm_proto.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm_proto.cc
g++ --std=c++11 -Wno-unused-result -o clstm_prefab.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm_prefab.cc
g++ --std=c++11 -Wno-unused-result -o tensor.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 tensor.cc
g++ --std=c++11 -Wno-unused-result -o batches.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 batches.cc
g++ --std=c++11 -Wno-unused-result -o extras.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 extras.cc
g++ --std=c++11 -Wno-unused-result -o clstm.pb.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm.pb.cc
g++ --std=c++11 -Wno-unused-result -o clstm_compute.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm_compute.cc
g++ --std=c++11 -Wno-unused-result -o clstmfiltertrain.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstmfiltertrain.cc
g++ --std=c++11 -Wno-unused-result -o clstmfilter.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstmfilter.cc
g++ --std=c++11 -Wno-unused-result -o clstmocr.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstmocr.cc
g++ --std=c++11 -Wno-unused-result -o test-lstm.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 test-lstm.cc
ar rc libclstm.a clstm.o ctc.o clstm_proto.o clstm_prefab.o tensor.o batches.o extras.o clstm.pb.o clstm_compute.o
ranlib libclstm.a
g++ --std=c++11 -Wno-unused-result -o test-lstm test-lstm.o libclstm.a -lpng -lprotobuf
g++ --std=c++11 -Wno-unused-result -o clstmocr clstmocr.o libclstm.a -lpng -lprotobuf
g++ --std=c++11 -Wno-unused-result -o clstmocrtrain clstmocrtrain.o libclstm.a -lpng -lprotobuf
g++ --std=c++11 -Wno-unused-result -o clstmfiltertrain clstmfiltertrain.o libclstm.a -lpng -lprotobuf
g++ --std=c++11 -Wno-unused-result -o clstmfilter clstmfilter.o libclstm.a -lpng -lprotobuf
scons: done building targets.
>>>>>>> ./test-lstm
#: ntrain = 100000
#: ntest = 1000
#: gpu = -1
training 1:4:2 network to learn delay
.Stacked <<<0.0001 0.9 in 20 1 out 20 2>>>
.Stacked inputs 20 1 1 Seq:[0.000000|0.400000|1.000000:20][-0.030502|-0.001325|0.025830:20]
.Stacked outputs 20 2 1 Seq:[0.000045|0.500000|0.999955:40][-0.003537|0.000000|0.003537:40]
.Stacked.NPLSTM <<<0.0001 0.9 in 20 1 out 20 4>>>
.Stacked.NPLSTM WCI 4 6 Bat:[-2.371926|-0.180369|1.806145:24][-0.042775|0.002258|0.046467:24]
.Stacked.NPLSTM WGF 4 6 Bat:[-1.157534|0.080894|1.083409:24][-0.006257|0.002911|0.012813:24]
.Stacked.NPLSTM WGI 4 6 Bat:[-0.212294|0.408031|2.245568:24][-0.002479|0.003996|0.014401:24]
.Stacked.NPLSTM WGO 4 6 Bat:[-2.919670|0.557016|3.321400:24][-0.018948|0.007477|0.029906:24]
.Stacked.NPLSTM inputs 20 1 1 Seq:[0.000000|0.400000|1.000000:20][-0.030502|-0.001325|0.025830:20]
.Stacked.NPLSTM outputs 20 4 1 Seq:[-0.953226|0.038960|0.739090:80][-0.043424|0.000437|0.023548:80]
.Stacked.NPLSTM ci 20 4 1 Seq:[-0.999453|-0.009550|0.961935:80][-0.009550|-0.000172|0.014207:80]
.Stacked.NPLSTM gf 20 4 1 Seq:[0.103049|0.455415|0.836064:80][-0.001099|-0.000045|0.001491:80]
.Stacked.NPLSTM gi 20 4 1 Seq:[0.691834|0.832398|0.977853:80][-0.000271|0.000066|0.000974:80]
.Stacked.NPLSTM go 20 4 1 Seq:[0.063462|0.769145|0.996185:80][-0.001031|0.000088|0.004340:80]
.Stacked.NPLSTM source 20 5 1 Seq:[-0.953226|0.096544|1.000000:100][-0.030502|-0.000562|0.027306:100]
.Stacked.NPLSTM state 20 4 1 Seq:[-1.978872|-0.030874|1.210067:80][-0.010696|-0.000545|0.015035:80]
.Stacked.SoftmaxLayer <<<0.0001 0.9 in 20 4 out 20 2>>>
.Stacked.SoftmaxLayer W1 2 5 Bat:[-6.143187|-0.003997|6.134448:10][-0.061927|-0.000000|0.061927:10]
.Stacked.SoftmaxLayer inputs 20 4 1 Seq:[-0.953226|0.038960|0.739090:80][-0.043424|0.000437|0.023548:80]
.Stacked.SoftmaxLayer outputs 20 2 1 Seq:[0.000045|0.500000|0.999955:40][-0.003537|0.000000|0.003537:40]
#: verbose = 0
OK (pre-save) 0.00620409
saving
loading
OK 0.00620409
nparams 106
OK (params) 0.00620409
OK (hacked-params) 0.5
OK (restored-params) 0.00620409
real 0m11.140s
user 0m11.141s
sys 0m0.003s
>>>>>>> ./test-xps.sh
#: ntrain = 400000
#: save_name = xps-200
#: report_time = 1
#: charsep =
got 7372 files, 222 tests
#: load = arabic-beirut-200.clstm
./test-xps.sh: line 20: 1843 Segmentation fault (core dumped) ./clstmocrtrain xps-train-200 xps-test-200
>>>>>>> echo TEST FAILED
TEST FAILED
root@4ef282b20886:/work/clstm-based-japan# cat ./test-xps.sh
#!/bin/bash
set -ea
find work -name '*.bin.png' | sort -r > xps-200-zh-char
sed 1,200d xps-200-zh-char > xps-train-200
sed 1,7350d xps-200-zh-char > xps-test-200
report_every=1
save_every=1000
ntrain=400000
dewarp=center
display_every=100
test_every=100
display_every=100
testset=xps-test.h5
hidden=800
lrate=1e-4
save_name=xps-200
report_time=1:
load=arabic-beirut-200.clstm
# gdb --ex run --args \
./clstmocrtrain xps-train-200 xps-test-200
-
- against arabic-alam-al-kutub.clstm
root@4ef282b20886:/work/clstm-based-japan# ./run-xps-tests
>>>>>>> export seed=0.222
>>>>>>> seed=0.222
>>>>>>> scons -s -c
>>>>>>> rm -f test-lstm.o '*.a'
>>>>>>> scons -j 4 gpu=0 debug=0 options= clstmocrtrain clstmfiltertrain clstmfilter clstmocr test-lstm
scons: Reading SConscript files ...
scons: done reading SConscript files.
scons: Building targets ...
g++ --std=c++11 -Wno-unused-result -o clstmocrtrain.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstmocrtrain.cc
g++ --std=c++11 -Wno-unused-result -o clstm.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm.cc
g++ --std=c++11 -Wno-unused-result -o ctc.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 ctc.cc
protoc(["clstm", "clstm.pb.cc", "clstm.pb.h"], ["clstm.proto"])
g++ --std=c++11 -Wno-unused-result -o clstm_proto.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm_proto.cc
g++ --std=c++11 -Wno-unused-result -o clstm_prefab.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm_prefab.cc
g++ --std=c++11 -Wno-unused-result -o tensor.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 tensor.cc
g++ --std=c++11 -Wno-unused-result -o batches.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 batches.cc
g++ --std=c++11 -Wno-unused-result -o extras.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 extras.cc
g++ --std=c++11 -Wno-unused-result -o clstm.pb.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm.pb.cc
g++ --std=c++11 -Wno-unused-result -o clstm_compute.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm_compute.cc
g++ --std=c++11 -Wno-unused-result -o clstmfiltertrain.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstmfiltertrain.cc
g++ --std=c++11 -Wno-unused-result -o clstmfilter.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstmfilter.cc
g++ --std=c++11 -Wno-unused-result -o clstmocr.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstmocr.cc
g++ --std=c++11 -Wno-unused-result -o test-lstm.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 test-lstm.cc
ar rc libclstm.a clstm.o ctc.o clstm_proto.o clstm_prefab.o tensor.o batches.o extras.o clstm.pb.o clstm_compute.o
ranlib libclstm.a
g++ --std=c++11 -Wno-unused-result -o test-lstm test-lstm.o libclstm.a -lpng -lprotobuf
g++ --std=c++11 -Wno-unused-result -o clstmocr clstmocr.o libclstm.a -lpng -lprotobuf
g++ --std=c++11 -Wno-unused-result -o clstmocrtrain clstmocrtrain.o libclstm.a -lpng -lprotobuf
g++ --std=c++11 -Wno-unused-result -o clstmfiltertrain clstmfiltertrain.o libclstm.a -lpng -lprotobuf
g++ --std=c++11 -Wno-unused-result -o clstmfilter clstmfilter.o libclstm.a -lpng -lprotobuf
scons: done building targets.
>>>>>>> ./test-lstm
#: ntrain = 100000
#: ntest = 1000
#: gpu = -1
training 1:4:2 network to learn delay
.Stacked <<<0.0001 0.9 in 20 1 out 20 2>>>
.Stacked inputs 20 1 1 Seq:[0.000000|0.400000|1.000000:20][-0.030502|-0.001325|0.025830:20]
.Stacked outputs 20 2 1 Seq:[0.000045|0.500000|0.999955:40][-0.003537|0.000000|0.003537:40]
.Stacked.NPLSTM <<<0.0001 0.9 in 20 1 out 20 4>>>
.Stacked.NPLSTM WCI 4 6 Bat:[-2.371926|-0.180369|1.806145:24][-0.042775|0.002258|0.046467:24]
.Stacked.NPLSTM WGF 4 6 Bat:[-1.157534|0.080894|1.083409:24][-0.006257|0.002911|0.012813:24]
.Stacked.NPLSTM WGI 4 6 Bat:[-0.212294|0.408031|2.245568:24][-0.002479|0.003996|0.014401:24]
.Stacked.NPLSTM WGO 4 6 Bat:[-2.919670|0.557016|3.321400:24][-0.018948|0.007477|0.029906:24]
.Stacked.NPLSTM inputs 20 1 1 Seq:[0.000000|0.400000|1.000000:20][-0.030502|-0.001325|0.025830:20]
.Stacked.NPLSTM outputs 20 4 1 Seq:[-0.953226|0.038960|0.739090:80][-0.043424|0.000437|0.023548:80]
.Stacked.NPLSTM ci 20 4 1 Seq:[-0.999453|-0.009550|0.961935:80][-0.009550|-0.000172|0.014207:80]
.Stacked.NPLSTM gf 20 4 1 Seq:[0.103049|0.455415|0.836064:80][-0.001099|-0.000045|0.001491:80]
.Stacked.NPLSTM gi 20 4 1 Seq:[0.691834|0.832398|0.977853:80][-0.000271|0.000066|0.000974:80]
.Stacked.NPLSTM go 20 4 1 Seq:[0.063462|0.769145|0.996185:80][-0.001031|0.000088|0.004340:80]
.Stacked.NPLSTM source 20 5 1 Seq:[-0.953226|0.096544|1.000000:100][-0.030502|-0.000562|0.027306:100]
.Stacked.NPLSTM state 20 4 1 Seq:[-1.978872|-0.030874|1.210067:80][-0.010696|-0.000545|0.015035:80]
.Stacked.SoftmaxLayer <<<0.0001 0.9 in 20 4 out 20 2>>>
.Stacked.SoftmaxLayer W1 2 5 Bat:[-6.143187|-0.003997|6.134448:10][-0.061927|-0.000000|0.061927:10]
.Stacked.SoftmaxLayer inputs 20 4 1 Seq:[-0.953226|0.038960|0.739090:80][-0.043424|0.000437|0.023548:80]
.Stacked.SoftmaxLayer outputs 20 2 1 Seq:[0.000045|0.500000|0.999955:40][-0.003537|0.000000|0.003537:40]
#: verbose = 0
OK (pre-save) 0.00620409
saving
loading
OK 0.00620409
nparams 106
OK (params) 0.00620409
OK (hacked-params) 0.5
OK (restored-params) 0.00620409
real 0m10.910s
user 0m10.910s
sys 0m0.004s
>>>>>>> ./test-xps.sh
#: ntrain = 400000
#: save_name = xps-200
#: report_time = 1
#: charsep =
got 7372 files, 222 tests
#: load = arabic-alam-al-kutub.clstm
./test-xps.sh: line 20: 1956 Segmentation fault (core dumped) ./clstmocrtrain xps-train-200 xps-test-200
>>>>>>> echo TEST FAILED
TEST FAILED
-
- against
from awesome-ocr.
wanghaisheng
commented on June 2, 2024
test japanese models
git clone https://github.com/isaomatsunami/clstm-Japanese clstm-Japanese-model
/run-xps-tests-third-japanese-input
root@4ef282b20886:/work/clstm-for-japan# cat ./run-xps-tests-third-japanese-input
#!/bin/bash
##2016-06-03 chongxian riben ren tidao de diedai 15 wan ci shoulian
set -e
debug=${debug:-0}
options=${options:-}
export PS4='
>>>>>>> '
trap "echo TEST FAILED" EXIT
set -x
export seed=0.222
scons -s -c; rm -f *.o *.a
scons -j 4 gpu=0 debug=$debug options="$options" clstmocrtrain clstmfiltertrain clstmfilter clstmocr test-lstm
time ./test-lstm
time ./test-japanese.sh
#time ./test-filter.sh
#time ./test-ocr.sh
scons -s -c; rm -f *.o *.a
scons -j 4 gpu=0 double=1 debug=$debug options="$options" test-cderiv test-deriv test-ctc
#./test-cderiv
#./test-deriv
./test-japanese.sh
rm -f *.pb.h *.pb.cc
scons -c all
#scons -s -c pyswig
#scons pyswig
#python test-lstm.py
#set +x
#scons -s -c all pyswig
trap "echo ALL TESTS PASS" EXIT
./test-japanese.sh
#!/bin/bash
set -ea
find ../clstm-Japanese-model/testdata3877/mincho/ -name '*.bin.png' | sort -r > japanese-3877-char
sed 1,0d japanese-3877-char > japanese-train-3877
sed 1,4950d japanese-3877-char > japanese-test-3877
report_every=1
save_every=1000
ntrain=400000
dewarp=center
display_every=100
test_every=100
display_every=100
testset=japanese-test.h5
hidden=800
lrate=1e-4
save_name=japanese-3877
report_time=1
:
#load=japanese-3877-144000.clstm
load=jp3000-180000.clstm
# gdb --ex run --args \
./clstmocrtrain japanese-train-3877 japanese-test-3877
logs
root@4ef282b20886:/work/clstm-for-japan# ./run-xps-tests-third-japanese-input
>>>>>>> export seed=0.222
>>>>>>> seed=0.222
>>>>>>> scons -s -c
>>>>>>> rm -f test-lstm.o '*.a'
>>>>>>> scons -j 4 gpu=0 debug=0 options= clstmocrtrain clstmfiltertrain clstmfilter clstmocr test-lstm
scons: Reading SConscript files ...
scons: done reading SConscript files.
scons: Building targets ...
g++ --std=c++11 -Wno-unused-result -o clstmocrtrain.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstmocrtrain.cc
g++ --std=c++11 -Wno-unused-result -o clstm.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm.cc
g++ --std=c++11 -Wno-unused-result -o ctc.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 ctc.cc
protoc(["clstm", "clstm.pb.cc", "clstm.pb.h"], ["clstm.proto"])
g++ --std=c++11 -Wno-unused-result -o clstm_proto.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm_proto.cc
g++ --std=c++11 -Wno-unused-result -o clstm_prefab.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm_prefab.cc
g++ --std=c++11 -Wno-unused-result -o tensor.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 tensor.cc
g++ --std=c++11 -Wno-unused-result -o batches.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 batches.cc
g++ --std=c++11 -Wno-unused-result -o extras.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 extras.cc
g++ --std=c++11 -Wno-unused-result -o clstm.pb.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm.pb.cc
g++ --std=c++11 -Wno-unused-result -o clstm_compute.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstm_compute.cc
g++ --std=c++11 -Wno-unused-result -o clstmfiltertrain.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstmfiltertrain.cc
g++ --std=c++11 -Wno-unused-result -o clstmfilter.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstmfilter.cc
g++ --std=c++11 -Wno-unused-result -o clstmocr.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 clstmocr.cc
g++ --std=c++11 -Wno-unused-result -o test-lstm.o -c -g -O3 -DEIGEN_NO_DEBUG -DCATCH=catch -DCLSTM_ALL_TENSOR=1 -DNODISPLAY=1 -DTHROW=throw -DTRY=try -I/usr/local/include/eigen3 test-lstm.cc
ar rc libclstm.a clstm.o ctc.o clstm_proto.o clstm_prefab.o tensor.o batches.o extras.o clstm.pb.o clstm_compute.o
ranlib libclstm.a
g++ --std=c++11 -Wno-unused-result -o clstmocrtrain clstmocrtrain.o libclstm.a -lpng -lprotobuf
g++ --std=c++11 -Wno-unused-result -o clstmfiltertrain clstmfiltertrain.o libclstm.a -lpng -lprotobuf
g++ --std=c++11 -Wno-unused-result -o clstmfilter clstmfilter.o libclstm.a -lpng -lprotobuf
g++ --std=c++11 -Wno-unused-result -o test-lstm test-lstm.o libclstm.a -lpng -lprotobuf
g++ --std=c++11 -Wno-unused-result -o clstmocr clstmocr.o libclstm.a -lpng -lprotobuf
scons: done building targets.
>>>>>>> ./test-lstm
#: ntrain = 100000
#: ntest = 1000
#: gpu = -1
training 1:4:2 network to learn delay
.Stacked <<<0.0001 0.9 in 20 1 out 20 2>>>
.Stacked inputs 20 1 1 Seq:[0.000000|0.400000|1.000000:20][-0.030502|-0.001325|0.025830:20]
.Stacked outputs 20 2 1 Seq:[0.000045|0.500000|0.999955:40][-0.003537|0.000000|0.003537:40]
.Stacked.NPLSTM <<<0.0001 0.9 in 20 1 out 20 4>>>
.Stacked.NPLSTM WCI 4 6 Bat:[-2.371926|-0.180369|1.806145:24][-0.042775|0.002258|0.046467:24]
.Stacked.NPLSTM WGF 4 6 Bat:[-1.157534|0.080894|1.083409:24][-0.006257|0.002911|0.012813:24]
.Stacked.NPLSTM WGI 4 6 Bat:[-0.212294|0.408031|2.245568:24][-0.002479|0.003996|0.014401:24]
.Stacked.NPLSTM WGO 4 6 Bat:[-2.919670|0.557016|3.321400:24][-0.018948|0.007477|0.029906:24]
.Stacked.NPLSTM inputs 20 1 1 Seq:[0.000000|0.400000|1.000000:20][-0.030502|-0.001325|0.025830:20]
.Stacked.NPLSTM outputs 20 4 1 Seq:[-0.953226|0.038960|0.739090:80][-0.043424|0.000437|0.023548:80]
.Stacked.NPLSTM ci 20 4 1 Seq:[-0.999453|-0.009550|0.961935:80][-0.009550|-0.000172|0.014207:80]
.Stacked.NPLSTM gf 20 4 1 Seq:[0.103049|0.455415|0.836064:80][-0.001099|-0.000045|0.001491:80]
.Stacked.NPLSTM gi 20 4 1 Seq:[0.691834|0.832398|0.977853:80][-0.000271|0.000066|0.000974:80]
.Stacked.NPLSTM go 20 4 1 Seq:[0.063462|0.769145|0.996185:80][-0.001031|0.000088|0.004340:80]
.Stacked.NPLSTM source 20 5 1 Seq:[-0.953226|0.096544|1.000000:100][-0.030502|-0.000562|0.027306:100]
.Stacked.NPLSTM state 20 4 1 Seq:[-1.978872|-0.030874|1.210067:80][-0.010696|-0.000545|0.015035:80]
.Stacked.SoftmaxLayer <<<0.0001 0.9 in 20 4 out 20 2>>>
.Stacked.SoftmaxLayer W1 2 5 Bat:[-6.143187|-0.003997|6.134448:10][-0.061927|-0.000000|0.061927:10]
.Stacked.SoftmaxLayer inputs 20 4 1 Seq:[-0.953226|0.038960|0.739090:80][-0.043424|0.000437|0.023548:80]
.Stacked.SoftmaxLayer outputs 20 2 1 Seq:[0.000045|0.500000|0.999955:40][-0.003537|0.000000|0.003537:40]
#: verbose = 0
OK (pre-save) 0.00620409
saving
loading
OK 0.00620409
nparams 106
OK (params) 0.00620409
OK (hacked-params) 0.5
OK (restored-params) 0.00620409
real 0m10.997s
user 0m10.997s
sys 0m0.003s
>>>>>>> ./test-japanese.sh
#: ntrain = 400000
#: save_name = japanese-3877
#: report_time = 1
#: charsep =
got 4999 files, 50 tests
#: load = jp3000-180000.clstm
[libprotobuf WARNING google/protobuf/io/coded_stream.cc:537] Reading dangerously large protocol message. If the message turns out to be larger than 67108864 bytes, parsing will be halted for security reasons. To increase the limit (or to disable these warnings), see CodedInputStream::SetTotalBytesLimit() in google/protobuf/io/coded_stream.h.
[libprotobuf WARNING google/protobuf/io/coded_stream.cc:78] The total number of bytes read was 58218838
.Stacked: 0.0001 0.9 in 0 48 out 0 3877
.Stacked.Parallel: 0.0001 0.9 in 0 48 out 0 1600
.Stacked.Parallel.NPLSTM: 0.0001 0.9 in 0 48 out 0 800
.Stacked.Parallel.Reversed: 0.0001 0.9 in 0 48 out 0 800
.Stacked.Parallel.Reversed.NPLSTM: 0.0001 0.9 in 0 48 out 0 800
.Stacked.SoftmaxLayer: 0.0001 0.9 in 0 1600 out 0 3877
#: start = -1
start 180001
#: test_every = 100
#: save_every = 1000
#: report_every = 1
#: display_every = 100
180001
TRU 莞援狸卒臺セ、藻屯俄獅貧廷暉炊滞補旺栗ヤ想熱臓雅零哉微唯軽賀
ALN 莞援狸卒臺セ、藻屯俄獅貧廷暉炊滞補旺栗ヤ想熱臓雅零哉微唯軽賀
OUT 莞援狸卒臺セ、藻屯俄獅貧廷暉炊滞補旺栗ヤ想熱臓雅零哉微唯軽賀
from awesome-ocr.
wanghaisheng
commented on June 2, 2024
第一个模型无法加载 出现segmentation error 初步猜测是由于训练数据中每行出现的字符个数与待加载模型中数量对不上
tmbdev/clstm#94
这里提到pillow版本冲突的问题
The segfault was due to a compatibility problem with older versions of Pillow, Jessie uses 2.6.1 while I used 3.4.2 for developing. 2.9.0 added the width and height attributes, which I used to differentiate between Pillow.Image and numpy.ndarray in the image loading logic. Since images loaded with 2.6.1 did not have either of these attributes, they were interpreted as numpy arrays. Funnily, all the interfaces on Pillow.Image I accessed during image loading were also present on numpy.ndarray, but returned different things, which led to segfaults pretty deep into the stack.
另外就是模型训练时间的问题
Sorry for spamming but there's one major reason for using the lower level interface. By preloading the entire training set into memory and doing all the normalization, encoding, etc. once, I've just now decreased training time by ~2/3. While I'm fairly sure the main reason is just having everything in main memory rerunning the codec and line normalization over and over again seems needlessly wasteful.
from awesome-ocr.
xray1111
commented on June 2, 2024
你是否已经开始训练中文的识别模型了呢？我打算开始训练中文，有没有什么建议呢？
from awesome-ocr.
wanghaisheng
commented on June 2, 2024
@xray1111 可以先造几百个字符的训练数据进行测试
from awesome-ocr.
xray1111
commented on June 2, 2024
我下载了一篇TXT的小说，用ocropus-linegen生成了几百张图片，每个图片里面是一个短句子，然后开始训练了。感觉训练速度比英文慢很多，而且到目前为止还没有看到比较长的连贯输出。我现在用的参数就只把隐层数改为300，其他没动。
from awesome-ocr.
wanghaisheng
commented on June 2, 2024
@xray1111 和你思路一致 我使用总数200-300个汉字 大概训练1万5 就可以收敛 你这个估计很难收敛 怕是要耗费很长时间
参数这里你参考下 https://github.com/isaomatsunami/clstm-Japanese/issues
nhidden=800, lrate=1e-4
178000 iterations, 2.23% error rate for first 100 samples (which are not used for training)
使用它提供的训练数据 我没有重现出来 它的结果
日志和具体参数你可以看下这里
https://github.com/wanghaisheng/awesome-ocr/wiki/重现日本人的模型训练
After 50 days of training on my MacBookPro, 130000 iterations, against 3877 classes of characters, the clstm has achieved 3.6% error rate so far... The hidden nodes is 800, The trained data is 58.2MB.
from awesome-ocr.
xray1111
commented on June 2, 2024
不好意思刚刚看到你的回复。
是的，我已经调整了模型和语料，增加了隐层单元数，现在效果要好很多。
我在想，他这个模型似乎只有一层隐层单元，能否增加多层呢？
2016-11-06 13:55 GMT+08:00 wanghaisheng [email protected]:
@xray1111 https://github.com/xray1111 和你思路一致 我使用总数200-300个汉字 大概训练1万5
就可以收敛 你这个估计很难收敛 怕是要耗费很长时间—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub
#14 (comment),
or mute the thread
https://github.com/notifications/unsubscribe-auth/AMx26JiPsXt4HIO2TM-pemHVEFdrypZLks5q7WvHgaJpZM4KRpZp
.
from awesome-ocr.
wanghaisheng
commented on June 2, 2024
@xray1111 效果好很多 你的很快收敛了么 分享下
作者说这个库他已经停更了 全力在研发另外一个 估计是基于tensorflow在做
隐层肯定是能加的 我自己没有试
from awesome-ocr.
xray1111
commented on June 2, 2024
没有很快收敛，应该也需要到１万次以上才行，而且我发现到了４万次左右会振荡，就是误差会放大很多，继续训练会慢慢降下去，但后面改善得不多
在 2016年11月17日 下午9:35，wanghaisheng [email protected]写道：
@xray1111 https://github.com/xray1111 效果好很多 你的很快收敛了么 分享下
作者说这个库他已经停更了 全力在研发另外一个 估计是基于tensorflow在做
隐层肯定是能加的 我自己没有试—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub
#14 (comment),
or mute the thread
https://github.com/notifications/unsubscribe-auth/AMx26JQGfrOTLIP7W9UJGPhiEhxtsuRkks5q_FgzgaJpZM4KRpZp
.
from awesome-ocr.
wanghaisheng
commented on June 2, 2024
你有统计过你的txt里总共有多少个汉字么 去重后 大概多少张图片(一行文本一张的话) 1万次-4万次大概的错误率是多少呢
from awesome-ocr.
xray1111
commented on June 2, 2024
我现在用了一个700多行的txt文件训练，共包含300个左右的中英文字符，生成了3000张图片（用３种字体，每种1000张图片）。你说的错误率是怎么统计的？
2016-11-17 22:01 GMT+08:00 wanghaisheng [email protected]:
你有统计过你的txt里总共有多少个汉字么 去重后 大概多少张图片(一行文本一张的话) 1万次-4万次大概的错误率是多少呢
—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub
#14 (comment),
or mute the thread
https://github.com/notifications/unsubscribe-auth/AMx26BTKy3M1bRQlPF5SooOwPhUeljHoks5q_F5JgaJpZM4KRpZp
.
from awesome-ocr.
wanghaisheng
commented on June 2, 2024
那是差不多 和我测试时的数据 1.5w基本上就收敛了
错误率在训练的日志里有 你留意一下
from awesome-ocr.
xray1111
commented on June 2, 2024
现在主要问题是训练时间太长了，我在笔记本上做的，cpu比较老，双核的。不知道你有没有用CLSTM做过？效果比python写的程序怎么样？另外我没有找到训练日志文件，我用ocropus-rtrain脚本训练的，生成的文件只有每1000次的一个新的model文件。
2016-11-17 22:32 GMT+08:00 wanghaisheng [email protected]:
那是差不多 和我测试时的数据 1.5w基本上就收敛了
错误率在训练的日志里有 你留意一下—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub
#14 (comment),
or mute the thread
https://github.com/notifications/unsubscribe-auth/AMx26CfCvm5WK6LcuKA31hYizg0wbLSLks5q_GWJgaJpZM4KRpZp
.
from awesome-ocr.
wanghaisheng
commented on June 2, 2024
我刚才跟你说的就是clstm 不是ocropy 训练数据我也是用ocropy生成的
时间这个问题我也没有解决 一直是单线程
from awesome-ocr.
xray1111
commented on June 2, 2024
好的，多谢！我把ocropy搞熟悉了以后会去试一下CLSTM，现在还在研究他里面的算法。
2016-11-17 22:59 GMT+08:00 wanghaisheng [email protected]:
我刚才跟你说的就是clstm 不是ocropy 训练数据我也是用ocropy生成的
时间这个问题我也没有解决 一直是单线程—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub
#14 (comment),
or mute the thread
https://github.com/notifications/unsubscribe-auth/AMx26DJ0hMJjdgr7TVbVd5xBnj75RRhPks5q_GvYgaJpZM4KRpZp
.
from awesome-ocr.
wanghaisheng
commented on June 2, 2024
@xray1111 建议直接上手clstm吧 然后坐等作者出新库
学生么？
from awesome-ocr.
xray1111
commented on June 2, 2024
我已经工作了，目前通过ocropy熟悉算法部分。
CLSTM与原来的程序比较，有哪些差异，除了实现用C++以外？python的貌似没有用其他库，全是自己实现的，不知道CLSTM有没有和其他第三方工具结合，在并行和GPU上能充分利用硬件资源？
2016-11-18 0:03 GMT+08:00 wanghaisheng [email protected]:
@xray1111 https://github.com/xray1111 建议直接上手clstm吧 然后坐等作者出新库
学生么？—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub
#14 (comment),
or mute the thread
https://github.com/notifications/unsubscribe-auth/AMx26M4FIMJl0aiz9oogy8jCHEeg6xXHks5q_Hq8gaJpZM4KRpZp
.
from awesome-ocr.
wanghaisheng
commented on June 2, 2024
算法实现我没有研究 第二个问题 据我所知 暂时没有方案
from awesome-ocr.
Related Issues (20)
- OCR basics HOT 1
- EAST:An Efficient and Accurate Scene Text Detector HOT 1
- Robust, Simple Page Segmentation using Hybrid Convolutional MDLSTM Networks
- PixelLink: Detecting Scene Text via Instance Segmentation
- Table-to-Text: Describing Table Region with Natural Language
- lable tools
- how to modify the connectionist Temporal Classification (CTC) layer of the network to also give us a confidence score? HOT 2
- Confidence Prediction for Lexicon-Free OCR HOT 1
- 工业制造——Workplace of automated control of vibration output circular trays HOT 3
- Tesseract for R HOT 1
- Cut, Paste and Learn: Surprisingly Easy Synthesis for Instance Detection
- 【Rosetta：大规模图像文字检测识别系统】《Rosetta: Large scale system for text detection and recognition in images》[Facebook] (2018) O HOT 4
- Radical analysis network for zero-shot learning in printed Chinese character recognition HOT 3
- DenseRAN for Offline Handwritten Chinese Character Recognition HOT 3
- Deep TextSpotter: An End-to-End Trainable Scene Text Localization and Recognition Framework
- in marmot data set the table BBOX are not matching with original images
- dhSegment: A generic deep-learning approach for document segmentation
- null
- 2018年末撸串计划 HOT 5
- 希望可以增加PaddleOCR、AgentOCR HOT 3
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from awesome-ocr.