yanboliang / spark-vlbfgs Goto Github PK

Could you add a license please?

having issue when running example on a9a data

bin/spark-shell --master 'local[7]' --jars target/spark-vlbfgs-0.1-SNAPSHOT.jar --driver-memory 10g --executor-memory 10g

scala> import org.apache.spark.sql.Dataset
import org.apache.spark.sql.Dataset

scala> import org.apache.spark.ml.classification.VLogisticRegression
import org.apache.spark.ml.classification.VLogisticRegression

scala> val dataset: Dataset[] = spark.read.format("libsvm").load("data/a9a")
dataset: org.apache.spark.sql.Dataset[] = [label: double, features: vector]

scala> val trainer = new VLogisticRegression().setColsPerBlock(100).setRowsPerBlock(10).setColPartitions(3).setRowPartitions(3).setRegParam(0.5)
trainer: org.apache.spark.ml.classification.VLogisticRegression = vector-free-logreg_53218448522f

scala> val model = trainer.fit(dataset)
18/06/02 16:24:04 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS
18/06/02 16:24:04 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS
18/06/02 16:24:14 ERROR StrongWolfeLineSearch: Encountered bad values in function evaluation. Decreasing step size to 0.041010631451266606
18/06/02 16:24:22 ERROR StrongWolfeLineSearch: Encountered bad values in function evaluation. Decreasing step size to 0.020505315725633303
18/06/02 16:24:31 ERROR StrongWolfeLineSearch: Encountered bad values in function evaluation. Decreasing step size to 0.010252657862816652
18/06/02 16:24:40 ERROR StrongWolfeLineSearch: Encountered bad values in function evaluation. Decreasing step size to 0.005126328931408326
18/06/02 16:24:46 ERROR StrongWolfeLineSearch: Encountered bad values in function evaluation. Decreasing step size to 0.002563164465704163
18/06/02 16:24:53 ERROR StrongWolfeLineSearch: Encountered bad values in function evaluation. Decreasing step size to 0.0012815822328520814
18/06/02 16:25:00 ERROR StrongWolfeLineSearch: Encountered bad values in function evaluation. Decreasing step size to 6.407911164260407E-4
18/06/02 16:25:07 ERROR StrongWolfeLineSearch: Encountered bad values in function evaluation. Decreasing step size to 3.2039555821302036E-4
18/06/02 16:25:15 ERROR StrongWolfeLineSearch: Encountered bad values in function evaluation. Decreasing step size to 1.6019777910651018E-4
18/06/02 16:25:22 ERROR StrongWolfeLineSearch: Encountered bad values in function evaluation. Decreasing step size to 8.009888955325509E-5
18/06/02 16:25:22 ERROR VLBFGS: LBFGS search failed: breeze.optimize.FirstOrderException: Line search failed
breeze.optimize.FirstOrderException: Line search failed
at breeze.optimize.StrongWolfeLineSearch.minimize(StrongWolfe.scala:179)
at org.apache.spark.ml.optim.VLBFGS.determineAndTakeStepSize(VLBFGS.scala:80)
at org.apache.spark.ml.optim.VLBFGS$$anonfun$8.apply(VLBFGS.scala:173)
at org.apache.spark.ml.optim.VLBFGS$$anonfun$8.apply(VLBFGS.scala:169)
at scala.collection.Iterator$$anon$7.next(Iterator.scala:129)
at breeze.util.IteratorImplicits$RichIterator$$anon$2.next(Implicits.scala:71)
at org.apache.spark.ml.classification.VLogisticRegression.train(VLogisticRegression.scala:382)
at org.apache.spark.ml.classification.VLogisticRegression.train(VLogisticRegression.scala:51)
at org.apache.spark.ml.Predictor.fit(Predictor.scala:90)
at $line18.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:29)
at $line18.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:34)
at $line18.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:36)
at $line18.$read$$iw$$iw$$iw$$iw$$iw.(:38)
at $line18.$read$$iw$$iw$$iw$$iw.(:40)
at $line18.$read$$iw$$iw$$iw.(:42)
at $line18.$read$$iw$$iw.(:44)
at $line18.$read$$iw.(:46)
at $line18.$read.(:48)
at $line18.$read$.(:52)
at $line18.$read$.()
at $line18.$eval$.$print$lzycompute(:7)
at $line18.$eval$.$print(:6)
at $line18.$eval.$print()
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
at scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
at scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415)
at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923)
at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
at scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
at org.apache.spark.repl.Main$.doMain(Main.scala:68)
at org.apache.spark.repl.Main$.main(Main.scala:51)
at org.apache.spark.repl.Main.main(Main.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:729)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:185)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:210)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:124)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
18/06/02 16:25:22 ERROR VLBFGS: Vector-Free LBFGS search failed.
java.lang.AssertionError: assertion failed
at scala.Predef$.assert(Predef.scala:156)
at org.apache.spark.ml.classification.VLogisticRegression.train(VLogisticRegression.scala:394)
at org.apache.spark.ml.classification.VLogisticRegression.train(VLogisticRegression.scala:51)
at org.apache.spark.ml.Predictor.fit(Predictor.scala:90)
... 48 elided

scala>

scala> println(s"Vector-free logistic regression coefficients: ${model.coefficients}")
:26: error: not found: value model
println(s"Vector-free logistic regression coefficients: ${model.coefficients}")

Hey,
Does this implementation support incremental learning where I can set previous modelpath or initial weights ?

Regards
Sundeep