Return of org.eclipse.jgit.errors.RevWalkException: Walk failure about jgit-spark-connector

[r0mainK]

Seems I also got this when running my job on hdfs://hdfs-namenode/siva/latest/0b, although it killed the job in this time (about 8min), log is similar:

[Stage 0:=====================================================>   (82 + 6) / 88]18/04/23 09:40:23 WARN TaskSetManager: Lost task 35.0 in stage 0.0 (TID 36, 10.2.13.51, executor 13): tech.sourced.engine.iterator.RepositoryException: Repository error with data: Repository[/spark-temp-data/spark-cba44e40-3ea0-475d-b20e-dcdbc58dedbe/processing-repositories/D9DD063AE1D1DBA252A1331AD729FBEB/0b666f81da14bf46cada222856762f7fd6641c26.siva]; urls https://github.com/linux-sunxi/u-boot-sunxi, https://github.com/NextThingCo/CHIP-u-boot, https://github.com/RobertCNelson/u-boot, https://github.com/gonzoua/u-boot-pi, https://github.com/Xilinx/u-boot-xlnx, https://github.com/hardkernel/u-boot
Caused by: org.eclipse.jgit.errors.MissingObjectException: Missing tree d42fbb6d72e8e2dc6082765dfe983631dd8f212e
	at org.eclipse.jgit.internal.storage.file.WindowCursor.open(WindowCursor.java:164)
	at org.eclipse.jgit.treewalk.CanonicalTreeParser.reset(CanonicalTreeParser.java:214)
	at org.eclipse.jgit.treewalk.TreeWalk.parserFor(TreeWalk.java:1347)
	at org.eclipse.jgit.treewalk.TreeWalk.addTree(TreeWalk.java:741)
	at tech.sourced.engine.iterator.GitTreeEntryIterator$.tech$sourced$engine$iterator$GitTreeEntryIterator$$getTreeEntries(GitTreeEntryIterator.scala:146)
	at tech.sourced.engine.iterator.GitTreeEntryIterator$$anonfun$1.apply(GitTreeEntryIterator.scala:124)
	at tech.sourced.engine.iterator.GitTreeEntryIterator$$anonfun$1.apply(GitTreeEntryIterator.scala:124)
	at scala.collection.Iterator$$anon$12.nextCur(Iterator.scala:434)
	at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:440)
	at tech.sourced.engine.iterator.ChainableIterator.hasNext(ChainableIterator.scala:91)
	at scala.collection.Iterator$class.isEmpty(Iterator.scala:330)
	at tech.sourced.engine.iterator.ChainableIterator.isEmpty(ChainableIterator.scala:17)
	at tech.sourced.engine.iterator.ChainableIterator.hasNext(ChainableIterator.scala:92)
	at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
	at tech.sourced.engine.iterator.CleanupIterator.hasNext(CleanupIterator.scala:23)
	at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:438)
	at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator.sort_addToSorter$(Unknown Source)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$8$$anon$1.hasNext(WholeStageCodegenExec.scala:395)
	at org.apache.spark.sql.execution.aggregate.SortAggregateExec$$anonfun$doExecute$1$$anonfun$3.apply(SortAggregateExec.scala:80)
	at org.apache.spark.sql.execution.aggregate.SortAggregateExec$$anonfun$doExecute$1$$anonfun$3.apply(SortAggregateExec.scala:77)
	at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:827)
	at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:827)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)
	at org.apache.spark.scheduler.Task.run(Task.scala:108)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:335)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

[Stage 0:=====================================================>   (83 + 5) / 88] 18/04/23 09:41:43 WARN TaskSetManager: Lost task 35.1 in stage 0.0 (TID 88, 10.2.7.90, executor 2): org.eclipse.jgit.errors.RevWalkException: Walk failure.
	at org.eclipse.jgit.revwalk.RevWalk$3.next(RevWalk.java:1353)
	at org.eclipse.jgit.revwalk.RevWalk$3.next(RevWalk.java:1338)
	at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:43)
	at scala.collection.Iterator$$anon$10.next(Iterator.scala:393)
	at tech.sourced.engine.iterator.RefWithCommitIterator.next(CommitIterator.scala:161)
	at tech.sourced.engine.iterator.RefWithCommitIterator.next(CommitIterator.scala:123)
	at tech.sourced.engine.iterator.ChainableIterator.nextRaw(ChainableIterator.scala:120)
	at tech.sourced.engine.iterator.ChainableIterator.hasNext(ChainableIterator.scala:94)
	at scala.collection.Iterator$class.isEmpty(Iterator.scala:330)
	at tech.sourced.engine.iterator.ChainableIterator.isEmpty(ChainableIterator.scala:16)
	at tech.sourced.engine.iterator.ChainableIterator.hasNext(ChainableIterator.scala:90)
	at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
	at tech.sourced.engine.iterator.CleanupIterator.hasNext(CleanupIterator.scala:23)
	at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:438)
	at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator.sort_addToSorter$(Unknown Source)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$8$$anon$1.hasNext(WholeStageCodegenExec.scala:395)
	at org.apache.spark.sql.execution.aggregate.SortAggregateExec$$anonfun$doExecute$1$$anonfun$3.apply(SortAggregateExec.scala:80)
	at org.apache.spark.sql.execution.aggregate.SortAggregateExec$$anonfun$doExecute$1$$anonfun$3.apply(SortAggregateExec.scala:77)
	at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:827)
	at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:827)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)
	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)
	at org.apache.spark.scheduler.Task.run(Task.scala:108)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:335)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.eclipse.jgit.errors.MissingObjectException: Missing commit 6528ff0109d81c1f21d20f9f1370782bccf87bcb
	at org.eclipse.jgit.internal.storage.file.WindowCursor.open(WindowCursor.java:164)
	at org.eclipse.jgit.revwalk.RevWalk.getCachedBytes(RevWalk.java:903)
	at org.eclipse.jgit.revwalk.RevCommit.parseHeaders(RevCommit.java:155)
	at org.eclipse.jgit.revwalk.PendingGenerator.next(PendingGenerator.java:147)
	at org.eclipse.jgit.revwalk.RevWalk.next(RevWalk.java:435)
	at org.eclipse.jgit.revwalk.RevWalk$3.next(RevWalk.java:1350)
	... 35 more

[r0mainK]

Seems to be solved with engine 0.6.1, closing