aserrallerios / kcl-akka-stream Goto Github PK

For a split or merge operation, the KCL won't start processing the new shards until the processors for the original shards have called checkpoint to signal that all processing on the original shards is complete.

Sep 27 20:48:38 ip-172-31-81-83 java[12490]: 20:48:38.826 ERROR [RecordProcessor-0057] c.a.s.k.c.lib.worker.ShutdownTask - Application exception.
Sep 27 20:48:38 ip-172-31-81-83 java[12490]: java.lang.IllegalArgumentException: Application didn't checkpoint at end of shard shardId-000000000130

    // Additional processor to be used by a separate shard worker
    var recordProcessor2: v2.IRecordProcessor = _
    ...
    recordProcessor2 = x.createProcessor()

"KinesisWorker Source" must {
    "not drop messages in case of back-pressure with multiple shard workers" in new KinesisWorkerContext
                                                        with TestData {
      recordProcessor.initialize(initializationInput)
      recordProcessor2.initialize(initializationInput.withShardId("shard2"))

      for (i <- 1 to 5) { // 10 is a buffer size
        val record = org.mockito.Mockito.mock(classOf[Record])
        when(record.getSequenceNumber).thenReturn(i.toString)
        recordProcessor.processRecords(recordsInput.withRecords(List(record).asJava))
        recordProcessor2.processRecords(recordsInput.withRecords(List(record).asJava))
      }

      //expect to consume all 10 across both shards
      for (_ <- 1 to 10) sinkProbe.requestNext()

      // Each shard is assigned its own worker thread, so we get messages
      // from each thread simultaneously.
      def simulateWorkerThread(rp: v2.IRecordProcessor): Future[Unit] = {
        Future {
          for (i <- 1 to 25) { // 10 is a buffer size
            val record = org.mockito.Mockito.mock(classOf[Record])
            when(record.getSequenceNumber).thenReturn(i.toString)
            rp.processRecords(recordsInput.withRecords(List(record).asJava))
          }
        }
      }

      //send another batch to exceed the queue size - this is shard 1
      simulateWorkerThread(recordProcessor)

      //send another batch to exceed the queue size - this is shard 2
      simulateWorkerThread(recordProcessor2)

      //expect to consume all 25 with slow consumer
      for (_ <- 1 to 25) {
        sinkProbe.requestNext()
        Thread.sleep(100)
      }

      killSwitch.shutdown()
      sinkProbe.expectComplete()
    }
}

java.lang.IllegalStateException: You have to wait for previous offer to be resolved to send another request