dlshade.org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion
at dlshade.org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.util.Utils$5.processResult(Utils.java:395) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
at dlshade.org.apache.bookkeeper.zookeeper.ZooKeeperClient$22$1.processResult(ZooKeeperClient.java:1091) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
at dlshade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:566) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
at dlshade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:508) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
14:02:54.222 [local-worker-1-1] WARN - Write error on message
java.util.concurrent.CompletionException: org.apache.distributedlog.exceptions.WriteException: Write rejected because stream test-stream-Ir8h5B8-0000 has encountered an error : writer has been closed due to error. at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:331) ~[?:?]
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:346) ~[?:?]
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:870) [?:?]
at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:883) [?:?]
at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2251) [?:?]
at dlshade.org.apache.bookkeeper.common.concurrent.FutureUtils.stats(FutureUtils.java:365) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.BKAsyncLogWriter.write(BKAsyncLogWriter.java:415) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at io.openmessaging.benchmark.driver.bookkeeper.DlogBenchmarkProducer.sendAsync(DlogBenchmarkProducer.java:50) ~[driver-bookkeeper-0.0.1-SNAPSHOT.jar:?]
at io.openmessaging.benchmark.worker.LocalWorker.lambda$submitProducersToExecutor$10(LocalWorker.java:219) [classes/:?]
at java.util.ArrayList.forEach(ArrayList.java:1540) [?:?]
at io.openmessaging.benchmark.worker.LocalWorker.lambda$submitProducersToExecutor$11(LocalWorker.java:216) [classes/:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138) [netty-all-4.1.12.Final.jar:4.1.12.Final]
at java.lang.Thread.run(Thread.java:834) [?:?]
Caused by: org.apache.distributedlog.exceptions.WriteException: Write rejected because stream test-stream-Ir8h5B8-0000 has encountered an error : writer has been closed due to error.
at org.apache.distributedlog.BKAsyncLogWriter.getCachedLogSegmentWriter(BKAsyncLogWriter.java:189) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.BKAsyncLogWriter.asyncWrite(BKAsyncLogWriter.java:280) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.BKAsyncLogWriter.write(BKAsyncLogWriter.java:416) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
... 10 more
14:02:52.937 [main] INFO - Starting benchmark with config: {
"drivers" : [ "driver-bookkeeper/bookkeeper.yaml" ],
"workers" : null,
"workersFile" : null,
"workloads" : [ "workloads/1-topic-1-partition-100b.yaml" ]
}
14:02:52.975 [main] INFO - Workloads: {
"1-topic-1-partition-100b" : {
"name" : "1 topic / 1 partition / 100b",
"topics" : 1,
"partitionsPerTopic" : 1,
"keyDistributor" : "NO_KEY",
"messageSize" : 100,
"payloadFile" : "payload/payload-100b.data",
"subscriptionsPerTopic" : 1,
"producersPerTopic" : 2,
"consumerPerSubscription" : 1,
"producerRate" : 1000,
"consumerBacklogSizeGB" : 0,
"testDurationMinutes" : 1
}
}
14:02:53.026 [main] INFO - --------------- WORKLOAD : 1 topic / 1 partition / 100b --- DRIVER : BookKeeper---------------
14:02:53.197 [main] INFO - Driver: {
"name" : "BookKeeper",
"driverClass" : "io.openmessaging.benchmark.driver.bookkeeper.DlogBenchmarkDriver"
}
14:02:53.337 [main] INFO - DistributedLog NamespaceDriverManager initialized
14:02:53.337 [main] INFO - DistributedLog NamespaceDriverManager initialized
14:02:53.355 [main] INFO - No feature provider is set. All features are disabled now.
14:02:53.388 [main] INFO - Created shared zooKeeper client builder dlzk:distributedlog://zookeeper:2181/distributedlog:factory_writer_shared: zkServers = zookeeper:2181, numRetries = 0, sessionTimeout = 30000, retryBackoff = 100, maxRetryBackoff = 200, zkAclId = null.
14:02:53.409 [main] INFO - Client environment:zookeeper.version=3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03, built on 06/29/2018 00:39 GMT
14:02:53.409 [main] INFO - Client environment:host.name=5e95860dad58
14:02:53.409 [main] INFO - Client environment:java.version=11.0.6
14:02:53.409 [main] INFO - Client environment:java.vendor=Oracle Corporation
14:02:53.409 [main] INFO - Client environment:java.home=/usr/local/openjdk-11
14:02:53.411 [main] INFO - Client environment:java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib
14:02:53.411 [main] INFO - Client environment:java.io.tmpdir=/tmp
14:02:53.411 [main] INFO - Client environment:java.compiler=<NA>
14:02:53.411 [main] INFO - Client environment:os.name=Linux
14:02:53.411 [main] INFO - Client environment:os.arch=amd64
14:02:53.411 [main] INFO - Client environment:os.version=3.10.0-957.el7.x86_64
14:02:53.411 [main] INFO - Client environment:user.name=root
14:02:53.411 [main] INFO - Client environment:user.home=/root
14:02:53.411 [main] INFO - Client environment:user.dir=/opt/openmessaging-benchmark
14:02:53.414 [main] INFO - Initiating client connection, connectString=zookeeper:2181 sessionTimeout=30000 watcher=dlshade.org.apache.bookkeeper.zookeeper.ZooKeeperWatcherBase@6de7778f
14:02:53.431 [main-SendThread(zookeeper:2181)] INFO - Opening socket connection to server zookeeper/172.22.0.2:2181. Will not attempt to authenticate using SASL (unknown error)
14:02:53.439 [main-SendThread(zookeeper:2181)] INFO - Socket connection established to zookeeper/172.22.0.2:2181, initiating session
14:02:53.444 [main-SendThread(zookeeper:2181)] INFO - Session establishment complete on server zookeeper/172.22.0.2:2181, sessionid = 0x1021228d8370018, negotiated timeout = 30000
14:02:53.446 [main-EventThread] INFO - ZooKeeper client is connected now.
14:02:53.539 [main] INFO - Created shared client builder bk:distributedlog://zookeeper:2181/distributedlog:factory_writer_shared : zkServers = zookeeper:2181, ledgersPath = /ledgers, numIOThreads = 8
14:02:53.541 [main] INFO - Propagate BKDLConfig to DLConfig : encodeRegionID = false, firstLogSegmentSequenceNumber = 1, createStreamIfNotExists = true, isFederated = false.
14:02:53.550 [main] INFO - Initialized BK namespace driver: clientId = 5e95860dad58/172.22.0.3, regionId = 0, federated = false.
14:02:53.568 [main] INFO - Log segment cache is enabled = true
14:02:53.568 [main] INFO - Initialized distributedlog namespace at distributedlog://zookeeper:2181/distributedlog
14:02:53.606 [ForkJoinPool.commonPool-worker-3] INFO - Successfully create topic test-stream-Ir8h5B8-0000 with 1 partitions
14:02:53.606 [main] INFO - Created 1 topics in 35.403847 ms
14:02:53.606 [main] INFO - Created 1 topics in 35.751094 ms
14:02:53.619 [ForkJoinPool.commonPool-worker-3] INFO - Open stream test-stream-Ir8h5B8-0000 for consumer
14:02:53.621 [dlog-benchmark-reader-thread-0] INFO - Create sync reader starting from DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0}
14:02:53.627 [main] INFO - Created 1 consumers in 19.870688 ms
14:02:53.627 [main] INFO - Created 1 consumers in 19.997035 ms
14:02:53.635 [ForkJoinPool.commonPool-worker-3] INFO - Open stream test-stream-Ir8h5B8-0000 for producer
14:02:53.636 [ForkJoinPool.commonPool-worker-5] INFO - Open stream test-stream-Ir8h5B8-0000 for producer
14:02:53.645 [main-EventThread] INFO - Ledger allocator for /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved version from -1 to 0.
14:02:53.647 [dlog-benchmark-reader-thread-0] INFO - Successfully open log reader for stream test-stream-Ir8h5B8-0000 at DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0}
14:02:53.658 [main-EventThread] INFO - Ledger allocator for /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved version from -1 to 0.
14:02:53.668 [main-EventThread] INFO - Starting the readahead entry reader for test-stream-Ir8h5B8-0000:<default> : segments = []
14:02:53.668 [main-EventThread] INFO - Initiating Recovery For test-stream-Ir8h5B8-0000:<default> : []
14:02:53.669 [main-EventThread] INFO - Initiating Recovery For test-stream-Ir8h5B8-0000:<default> : []
14:02:53.670 [main] INFO - Created 2 producers in 43.022277 ms
14:02:53.670 [main] INFO - Created 2 producers in 43.173542 ms
14:02:53.671 [main] INFO - Waiting for consumers to be ready
14:02:53.690 [main] INFO - Initiating Recovery For test-stream-Ir8h5B8-0000:<default> : []
14:02:53.693 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase ALLOCATING : version = 0.
14:02:53.695 [main] INFO - Initiating Recovery For test-stream-Ir8h5B8-0000:<default> : []
14:02:53.696 [DLM-/distributedlog-OrderedScheduler-3-0] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase ALLOCATING : version = 0.
14:02:53.711 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - Initiating client connection, connectString=zookeeper:2181 sessionTimeout=60000 watcher=dlshade.org.apache.bookkeeper.zookeeper.ZooKeeperWatcherBase@4b17568f
14:02:53.713 [DLM-/distributedlog-OrderedScheduler-0-0-SendThread(zookeeper:2181)] INFO - Opening socket connection to server zookeeper/172.22.0.2:2181. Will not attempt to authenticate using SASL (unknown error)
14:02:53.713 [DLM-/distributedlog-OrderedScheduler-0-0-SendThread(zookeeper:2181)] INFO - Socket connection established to zookeeper/172.22.0.2:2181, initiating session
14:02:53.715 [DLM-/distributedlog-OrderedScheduler-0-0-SendThread(zookeeper:2181)] INFO - Session establishment complete on server zookeeper/172.22.0.2:2181, sessionid = 0x1021228d8370019, negotiated timeout = 40000
14:02:53.715 [DLM-/distributedlog-OrderedScheduler-0-0-EventThread] INFO - ZooKeeper client is connected now.
14:02:53.733 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - BookKeeper metadata driver manager initialized
14:02:53.733 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - BookKeeper metadata driver manager initialized
14:02:53.734 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - BookKeeper metadata driver manager initialized
14:02:53.735 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - Initialize zookeeper metadata driver with external zookeeper client : ledgersRootPath = /ledgers.
14:02:53.753 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - Initialize rackaware ensemble placement policy @ <Bookie:172.22.0.3:0> @ /default-region/default-rack : org.apache.distributedlog.net.DNSResolverForRacks.
14:02:53.754 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - Not weighted
14:02:53.758 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - Weighted ledger placement is not enabled
14:02:53.767 [BookKeeperClientScheduler-OrderedScheduler-0-0] INFO - Adding a new node: /default-region/default-rack/172.22.0.6:3181
14:02:53.769 [BookKeeperClientScheduler-OrderedScheduler-0-0] INFO - Adding a new node: /default-region/default-rack/172.22.0.5:3181
14:02:53.769 [BookKeeperClientScheduler-OrderedScheduler-0-0] INFO - Adding a new node: /default-region/default-rack/172.22.0.7:3181
14:02:53.773 [DLM-/distributedlog-OrderedScheduler-0-0] WARN - Failed to instantiate ledger manager factory org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory, trying its shaded class dlshade.org.apac
he.bookkeeper.meta.HierarchicalLedgerManagerFactory
14:02:53.782 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - BookKeeper Client created bk:distributedlog://zookeeper:2181/distributedlog:factory_writer_shared with its own ZK Client : ledgersPath = /ledgers,
numRetries = 20, sessionTimeout = 60000, backoff = 100, maxBackoff = 200, dnsResolver =
14:02:53.901 [DLM-/distributedlog-OrderedScheduler-0-0-EventThread] INFO - Ensemble: [172.22.0.7:3181, 172.22.0.5:3181, 172.22.0.6:3181] for ledger: 8
14:02:53.903 [DLM-/distributedlog-OrderedScheduler-0-0-EventThread] INFO - Ensemble: [172.22.0.6:3181, 172.22.0.5:3181, 172.22.0.7:3181] for ledger: 9
14:02:53.905 [main-EventThread] INFO - Ledger allocator for /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved version from 0 to 1.
14:02:53.905 [main-EventThread] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase ALLOCATED : version = 1.
14:02:53.906 [main-EventThread] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase HANDING_OVER : version = 1.
14:02:53.907 [main-EventThread] INFO - No max ledger sequence number found while creating log segment 1 for test-stream-Ir8h5B8-0000:<default>.
14:02:53.913 [main-EventThread] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase ERROR : version = 0.
14:02:53.918 [main-EventThread] ERROR - Fail mark ledger 9 as allocated under /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation :
dlshade.org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion
dlshade.org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion
at dlshade.org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.util.Utils$5.processResult(Utils.java:395) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
at dlshade.org.apache.bookkeeper.zookeeper.ZooKeeperClient$22$1.processResult(ZooKeeperClient.java:1091) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
at dlshade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:566) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
at dlshade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:508) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
14:02:53.929 [main-EventThread] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase HANDED_OVER : version = 1.
14:02:53.929 [main-EventThread] INFO - Ledger allocator for /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved version from 1 to 2.
14:02:53.929 [main-EventThread] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase ALLOCATING : version = 2.
14:02:53.932 [main-EventThread] INFO - test-stream-Ir8h5B8-0000 added log segment (inprogress_000000000000000001 : [LogSegmentId:8, firstTxId:1583503373695, lastTxId:-999, version:VERSION_V5_SEQUENCE_ID, completionTime:0, recordCount:0, regionId:0, status:0, logSegmentSequenceNumber:1, lastEntryId:-1, lastSlotId:-1, inprogress:true, minActiveDLSN:DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0}, startSequenceId:-1]) to cache.
14:02:53.933 [DLM-/distributedlog-OrderedScheduler-1-0] INFO - test-stream-Ir8h5B8-0000 added log segment (inprogress_000000000000000001 : [LogSegmentId:8, firstTxId:1583503373695, lastTxId:-999, version:VERSION_V5_SEQUENCE_ID, completionTime:0, recordCount:0, regionId:0, status:0, logSegmentSequenceNumber:1, lastEntryId:-1, lastSlotId:-1, inprogress:true, minActiveDLSN:DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0}, startSequenceId:-1]) to cache.
14:02:53.934 [DLM-/distributedlog-OrderedScheduler-1-0] INFO - segments is updated with [[LogSegmentId:8, firstTxId:1583503373695, lastTxId:-999, version:VERSION_V5_SEQUENCE_ID, completionTime:0, recordCount:0, regionId:0, status:0, logSegmentSequenceNumber:1, lastEntryId:-1, lastSlotId:-1, inprogress:true, minActiveDLSN:DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0}, startSequenceId:0]]
14:02:53.942 [DLM-/distributedlog-OrderedScheduler-0-0-EventThread] INFO - Ensemble: [172.22.0.7:3181, 172.22.0.6:3181, 172.22.0.5:3181] for ledger: 10
14:02:53.944 [main-EventThread] INFO - Ledger allocator for /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved version from 2 to 3.
14:02:53.944 [main-EventThread] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase ALLOCATED : version = 3.
14:02:53.963 [main-EventThread] INFO - Deleting log segments older than 1583330573960 for test-stream-Ir8h5B8-0000:<default> : []
14:02:53.993 [BookKeeperClientWorker-OrderedExecutor-8-0] INFO - Closing the per channel bookie client for 172.22.0.7:3181
14:02:54.073 [DL-io-2] INFO - Successfully connected to bookie: [id: 0xde3e7c61, L:/172.22.0.3:53182 - R:builds_bookie1_1.builds_default/172.22.0.6:3181]
14:02:54.073 [DL-io-1] INFO - Successfully connected to bookie: [id: 0x5815edbc, L:/172.22.0.3:49790 - R:172.22.0.7/172.22.0.7:3181]
14:02:54.073 [DL-io-0] INFO - Successfully connected to bookie: [id: 0x58af16d5, L:/172.22.0.3:43310 - R:builds_bookie2_1.builds_default/172.22.0.5:3181]
14:02:54.082 [DL-io-2] INFO - Successfully connected to bookie: 172.22.0.6:3181
14:02:54.082 [DL-io-1] INFO - Successfully connected to bookie: 172.22.0.7:3181
14:02:54.082 [DL-io-0] INFO - Successfully connected to bookie: 172.22.0.5:3181
14:02:54.101 [DL-io-0] INFO - connection [id: 0x58af16d5, L:/172.22.0.3:43310 - R:builds_bookie2_1.builds_default/172.22.0.5:3181] authenticated as BookKeeperPrincipal{ANONYMOUS}
14:02:54.101 [DL-io-2] INFO - connection [id: 0xde3e7c61, L:/172.22.0.3:53182 - R:builds_bookie1_1.builds_default/172.22.0.6:3181] authenticated as BookKeeperPrincipal{ANONYMOUS}
14:02:54.102 [DL-io-1] INFO - connection [id: 0x5815edbc, L:/172.22.0.3:49790 - R:172.22.0.7/172.22.0.7:3181] authenticated as BookKeeperPrincipal{ANONYMOUS}
14:02:54.168 [BookKeeperClientWorker-OrderedExecutor-8-0] INFO - ReadAhead for test-stream-Ir8h5B8-0000:<default> is caught up
14:02:54.196 [main] INFO - All consumers are ready
14:02:54.198 [main] INFO - ----- Starting warm-up traffic ------
14:02:54.222 [local-worker-1-1] WARN - Write error on message
java.util.concurrent.CompletionException: org.apache.distributedlog.exceptions.WriteException: Write rejected because stream test-stream-Ir8h5B8-0000 has encountered an error : writer has been closed due to er
ror.
at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:331) ~[?:?]
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:346) ~[?:?]
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:870) [?:?]
at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:883) [?:?]
at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2251) [?:?]
at dlshade.org.apache.bookkeeper.common.concurrent.FutureUtils.stats(FutureUtils.java:365) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.BKAsyncLogWriter.write(BKAsyncLogWriter.java:415) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at io.openmessaging.benchmark.driver.bookkeeper.DlogBenchmarkProducer.sendAsync(DlogBenchmarkProducer.java:50) ~[driver-bookkeeper-0.0.1-SNAPSHOT.jar:?]
at io.openmessaging.benchmark.worker.LocalWorker.lambda$submitProducersToExecutor$10(LocalWorker.java:219) [classes/:?]
at java.util.ArrayList.forEach(ArrayList.java:1540) [?:?]
at io.openmessaging.benchmark.worker.LocalWorker.lambda$submitProducersToExecutor$11(LocalWorker.java:216) [classes/:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138) [netty-all-4.1.12.Final.jar:4.1.12.Final]
at java.lang.Thread.run(Thread.java:834) [?:?]
Caused by: org.apache.distributedlog.exceptions.WriteException: Write rejected because stream test-stream-Ir8h5B8-0000 has encountered an error : writer has been closed due to error.
at org.apache.distributedlog.BKAsyncLogWriter.getCachedLogSegmentWriter(BKAsyncLogWriter.java:189) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.BKAsyncLogWriter.asyncWrite(BKAsyncLogWriter.java:280) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.BKAsyncLogWriter.write(BKAsyncLogWriter.java:416) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
... 10 more
![dependabot[bot] avatar](https://avatars.githubusercontent.com/in/29110?v=4 "dependabot[bot]")
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent