segFault while write large data on multiple thread about rocksdb

[alanpaxton]

Hi @ChiragGangwani If you could provide further details of what you are doing, with a reproduction if possible, plus platform and RocksDB version you are using, it would make it more possible for someone to understand what might be going wrong..

[ChiragGangwani]

Hi @alanpaxton thanks for the response. Please find the further deatils below.

RocksDB version - 8.5.4

So we are doing high amount of write in multiple db & column families in multiple thread with write batch functionality.
There is no reproduction code because it happen unexpected anytime but only with large data.

write batch size - 50 mb.
Here some configuration -

DB Creation code -

private void initDB() throws StoreInitException {
        Logger.info("MapStore write batch size - " + storeProps.getWriteBatchSize());
        this.options.setStatistics(new Statistics());
        this.initCache();
        this.options.setAllowMmapReads(true);
        this.options.setInfoLogLevel(InfoLogLevel.WARN_LEVEL);
        this.options.setMaxLogFileSize(1000);
        this.options.setKeepLogFileNum(1);
        if (storeProps.isRead()) {
            createDescriptors();
            try
            {
                this.db = RocksDB.openReadOnly(dbOptions, this.storeProps.getDbFolder(), descriptors,
                        columnFamilyHandles);
            }
            catch (RocksDBException e)
            {
                throw new StoreInitException(e);
            }
            createDescriptorHandleMap();
            try {
                this.mapStoreMetadata = MapStoreMetadata.deserializeMetadata(storeProps.getDbFolder());
            } catch (Exception e) {
                Logger.warn("Store metadata is not available, metadata operations might be slow.");
                Logger.warn(e);
            }
        }
        else
        {
            Utility.ensureEmptyFolder(this.storeProps.getDbFolder());
            this.options.setCreateIfMissing(true);
            this.options.setCreateMissingColumnFamilies(true);
            try
            {
                this.db = RocksDB.open(options, this.storeProps.getDbFolder());
            }
            catch (RocksDBException e)
            {
                Logger.error(e);
                throw new StoreInitException(e);
            }
            addSegments(storeProps.getSegmentList());
        }
        defaultColumnFamily = this.db.getDefaultColumnFamily();
        logInitializingStore();
    }

Cache -

synchronized private void initCache() {
      storeProps.getCacheProperty().ifPresent(cacheProperties -> {
          Function<String, Cache> cacheFunc = id -> CacheType.CLOCK.equals(cacheProperties.getCacheType())
                  ? new ClockCache(cacheProperties.getCacheSize())
                  : new LRUCache(cacheProperties.getCacheSize());

          Cache cache = cacheMap.computeIfAbsent(cacheProperties.getId(), cacheFunc);
          this.blockBasedTableConfig = new BlockBasedTableConfig().setBlockCache(cache);
          this.options.setWriteBufferSize(cacheProperties.getWriteBufferSize());
          this.options.setTableFormatConfig(blockBasedTableConfig);
      });
  }

Put Operation -

public void put(Optional<String> segmentId, K key, V val) throws StoreWriteException {
        byte[] keyArr = toByteArr(key);
        byte[] valArr = toByteArr(val);
            try {
                **if(storeProps.getWriteBatchSize()>0){
                    putInBatch(segmentId, key, val);
                }**
                else {
                    this.db.put(getColumnFamilyHandle(segmentId), keyArr, valArr);
                }
            } catch (RocksDBException e) {
                Logger.error(e);
                throw new StoreWriteException(e);
            }
    }

private void putInBatch(Optional<String> segmentId, K key, V val) throws StoreWriteException {
        byte[] keyArr = toByteArr(key);
        byte[] valArr = toByteArr(val);
           try {
               writeBatch.put(getColumnFamilyHandle(segmentId), keyArr, valArr);
           } catch (RocksDBException e) {
               throw new StoreWriteException(e);
           }
        if (writeBatch.getDataSize() >= storeProps.getWriteBatchSize()) {
            putBatchInDb();
        }
    }

private void putBatchInDb() throws StoreWriteException {
     try {
         if(writeBatch.count()>0){
             db.write(new WriteOptions(), writeBatch);
             writeBatch.clear();
         }
     } catch (RocksDBException e) {
         throw new StoreWriteException(e);
     }
 }

[ChiragGangwani]

Uploading hs_err_pid3859541.log…
Please check the crash logs for further details.