[aarch64] Scylla dies on keyspace RF change in debug mode (asan) about scylladb

[ptrsmrn]

Here's the parsed stacktrace:

[Backtrace #0]
std::char_traits<char>::eq(char const&, char const&) at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/char_traits.h:363
std::basic_string_view<char, std::char_traits<char> >::rfind(char, unsigned long) const at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/string_view.tcc:125
std::basic_string_view<char, std::char_traits<char> >::find_last_of(char, unsigned long) const at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/string_view:511
is_class_name_qualified(std::basic_string_view<char, std::char_traits<char> >) at ././utils/class_registrator.hh:24
nonstatic_class_registry<locator::abstract_replication_strategy, locator::replication_strategy_params>::to_qualified_class_name(std::basic_string_view<char, std::char_traits<char> >) const at ././utils/class_registrator.hh:110
class_registry<locator::abstract_replication_strategy, locator::replication_strategy_params>::to_qualified_class_name(std::basic_string_view<char, std::char_traits<char> >) at ././utils/class_registrator.hh:157
locator::abstract_replication_strategy::to_qualified_class_name(std::basic_string_view<char, std::char_traits<char> >) at ./locator/abstract_replication_strategy.cc:81
keyspace_metadata at ./data_dictionary/data_dictionary.cc:216
lw_shared_ptr_no_esft<std::basic_string_view<char, std::char_traits<char> > &, std::basic_string_view<char, std::char_traits<char> > &, std::map<seastar::basic_sstring<char, unsigned int, 15U, true>, seastar::basic_sstring<char, unsigned int, 15U, true>, std::less<seastar::basic_sstring<char, unsigned int, 15U, true> >, std::allocator<std::pair<const seastar::basic_sstring<char, unsigned int, 15U, true>, seastar::basic_sstring<char, unsigned int, 15U, true> > > > &, std::optional<unsigned int> &, bool &, std::vector<seastar::lw_shared_ptr<const schema>, std::allocator<seastar::lw_shared_ptr<const schema> > >, data_dictionary::user_types_metadata, data_dictionary::storage_options &> at ././seastar/include/seastar/core/shared_ptr.hh:177
seastar::lw_shared_ptr<data_dictionary::keyspace_metadata> seastar::lw_shared_ptr<data_dictionary::keyspace_metadata>::make<std::basic_string_view<char, std::char_traits<char> >&, std::basic_string_view<char, std::char_traits<char> >&, std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > >&, std::optional<unsigned int>&, bool&, std::vector<seastar::lw_shared_ptr<schema const>, std::allocator<seastar::lw_shared_ptr<schema const> > >, data_dictionary::user_types_metadata, data_dictionary::storage_options&>(std::basic_string_view<char, std::char_traits<char> >&, std::basic_string_view<char, std::char_traits<char> >&, std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > >&, std::optional<unsigned int>&, bool&, std::vector<seastar::lw_shared_ptr<schema const>, std::allocator<seastar::lw_shared_ptr<schema const> > >&&, data_dictionary::user_types_metadata&&, data_dictionary::storage_options&) at ././seastar/include/seastar/core/shared_ptr.hh:281
seastar::lw_shared_ptr<data_dictionary::keyspace_metadata> seastar::make_lw_shared<data_dictionary::keyspace_metadata, std::basic_string_view<char, std::char_traits<char> >&, std::basic_string_view<char, std::char_traits<char> >&, std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > >&, std::optional<unsigned int>&, bool&, std::vector<seastar::lw_shared_ptr<schema const>, std::allocator<seastar::lw_shared_ptr<schema const> > >, data_dictionary::user_types_metadata, data_dictionary::storage_options&>(std::basic_string_view<char, std::char_traits<char> >&, std::basic_string_view<char, std::char_traits<char> >&, std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > >&, std::optional<unsigned int>&, bool&, std::vector<seastar::lw_shared_ptr<schema const>, std::allocator<seastar::lw_shared_ptr<schema const> > >&&, data_dictionary::user_types_metadata&&, data_dictionary::storage_options&) at ././seastar/include/seastar/core/shared_ptr.hh:442
data_dictionary::keyspace_metadata::new_keyspace(std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > >, std::optional<unsigned int>, bool, data_dictionary::storage_options) at ./data_dictionary/data_dictionary.cc:242
service::topology_coordinator::handle_global_request(service::group0_guard) at ./service/topology_coordinator.cc:818
service::topology_coordinator::handle_topology_transition(service::group0_guard) at ./service/topology_coordinator.cc:1547
service::topology_coordinator::run() at ./service/topology_coordinator.cc:2889
std::__n4861::coroutine_handle<seastar::internal::coroutine_traits_base<void>::promise_type>::resume() const at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/coroutine:240
seastar::internal::coroutine_traits_base<void>::promise_type::run_and_dispose() at ././seastar/include/seastar/core/coroutine.hh:122
seastar::reactor::run_tasks(seastar::reactor::task_queue&) at ./build/debug/seastar/./seastar/src/core/reactor.cc:2694
seastar::reactor::run_some_tasks() at ./build/debug/seastar/./seastar/src/core/reactor.cc:3157
seastar::reactor::do_run() at ./build/debug/seastar/./seastar/src/core/reactor.cc:3325
seastar::reactor::run() at ./build/debug/seastar/./seastar/src/core/reactor.cc:3215
seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) at ./build/debug/seastar/./seastar/src/core/app-template.cc:276
seastar::app_template::run(int, char**, std::function<seastar::future<int> ()>&&) at ./build/debug/seastar/./seastar/src/core/app-template.cc:167
scylla_main(int, char**) at ./main.cc:682
int std::__invoke_impl<int, int (*&)(int, char**), int, char**>(std::__invoke_other, int (*&)(int, char**), int&&, char**&&) at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/invoke.h:61
std::enable_if<is_invocable_r_v<int, int (*&)(int, char**), int, char**>, int>::type std::__invoke_r<int, int (*&)(int, char**), int, char**>(int (*&)(int, char**), int&&, char**&&) at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/invoke.h:114
std::_Function_handler<int (int, char**), int (*)(int, char**)>::_M_invoke(std::_Any_data const&, int&&, char**&&) at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/std_function.h:290
std::function<int (int, char**)>::operator()(int, char**) const at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/std_function.h:591
main at ./main.cc:2172
/data/scylla-s3-reloc.cache/by-build-id/323f3456b437dbabc24cc9bc59b57919ae1b7805/extracted/scylla/libreloc/libc.so.6: ELF 64-bit LSB shared object, ARM aarch64, version 1 (GNU/Linux), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, BuildID[sha1]=7efdd76ace6e9030f23372c68ca7282477519669, for GNU/Linux 3.7.0, not stripped

__libc_start_call_main at ??:?
__libc_start_main_alias_2 at :?
_start at ??:?

[ptrsmrn]

I refactored the code as follows:

[piotrs@fedora scylla2]$ git diff
diff --git a/service/topology_coordinator.cc b/service/topology_coordinator.cc
index f95761616c..763d4b95df 100644
--- a/service/topology_coordinator.cc
+++ b/service/topology_coordinator.cc
@@ -815,8 +815,10 @@ class topology_coordinator : public endpoint_lifecycle_subscriber {
                                                              .done(error)
                                                              .build()));
                 if (error.empty()) {
-                    auto ks_md = keyspace_metadata::new_keyspace(ks_name, *new_ks_props.get_replication_strategy_class(),
-                                                                 repl_opts, new_ks_props.get_initial_tablets("NetworkTopologyStrategy", true),
+                    const sstring strategy_class = *new_ks_props.get_replication_strategy_class();
+                    const sstring unqualified_strategy_name = "NetworkTopologyStrategy";
+                    auto ks_md = keyspace_metadata::new_keyspace(ks_name, strategy_class, repl_opts,
+                                                                 new_ks_props.get_initial_tablets(unqualified_strategy_name, true),
                                                                  new_ks_props.get_durable_writes(), new_ks_props.get_storage_options());
                     auto schema_muts = prepare_keyspace_update_announcement(_db, ks_md, guard.write_timestamp());
                     for (auto& m: schema_muts) {

So the temporary objects have been extracted into const variables, which should fix the isssue.

[ptrsmrn]

Interestingly, that didn't help. I suspect it's some compiler optimization that reorders the code. Here's the stacktrace of the code refactored as mentioned above:

__pthread_mutex_clocklock_common at :?
__vstrfmon_l_buffer.constprop.0 at ??:?
__gconv_transform_internal_ucs2 at :?
__sanitizer::Abort() at ??:?
__sanitizer::Die() at ??:?
__asan::ScopedInErrorReport::~ScopedInErrorReport() at ??:?
__asan::ReportDeadlySignal(__sanitizer::SignalContext const&) at ??:?
__asan::AsanOnDeadlySignal(int, void*, void*) at ??:?
linux-vdso.so.1 0x7db 
std::char_traits<char>::eq(char const&, char const&) at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/char_traits.h:373
std::basic_string_view<char, std::char_traits<char> >::rfind(char, unsigned long) const at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/string_view.tcc:125
std::basic_string_view<char, std::char_traits<char> >::find_last_of(char, unsigned long) const at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/string_view:510
is_class_name_qualified(std::basic_string_view<char, std::char_traits<char> >) at ././utils/class_registrator.hh:24
nonstatic_class_registry<locator::abstract_replication_strategy, locator::replication_strategy_params>::to_qualified_class_name(std::basic_string_view<char, std::char_traits<char> >) const at ././utils/class_registrator.hh:110
class_registry<locator::abstract_replication_strategy, locator::replication_strategy_params>::to_qualified_class_name(std::basic_string_view<char, std::char_traits<char> >) at ././utils/class_registrator.hh:157
locator::abstract_replication_strategy::to_qualified_class_name(std::basic_string_view<char, std::char_traits<char> >) at ./locator/abstract_replication_strategy.cc:81
keyspace_metadata at ./data_dictionary/data_dictionary.cc:216
lw_shared_ptr_no_esft<std::basic_string_view<char, std::char_traits<char> > &, std::basic_string_view<char, std::char_traits<char> > &, std::map<seastar::basic_sstring<char, unsigned int, 15U, true>, seastar::basic_sstring<char, unsigned int, 15U, true>, std::less<seastar::basic_sstring<char, unsigned int, 15U, true> >, std::allocator<std::pair<const seastar::basic_sstring<char, unsigned int, 15U, true>, seastar::basic_sstring<char, unsigned int, 15U, true> > > > &, std::optional<unsigned int> &, bool &, std::vector<seastar::lw_shared_ptr<const schema>, std::allocator<seastar::lw_shared_ptr<const schema> > >, data_dictionary::user_types_metadata, data_dictionary::storage_options &> at ././seastar/include/seastar/core/shared_ptr.hh:177
seastar::lw_shared_ptr<data_dictionary::keyspace_metadata> seastar::lw_shared_ptr<data_dictionary::keyspace_metadata>::make<std::basic_string_view<char, std::char_traits<char> >&, std::basic_string_view<char, std::char_traits<char> >&, std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > >&, std::optional<unsigned int>&, bool&, std::vector<seastar::lw_shared_ptr<schema const>, std::allocator<seastar::lw_shared_ptr<schema const> > >, data_dictionary::user_types_metadata, data_dictionary::storage_options&>(std::basic_string_view<char, std::char_traits<char> >&, std::basic_string_view<char, std::char_traits<char> >&, std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > >&, std::optional<unsigned int>&, bool&, std::vector<seastar::lw_shared_ptr<schema const>, std::allocator<seastar::lw_shared_ptr<schema const> > >&&, data_dictionary::user_types_metadata&&, data_dictionary::storage_options&) at ././seastar/include/seastar/core/shared_ptr.hh:281
seastar::lw_shared_ptr<data_dictionary::keyspace_metadata> seastar::make_lw_shared<data_dictionary::keyspace_metadata, std::basic_string_view<char, std::char_traits<char> >&, std::basic_string_view<char, std::char_traits<char> >&, std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > >&, std::optional<unsigned int>&, bool&, std::vector<seastar::lw_shared_ptr<schema const>, std::allocator<seastar::lw_shared_ptr<schema const> > >, data_dictionary::user_types_metadata, data_dictionary::storage_options&>(std::basic_string_view<char, std::char_traits<char> >&, std::basic_string_view<char, std::char_traits<char> >&, std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > >&, std::optional<unsigned int>&, bool&, std::vector<seastar::lw_shared_ptr<schema const>, std::allocator<seastar::lw_shared_ptr<schema const> > >&&, data_dictionary::user_types_metadata&&, data_dictionary::storage_options&) at ././seastar/include/seastar/core/shared_ptr.hh:442
data_dictionary::keyspace_metadata::new_keyspace(std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, std::map<seastar::basic_sstring<char, unsigned int, 15u, true>, seastar::basic_sstring<char, unsigned int, 15u, true>, std::less<seastar::basic_sstring<char, unsigned int, 15u, true> >, std::allocator<std::pair<seastar::basic_sstring<char, unsigned int, 15u, true> const, seastar::basic_sstring<char, unsigned int, 15u, true> > > >, std::optional<unsigned int>, bool, data_dictionary::storage_options) at ./data_dictionary/data_dictionary.cc:242
service::topology_coordinator::handle_global_request(service::group0_guard) at ./service/topology_coordinator.cc:820
service::topology_coordinator::handle_topology_transition(service::group0_guard) at ./service/topology_coordinator.cc:1549
service::topology_coordinator::run() at ./service/topology_coordinator.cc:2891
std::__n4861::coroutine_handle<seastar::internal::coroutine_traits_base<void>::promise_type>::resume() const at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/coroutine:240
seastar::internal::coroutine_traits_base<void>::promise_type::run_and_dispose() at ././seastar/include/seastar/core/coroutine.hh:125
/jenkins/workspace/scylla-master/byo/byo_build_tests_dtest/scylla/build/debug/seastar/libseastar.so 0x3e8da57 
/jenkins/workspace/scylla-master/byo/byo_build_tests_dtest/scylla/build/debug/seastar/libseastar.so 0x3e96a8b 
/jenkins/workspace/scylla-master/byo/byo_build_tests_dtest/scylla/build/debug/seastar/libseastar.so 0x3e9a6f3 
/jenkins/workspace/scylla-master/byo/byo_build_tests_dtest/scylla/build/debug/seastar/libseastar.so 0x3e9878f 
/jenkins/workspace/scylla-master/byo/byo_build_tests_dtest/scylla/build/debug/seastar/libseastar.so 0x38a0603 
/jenkins/workspace/scylla-master/byo/byo_build_tests_dtest/scylla/build/debug/seastar/libseastar.so 0x389dd2b 
scylla_main(int, char**) at ./main.cc:682
int std::__invoke_impl<int, int (*&)(int, char**), int, char**>(std::__invoke_other, int (*&)(int, char**), int&&, char**&&) at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/invoke.h:61
std::enable_if<is_invocable_r_v<int, int (*&)(int, char**), int, char**>, int>::type std::__invoke_r<int, int (*&)(int, char**), int, char**>(int (*&)(int, char**), int&&, char**&&) at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/invoke.h:114
std::_Function_handler<int (int, char**), int (*)(int, char**)>::_M_invoke(std::_Any_data const&, int&&, char**&&) at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/std_function.h:290
std::function<int (int, char**)>::operator()(int, char**) const at /usr/bin/../lib/gcc/aarch64-redhat-linux/13/../../../../include/c++/13/bits/std_function.h:591
main at ./main.cc:2174
__gconv_transform_ucs2reverse_internal at :?
__gconv_transform_ucs2reverse_internal at :?
_start at ??:?

[ptrsmrn]

@xemul latest aarch64 debug build passed - https://jenkins.scylladb.com/job/scylla-master/job/byo/job/byo_build_tests_dtest/2282/
But I'd keep this issue opened until #16723 passes the whole suite