Random generic server crashes in the log about rabbitmq-server

[michaelklishin]

@jnordberg can you please post your publishing code (in particular the message metadata/BasicProperties) and the output of rabbitmq report? Thanks.

[jnordberg]

Reporting server status on {{2014,8,13},{14,33,50}}

 ...
Status of node rabbit@safetyos ...
[{pid,1401},
 {running_applications,
     [{rabbitmq_management,"RabbitMQ Management Console","3.3.5"},
      {rabbitmq_web_dispatch,"RabbitMQ Web Dispatcher","3.3.5"},
      {webmachine,"webmachine","1.10.3-rmq3.3.5-gite9359c7"},
      {mochiweb,"MochiMedia Web Server","2.7.0-rmq3.3.5-git680dba8"},
      {rabbitmq_management_agent,"RabbitMQ Management Agent","3.3.5"},
      {rabbit,"RabbitMQ","3.3.5"},
      {os_mon,"CPO  CXC 138 46","2.2.15"},
      {inets,"INETS  CXC 138 49","5.10.2"},
      {mnesia,"MNESIA  CXC 138 12","4.12.1"},
      {amqp_client,"RabbitMQ AMQP Client","3.3.5"},
      {xmerl,"XML parser","1.3.7"},
      {sasl,"SASL  CXC 138 11","2.4"},
      {stdlib,"ERTS  CXC 138 10","2.1"},
      {kernel,"ERTS  CXC 138 10","3.0.1"}]},
 {os,{unix,linux}},
 {erlang_version,
     "Erlang/OTP 17 [erts-6.1] [source-d2a4c20] [smp:2:2] [async-threads:30] [kernel-poll:true]\n"},
 {memory,
     [{total,23947552},
      {connection_procs,2776},
      {queue_procs,2776},
      {plugins,240824},
      {other_proc,8844672},
      {mnesia,31272},
      {mgmt_db,59388},
      {msg_index,12544},
      {other_ets,553828},
      {binary,198488},
      {code,10566398},
      {atom,531937},
      {other_system,2902649}]},
 {alarms,[]},
 {listeners,[{clustering,25672,"::"},{amqp,5672,"::"}]},
 {vm_memory_high_watermark,0.4},
 {vm_memory_limit,794127564},
 {disk_free_limit,50000000},
 {disk_free,13210066944},
 {file_descriptors,
     [{total_limit,924},{total_used,3},{sockets_limit,829},{sockets_used,1}]},
 {processes,[{limit,1048576},{used,179}]},
 {run_queue,0},
 {uptime,91}]

Cluster status of node rabbit@safetyos ...
[{nodes,[{disc,[rabbit@safetyos]}]},
 {running_nodes,[rabbit@safetyos]},
 {cluster_name,<<"rabbit@safetyos">>},
 {partitions,[]}]

Application environment of node rabbit@safetyos ...
[{auth_backends,[rabbit_auth_backend_internal]},
 {auth_mechanisms,['PLAIN','AMQPLAIN']},
 {backing_queue_module,rabbit_variable_queue},
 {channel_max,0},
 {cluster_nodes,{[],disc}},
 {cluster_partition_handling,ignore},
 {collect_statistics,fine},
 {collect_statistics_interval,5000},
 {default_permissions,[<<".*">>,<<".*">>,<<".*">>]},
 {default_user,<<"guest">>},
 {default_user_tags,[administrator]},
 {default_vhost,<<"/">>},
 {delegate_count,16},
 {disk_free_limit,50000000},
 {enabled_plugins_file,"/etc/rabbitmq/enabled_plugins"},
 {error_logger,{file,"/var/log/rabbitmq/[email protected]"}},
 {frame_max,131072},
 {halt_on_upgrade_failure,true},
 {heartbeat,580},
 {hipe_compile,false},
 {hipe_modules,[rabbit_reader,rabbit_channel,gen_server2,rabbit_exchange,
                rabbit_command_assembler,rabbit_framing_amqp_0_9_1,
                rabbit_basic,rabbit_event,lists,queue,priority_queue,
                rabbit_router,rabbit_trace,rabbit_misc,rabbit_binary_parser,
                rabbit_exchange_type_direct,rabbit_guid,rabbit_net,
                rabbit_amqqueue_process,rabbit_variable_queue,
                rabbit_binary_generator,rabbit_writer,delegate,gb_sets,lqueue,
                sets,orddict,rabbit_amqqueue,rabbit_limiter,gb_trees,
                rabbit_queue_index,rabbit_exchange_decorator,gen,dict,ordsets,
                file_handle_cache,rabbit_msg_store,array,
                rabbit_msg_store_ets_index,rabbit_msg_file,
                rabbit_exchange_type_fanout,rabbit_exchange_type_topic,mnesia,
                mnesia_lib,rpc,mnesia_tm,qlc,sofs,proplists,credit_flow,pmon,
                ssl_connection,tls_connection,ssl_record,tls_record,gen_fsm,
                ssl]},
 {included_applications,[]},
 {log_levels,[{connection,info}]},
 {loopback_users,[]},
 {msg_store_file_size_limit,16777216},
 {msg_store_index_module,rabbit_msg_store_ets_index},
 {plugins_dir,"/usr/lib/rabbitmq/lib/rabbitmq_server-3.3.5/sbin/../plugins"},
 {plugins_expand_dir,"/var/lib/rabbitmq/mnesia/rabbit@safetyos-plugins-expand"},
 {queue_index_max_journal_entries,65536},
 {reverse_dns_lookups,false},
 {sasl_error_logger,{file,"/var/log/rabbitmq/[email protected]"}},
 {server_properties,[]},
 {ssl_apps,[asn1,crypto,public_key,ssl]},
 {ssl_cert_login_from,distinguished_name},
 {ssl_listeners,[]},
 {ssl_options,[]},
 {tcp_listen_options,[binary,
                      {packet,raw},
                      {reuseaddr,true},
                      {backlog,128},
                      {nodelay,true},
                      {linger,{true,0}},
                      {exit_on_close,false}]},
 {tcp_listeners,[5672]},
 {trace_vhosts,[]},
 {vm_memory_high_watermark,0.4},
 {vm_memory_high_watermark_paging_ratio,0.5}]

Connections:

Channels:

Queues on /:

Exchanges on /:
name    type    durable auto_delete internal    arguments   policy
    direct  true    false   false   []
amq.direct  direct  true    false   false   []
amq.fanout  fanout  true    false   false   []
amq.headers headers true    false   false   []
amq.match   headers true    false   false   []
amq.rabbitmq.log    topic   true    false   true    []
amq.rabbitmq.trace  topic   true    false   true    []
amq.topic   topic   true    false   false   []
test-exchange   topic   true    false   false   []

Bindings on /:

Consumers on /:

Permissions on /:
user    configure   write   read
guest   .*  .*  .*

Policies on /:

Parameters on /:

...done.

[jnordberg]

I just figured it out when trying to write a small code example that could reproduce it, in some cases I was sending priority: 0 which causes this error.

var amqp = require('amqp')
var connection = amqp.createConnection({host: 'localhost'})

connection.on('error', function(error) {
  console.log('connection error', error)
})

connection.once('ready', function() {
  console.log('connection ready')

  var exchangeOpts = {
    durable: true,
    autoDelete: false,
    confirm: true
  }

  connection.exchange('test-exchange', exchangeOpts, function(exchange) {
    console.log('exchange open')
    var msg = {hello: 'world'}
    var opts = {priority: 0}
    console.log('publish message', msg);
    exchange.publish('test', msg, opts, function() {
      console.log('message published', msg, arguments)
    })
  })

  connection.queue('my-queue', function(queue) {
    console.log('queue ready')
    queue.bind('test-exchange', 'test')
    queue.subscribe(function(message) {
      console.log('got message', message);
    })
  })

})

$ node queue-crash.js
connection ready
exchange open
publish message { hello: 'world' }
queue ready
Unhandled connection error: INTERNAL_ERROR
connection error { [Error: INTERNAL_ERROR] code: 541 }

[michaelklishin]

@jnordberg is this using node-amqp? node-amqp is known to have issues and is not maintained, so I'd recommend trying with amqp.lib.

In general, priority = 0 should be fine but I cannot reproduce the issue so far with Ruby and Java clients. So my guess is that node-amqp simply does not serialize things properly when priority equals 0.

[jnordberg]

@michaelklishin yep, node-amqp was the culprit. Sorry for the hassle and thanks for the amqp.lib tip, setting priority to 0 there causes no problems.