Launching an instance, it fails with the following error message:
Error: Failed to perform requested operation on instance "Instance name", the instance has an error status: Please try again later [Error: Timed out waiting for a reply to message ID].
Message: Timed out waiting for a reply to message ID
Details: File "/usr/lib/python2.7/dist-packages/nova/conductor/manager.py", line 405, in build_instances context, request_spec, filter_properties) File "/usr/lib/python2.7/dist-packages/nova/conductor/manager.py", line 449, in _schedule_instances hosts = self.scheduler_client.select_destinations(context, spec_obj) File "/usr/lib/python2.7/dist-packages/nova/scheduler/utils.py", line 372, in wrapped return func(*args, **kwargs) File "/usr/lib/python2.7/dist-packages/nova/scheduler/client/__init__.py", line 51, in select_destinations return self.queryclient.select_destinations(context, spec_obj) File "/usr/lib/python2.7/dist-packages/nova/scheduler/client/__init__.py", line 37, in __run_method return getattr(self.instance, __name)(*args, **kwargs) File "/usr/lib/python2.7/dist-packages/nova/scheduler/client/query.py", line 32, in select_destinations return self.scheduler_rpcapi.select_destinations(context, spec_obj) File "/usr/lib/python2.7/dist-packages/osprofiler/profiler.py", line 154, in wrapper return f(*args, **kwargs) File "/usr/lib/python2.7/dist-packages/nova/scheduler/rpcapi.py", line 123, in select_destinations return cctxt.call(ctxt, 'select_destinations', **msg_args) File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/client.py", line 158, in call retry=self.retry) File "/usr/lib/python2.7/dist-packages/oslo_messaging/transport.py", line 90, in _send timeout=timeout, retry=retry) File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 470, in send retry=retry) File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 459, in _send result = self._waiter.wait(msg_id, timeout) File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 342, in wait message = self.waiters.get(msg_id, timeout=timeout) File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 244, in get 'to message ID %s' % msg_id)
VMware Integrated OpenStack 5.x
RabbitMQ has encountered a network partition
sudo su -
rabbitmqctl cluster_status
Cluster status of node rabbit@database01 ...
[{nodes,[{disc,[rabbit@database01,rabbit@database02,rabbit@database03]}]},
{running_nodes,[rabbit@database03,rabbit@database01]},
{cluster_name,<<"rabbit@database01">>},
{partitions,[{rabbit@database03,[rabbit@database02]}]}]
sudo su -
viocli services stop && viocli services start
Cluster status of node rabbit@database01 ...
[{nodes,[{disc,[rabbit@database01,rabbit@database02,rabbit@database03]}]},
{running_nodes,[rabbit@database02,rabbit@database03,rabbit@database01]},
{cluster_name,<<"rabbit@database01">>},