<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
</head>
<body>
<div><style id="ms-outlook-ios-style" type="text/css">html {
background-color: transparent !important;
}
body {
background-color: transparent !important;
color: #333;
line-height: 150%;
font-family: "-apple-system", "HelveticaNeue";
margin: 0;
}
.ms-outlook-ios-reference-expand {
display: block;
color: #999;
padding: 20px 0px;
text-decoration: none;
}
.ms-outlook-ios-availability-container {
max-width: 500px;
margin: auto;
padding: 12px 15px 15px 15px;
border: 1px solid #C7E0F4;
border-radius: 4px;
}
.ms-outlook-ios-availability-container > .ms-outlook-ios-availability-delete-button {
width: 25px;
height: 25px;
right: -12px;
top: -12px;
background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAAAXNSR0IArs4c6QAACxpJREFUeAHlnFuMXlUVx/fcOzNtp0ynF0U7hWKrEmKLosZEjUZ9MgZIQBNC0uAtJr745oOJIT74xgskJkQbAlQNJmBMfNDEG0YjEC7GIBQZ6IAI005L79O5+/+dfut0f5dzzt7nu8w37UrWt893zt5rr/U/e6+zL+ucHrcGtLq62q9qd4gnxTeKb6kcc267eEI8Kz4mnhFPi58Rv1g5nunp6VnS8ZVHAqdHPCk+KP6zuBWEHORNinvWNWoYIN4q/o74mLidhHzqob71AxzKij8g/p14LYh6qb97QUM58T7x38TdQOiBPt0FmhQaEf9M3I2EXiNr7tOkBK3pVvGCuJsJ/dCzqVbWWxZxVTygso+InxBz3M2Efuj5SEXvUrqWQloV7lRtT4vfX6rWtS30pqr/uMZp78SqEQ2WgPqQKvmnuKWtaWnFuaWVVbciXl51rk+a9fb2uP6EY80qzL+oHB8RYC8V5vQyRIEloD6tsk965UsdAsyZ+WV35uKyOz+/4uZ0YgmEMqhfyA3397rRoV63eUOf2zzUJxAzMsed/owA+2tokWCwmgVKDcadvLDsZs8vutMXV5zkhepYl08GurENvW5idMCNj/Q5Nb5mKBiwoGoqXe/fZTQCkmNnl9x/Ty+4xZzW0yeLB9WC6Hp0QbLSJRd0sAzSGTSgzO8bG3TbN/W7IGMay/lwSJcslC+gcOZviKN9FC3pjVML7uKi+l0NDakf0Sq2DPe5kYFeh9FZBMgXJOPU3HLSOufpxzW0QTJ2bRlMZNZcCvmLD9slwHKdfraGKi2gAGhKHPXUm1tcdVMn5t05+SWfMGjbaL+7RiABUFkCuHd1I46fX6q7ERvlz/ZsHXLDA7mmNaqap+QeAQZwDSlTooDiGuOouxqWzDjJ3X91dj55slkWWs216io7musqJi5N6Zwz6uJv1XRxnqA3TAwlrTbNHHZwWNnuFmAN+30eWLeqIAO5YHr7zKK63WLqvPFDOzcNuPeODSR+KFhQZEb82/9OL7p3zi6m/k0Gq1sOuPdsjvYet6nsrxup0BAstSrmUqfEQTVxG147seCOn7vcguly+7ZtKNMdGukZdI7uf+T4xaquuW3jgLt+62CM88eILQLsQm2ldY6j0v3uV8YgoBBYC9SYxkI37RzuKFDogZ+iXuo34gaiXwRh9/0VHKqK1bUsZdqnHC9X5cr5Q9ebfveyMnS73eODOSU6c+noyYWkW1ptk9cMxnbJD6p1HbHypFUtq4LmIT9D3jHOHB9l1C1AoQ83DH2M0BN9I+hQbeuqAkuCbhB/KkQg/oGnngQm2Wn63dCifN3Rx7okeqIvegcSOIBHSilYFRQfSK8UHDCOYuIL4cz3ypl3I6EX+kHoi94R9IDfulKwJGBc/KUQQYzMbcDJ8ICnXp8vKURIh/Kg1yX9Lrln9Eb/QAIPcEnIN/FOO5mX0paYwhjhF0qMlq14R1L0q/ZfCy64MzqX4pKAVWlq94ZozqTY5nqMzBlwrgdCT5t/oj92BNK91hWtZe1SwW1FhXFRrB4YMYXJmf9atiRl7vvz52fd4/86GXNXq2TYH1oFch59blZ+yM7mp+iJvkbYkbOYYdlIwQV8HNvo0OcuJfm/9HVbZsFpMtcLpV++MOvuPvyfJPs9n9jufnrnnphRdVoNQH3jsSl36Cl29l0i466b2e0vJvRlSkTLwg7smRi9PIDNkQA+D1nL+nZOxvQSC3dGrB7oZgXTcOWJRAEMxeAIv5HUUwsUJ325SaacH/RFbyPfHjuXkR7kfK/6I03sk/zJI5o7K5xGLLPE0O03jTtalFEsYI2AQt5tkhtDvt7YE9iNPyuckpXsj4VUxnq5CiRZWbiLXY/irtL1ygCWBVSZroze6A9hD3YF0g5KMRcsJDYYjFjhLENlAGslUKazr79vl13PSCeDwWIXxoil4LIUA1g7gEJvX3/frgKbbgSsvQWZkstsVxnFdkErZ2kIYO0CCh18/X27TL+M9BbA2ppxMT0NTravx/TGBndphhIHeYCx8ukPDxDfzHCjVj30xw4Iu7x2UJvV/z/Jc3STf6bRsU2YucZ2VavIAEOejZtIn5w6qxWCubSaVgJlQrFjrjIqxT7W7QsocfCFYPn7dnZHCgQHXzbA/Kdku4FCOd8O374cxXfSDYdzMiSX/GlB8Q0oklZ/HcAevGOPdmSqVeE/5wvveb3IwjO+Hb59OQXHAatuYb62QAnBtSJy/+PMv/WrqaquRwFaGOe53mrCLxoFepZZwDpnhbLSEk02S1TdeXSudeZ+C4sd6ddVkHGC0AAjQgYC6BhgnS3K6Ds/Yg9aRY2Awne9/P39pUb6MXr5dvj25ciYAawTORmSS8wOCPuBcIa28pCcKPmTBRRTGKoqOzUKUQf9zaljV2X2U1R0GrBeKcrFdeKjjIg1aIbygLIOQdouwHz9fbsKbHoGBKr2xrIKEEhmFLmlZMWSNAQoK9AuwHz9fbus3oz0xWCwiLYziljwtyJJGgOUFWwHYL7+RBIGUtINnw3JjFCCLSDio/ymHFK+DFAmt5WAobfFd2GP3wisvox0plcFpnXxtYwM6WlcFqGJRsRHxdATWjO3KQ3lYqcwWYAhN4Z8vbHHc8V5Yv4inJbM+j/l5bRrxHAaEUhGawmlOe+hEAuU1dEIMF+u5ctK0Re9jXx77FxG+hDnqZ8Vw68p+QXHecQ47vm3LqRDh93jQ9qPu7ymnVeWmT2bFqyZs8ScVJxXIOcaRtOiAOqr+ydCW4c2K5bc0ZOXdqRZeThw7Uho8O5ueqCBtVH1E085mqNjcolIu9e9CverwsoQrKjoml5nLP2Cd6Ov040O3J06LsV3CKzVpBvqgClPUJQfUcEWO8Dgjoi79UDoaYNp9MeOQPohQJHXfBbHD/NTRDRFooKN2IeLiEyxYh1N0e9t6WmE/hFu4DEr54P1B50MGs2z4E9UMMS0gdDE5eYG9YmsdvygF/rZxBm9/Q2Lgjp/r+vp4zYFS00Nc39cUDi9TPi0TUDZ4X1FCnUjoZfFZqAvekfQd60LUiYFqyLgUaXTlePchMgUwqclLMl3WvtvhCZ2E6EPekHoib4RET9/V7FXk8KVnyqwJJBByI/8DHnHbCkRPm2E/+oWwGpjStHT3wIznXPSe/xWRb4qsCoFDyl9qnJcmBBnTvi0EYC9NLN2PgwfRf3oYYR+kfHwYFDnvxs+FDRIPaDMfHQiaJbJc7U2vJvH85UWB98QLNnOqP4+Jd/jOJTW+g0Lhgf21MNHdeQNC8ARWAymcHIf5X8osVZ01b27AzgC7Holz4nH+B9KDAKvqrfCDBgB9hUdPy4O8l9WjpRFtqvmfUMzXIB9U8cP2v+YFOcf8yYr227sTLHCwexgXb3JasAIsB/oOHgMZuUsxXha2hX/jrQZ3CxgJoe1LSLuCCSLfvteczuWuANXOK3KrDT4ZXIEZA4dsqRXuuRPdD3ah2XJ5DwAEs1C16MV0hXpksznWgSMXz0j1vZ+18FqE2A4/YfFUU9JK7/G6Zuqv9QXQxpNdwpt0YDvN8p0szhoZ6hQYOcyHFZVvDSe+5Z9W9RRCxsU3ydeEnczteQrRy0BUSgdEP+jS9Hqju9n+UgLKL6l9XXx0S4BrTu/zFYDWr/AOig+skagdf83/3zAOBZQvOryRTEf+Donbid15GuS0eOsWlBC/gsl9iW/LP6C+PPi68TN0usS8EcxH6z4be2qZrPCG5XvCFi1FQu8SZ1j6YdXYeC9YuLxiZyGicQltpuoRPiEmJVLwqPgZwXOtNKO0v8BzRAPSFNM7HEAAAAASUVORK5CYII=");
background-size: 25px 25px;
background-position: center;
}
#ms-outlook-ios-main-container {
margin: 0 0 0 0;
margin-top: 120;
padding: 8;
}
#ms-outlook-ios-content-container {
padding: 0;
padding-top: 12;
padding-bottom: 20;
}
.ms-outlook-ios-mention {
color: #333;
background-color: #f1f1f1;
border-radius: 4px;
padding: 0 2px 0 2px;
pointer-events: none;
text-decoration: none;
}
.ms-outlook-ios-mention-external {
color: #ba8f0d;
background-color: #fdf7e7;
}
.ms-outlook-ios-mention-external-clear-design {
color: #ba8f0d;
background-color: #f1f1f1;
}</style><style id="ms-outlook-ios-dark-mode-style" type="text/css">.ms-outlook-ios-dark-mode {
color: #E1E1E1 !important;
}
.ms-outlook-ios-dark-mode .ms-outlook-ios-reference-expand {
color: #777777 !important;
}
.ms-outlook-ios-dark-mode a:not([class]) {
color: #0086F0;
}
.ms-outlook-ios-dark-mode font[color="#000000"] {
color: #E1E1E1 !important;
}
.ms-outlook-ios-dark-mode .ms-outlook-ios-availability-container {
border-color: #0086F0 !important;
}
.ms-outlook-ios-dark-mode .ms-outlook-ios-availability-container .ms-outlook-ios-availability-timeslot-container {
background-color: rgba(0, 120, 215, 0.2) !important;
}
.ms-outlook-ios-dark-mode .ms-outlook-ios-availability-container .ms-outlook-ios-availability-border {
border-top: 1px solid #0086F0 !important;
}
.ms-outlook-ios-dark-mode .ms-outlook-ios-availability-container > .ms-outlook-ios-availability-delete-button {
filter: invert(100%) grayscale(100%) brightness(90%) sepia(100%) hue-rotate(-180deg) saturate(700%) contrast(0.8);
}
.ms-outlook-ios-dark-mode .ms-outlook-ios-mention {
color: #A8A8AC !important;
background-color: #292929 !important;
}</style>
<meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0">
<div style="direction: ltr;">
<div>
<div></div>
</div>
<div style="direction: ltr;">Hi,</div>
<div style="direction: ltr;">Are you using ssl connections ?</div>
<div><br>
</div>
<div style="direction: ltr;">Can be this issue ? </div>
<div style="direction: ltr;">https://bugs.launchpad.net/ubuntu/+source/oslo.messaging/+bug/1800957</div>
<div><br>
</div>
<div><br>
</div>
</div>
<div> </div>
<hr style="display:inline-block;width:98%" tabindex="-1">
<div id="divRplyFwdMsg" dir="dir="ltr""><font face="Calibri, sans-serif" style="font-size:11pt" color="#000000"><b>From:</b> Laurent Dumont <laurentfdumont@gmail.com><br>
<b>Sent:</b> Wednesday, July 31, 2019 4:20 PM<br>
<b>To:</b> Grant Morley<br>
<b>Cc:</b> openstack-operators@lists.openstack.org<br>
<b>Subject:</b> Re: Slow instance launch times due to RabbitMQ
<div> </div>
</font></div>
<meta content="text/html; charset=utf-8">
<div dir="ltr">That is a bit strange, list_queues should return stuff. Couple of ideas :
<div>
<ul>
<li>Are the Rabbit connection failure logs on the compute pointing to a specific controller?</li><li>Are there any logs within Rabbit on the controller that would point to a transient issue?</li><li>cluster_status is a snapshot of the cluster at the time you ran the command. If the alarms have cleared, you won't see anything.</li><li>If you have the RabbitMQ management plugin activated, I would recommend a quick look to see the historical metrics and overall status.</li></ul>
</div>
</div>
<br>
<div class="gmail_quote">
<div dir="ltr" class="gmail_attr">On Wed, Jul 31, 2019 at 9:35 AM Grant Morley <<a href="mailto:grant@civo.com">grant@civo.com</a>> wrote:<br>
</div>
<blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex; border-left:1px solid rgb(204,204,204); padding-left:1ex">
<div bgcolor="#FFFFFF">
<p>Hi guys,</p>
<p>We are using Ubuntu 16 and OpenStack ansible to do our setup.</p>
<p>rabbitmqctl list_queues<br>
Listing queues<br>
</p>
<p>(Doesn't appear to be any queues )</p>
<p>rabbitmqctl cluster_status</p>
<p>Cluster status of node 'rabbit@management-1-rabbit-mq-container-b4d7791f'<br>
[{nodes,[{disc,['rabbit@management-1-rabbit-mq-container-b4d7791f',<br>
'rabbit@management-2-rabbit-mq-container-b455e77d',<br>
'rabbit@management-3-rabbit-mq-container-1d6ae377']}]},<br>
{running_nodes,['rabbit@management-3-rabbit-mq-container-1d6ae377',<br>
'rabbit@management-2-rabbit-mq-container-b455e77d',<br>
'rabbit@management-1-rabbit-mq-container-b4d7791f']},<br>
{cluster_name,<<"openstack">>},<br>
{partitions,[]},<br>
{alarms,[{'rabbit@management-3-rabbit-mq-container-1d6ae377',[]},<br>
{'rabbit@management-2-rabbit-mq-container-b455e77d',[]},<br>
{'rabbit@management-1-rabbit-mq-container-b4d7791f',[]}]}]</p>
<p>Regards,<br>
</p>
<div class="gmail-m_-173761228863317427moz-cite-prefix">On 31/07/2019 11:49, Laurent Dumont wrote:<br>
</div>
<blockquote type="cite">
<div dir="ltr">Could you forward the output of the following commands on a controller node? :
<div><br>
</div>
<div>rabbitmqctl cluster_status</div>
<div>rabbitmqctl list_queues</div>
<div><br>
</div>
<div>You won't necessarily see a high load on a Rabbit cluster that is in a bad state.</div>
</div>
<br>
<div class="gmail_quote">
<div dir="ltr" class="gmail_attr">On Wed, Jul 31, 2019 at 5:19 AM Grant Morley <<a href="mailto:grant@civo.com" target="_blank">grant@civo.com</a>> wrote:<br>
</div>
<blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex; border-left:1px solid rgb(204,204,204); padding-left:1ex">
<div bgcolor="#FFFFFF">
<p>Hi all,</p>
<p>We are randomly seeing slow instance launch / deletion times and it appears to be because of RabbitMQ. We are seeing a lot of these messages in the logs for Nova and Neutron:</p>
<p>ERROR oslo.messaging._drivers.impl_rabbit [-] [f4ab3ca0-b837-4962-95ef-dfd7d60686b6] AMQP server on
<a href="http://10.6.2.212:5671" target="_blank">10.6.2.212:5671</a> is unreachable: Too many heartbeats missed. Trying again in 1 seconds. Client port: 37098: ConnectionForced: Too many heartbeats missed</p>
<p>The RabbitMQ cluster isn't under high load and I am not seeing any packets drop over the network when I do some tracing.</p>
<p>We are only running 15 compute nodes currently and have >1000 instances so it isn't a large deployment.<br>
</p>
<p>Are there any good configuration tweaks for RabbitMQ running on OpenStack Queens? </p>
<p>Many Thanks,<br>
</p>
<div class="gmail-m_-173761228863317427gmail-m_4478358746559024905moz-signature">
-- <br>
<div><br>
</div>
<div>
<div style="font-family:helvetica,Arial; font-size:13px"><font face="Lucida Grande, Verdana, Arial, sans-serif" color="#A8D8FF"><span style="font-size:11px; line-height:15px; float:left; margin-right:10px"><img src="https://www.civo.com/images/email-logo.jpg"></span></font>
<div style="margin-left:90px; line-height:15px; color:rgb(109,190,255); margin-bottom:5px">
<span>Grant Morley</span> <font face="Hind, Lucida Grande,
Verdana, Arial, sans-serif">
<span style="font-size:15px"></span></font></div>
<div style="margin-left:90px; line-height:15px; color:rgb(136,136,136); margin-bottom:10px">
<font face="Hind, Lucida Grande, Verdana, Arial,
sans-serif"><span style="font-size:15px">Cloud Lead, Civo Ltd</span></font></div>
<div style="font-family:Hind,helvetica,arial; margin-left:90px"><a href="https://www.civo.com/" target="_blank">www.civo.com</a><font color="#888888"> |
</font><a href="https://www.civo.com/signup" target="_blank">Signup for an account!</a></div>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</blockquote>
<div class="gmail-m_-173761228863317427moz-signature">-- <br>
<div><br>
</div>
<div>
<div style="font-family:helvetica,Arial; font-size:13px"><font face="Lucida Grande, Verdana, Arial, sans-serif" color="#A8D8FF"><span style="font-size:11px; line-height:15px; float:left; margin-right:10px"><img src="https://www.civo.com/images/email-logo.jpg"></span></font>
<div style="margin-left:90px; line-height:15px; color:rgb(109,190,255); margin-bottom:5px">
<span style="font-family:Hind,"Lucida Grande",Verdana,Arial,sans-serif; font-size:18px; color:rgb(35,157,255)">Grant Morley</span>
<font face="Hind,
Lucida Grande, Verdana, Arial, sans-serif"><span style="font-size:15px"></span></font></div>
<div style="margin-left:90px; line-height:15px; color:rgb(136,136,136); margin-bottom:10px">
<font face="Hind, Lucida
Grande, Verdana, Arial, sans-serif"><span style="font-size:15px">Cloud Lead, Civo Ltd</span></font></div>
<div style="font-family:Hind,helvetica,arial; margin-left:90px"><a href="https://www.civo.com/" target="_blank" style="color:rgb(35,157,255); font-family:Hind,"Lucida Grande",Verdana,Arial,sans-serif; line-height:15px; text-decoration:none">www.civo.com</a><font color="#888888">
| </font><a href="https://www.civo.com/signup" target="_blank" style="color:rgb(35,157,255); font-family:Hind,"Lucida Grande",Verdana,Arial,sans-serif; line-height:15px; margin-right:6px; text-decoration:none">Signup for an account!</a></div>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</body>
</html>