<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns:mv="http://macVmlSchemaUri" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Title" content="">
<meta name="Keywords" content="">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:"Courier New";
panose-1:2 7 3 9 2 2 5 2 4 4;}
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0cm;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-language:EN-US;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:#954F72;
text-decoration:underline;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0cm;
margin-right:0cm;
margin-bottom:0cm;
margin-left:36.0pt;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-language:EN-US;}
span.EmailStyle17
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.EmailStyle18
{mso-style-type:personal-compose;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.msoIns
{mso-style-type:export-only;
mso-style-name:"";
text-decoration:underline;
color:teal;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-language:EN-US;}
@page WordSection1
{size:595.0pt 842.0pt;
margin:70.85pt 70.85pt 2.0cm 70.85pt;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:541289694;
mso-list-type:hybrid;
mso-list-template-ids:1486369216 134807567 134807577 134807579 134807567 134807577 134807579 134807567 134807577 134807579;}
@list l0:level1
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l0:level2
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l0:level3
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l0:level4
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l0:level5
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l0:level6
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l0:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l0:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l0:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l1
{mso-list-id:712390607;
mso-list-type:hybrid;
mso-list-template-ids:575955044 134807567 134807577 134807579 134807567 134807577 134807579 134807567 134807577 134807579;}
@list l1:level1
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level2
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level3
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l1:level4
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level5
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level6
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l1:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l1:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l2
{mso-list-id:1379084880;
mso-list-type:hybrid;
mso-list-template-ids:-1791485720 134807567 134807577 134807579 134807567 134807577 134807579 134807567 134807577 134807579;}
@list l2:level1
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level2
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level3
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l2:level4
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level5
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level6
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l2:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;}
@list l2:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l3
{mso-list-id:1901402746;
mso-list-type:hybrid;
mso-list-template-ids:2031386850 -1312776054 134807555 134807557 134807553 134807555 134807557 134807553 134807555 134807557;}
@list l3:level1
{mso-level-start-at:0;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:Calibri;
mso-bidi-font-family:"Times New Roman";}
@list l3:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New",serif;}
@list l3:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l3:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l3:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New",serif;}
@list l3:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l3:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l3:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New",serif;}
@list l3:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
ol
{margin-bottom:0cm;}
ul
{margin-bottom:0cm;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1027"/>
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1"/>
</o:shapelayout></xml><![endif]-->
</head>
<body bgcolor="white" lang="EN-GB" link="#0563C1" vlink="#954F72">
<div class="WordSection1">
<p class="MsoNormal"><span style="font-size:11.0pt">Hi zuulers,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">the zuul-executor resource governor topic seems to be a recurring now and we might want take the step and make it a bit smarter.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">I think the current approach of a set of on/off governors based on the current conditions may not be sufficient. I thought about that and like to have feedback about what you think about that.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">TLDR; I propose having a congestion control algorithm managing a congestion window utilizing a slow start with a generic sensor interface and weighted job costs.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Algorithm<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">--------------<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">The algorithm I propose would manage a congestion window of an abstract metric measured in points. This is intended to leverage some (simple) weighting of jobs as multi node jobs e.g. probably take more resources
than single node jobs.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">The algorithm consists of two threads. One managing the congestion window, one accepting the jobs.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Congestion window management:<o:p></o:p></span></p>
<ol style="margin-top:0cm" start="1" type="1">
<li class="MsoListParagraph" style="margin-left:0cm;mso-list:l1 level1 lfo2"><span style="font-size:11.0pt">Start with a window of size START_WINDOW_SIZE points<o:p></o:p></span></li><li class="MsoListParagraph" style="margin-left:0cm;mso-list:l1 level1 lfo2"><span style="font-size:11.0pt">Get current used percentage of window<o:p></o:p></span></li><li class="MsoListParagraph" style="margin-left:0cm;mso-list:l1 level1 lfo2"><span style="font-size:11.0pt">Ask sensors for green/red<o:p></o:p></span></li><li class="MsoListParagraph" style="margin-left:0cm;mso-list:l1 level1 lfo2"><span style="font-size:11.0pt">If all green AND window-usage > INCREASE_WINDOW_THRESHOLD<o:p></o:p></span>
<ol style="margin-top:0cm" start="1" type="a">
<li class="MsoListParagraph" style="margin-left:0cm;mso-list:l1 level2 lfo2"><span style="font-size:11.0pt">Increase window<o:p></o:p></span></li></ol>
</li><li class="MsoListParagraph" style="margin-left:0cm;mso-list:l1 level1 lfo2"><span style="font-size:11.0pt">If one red<o:p></o:p></span>
<ol style="margin-top:0cm" start="1" type="a">
<li class="MsoListParagraph" style="margin-left:0cm;mso-list:l1 level2 lfo2"><span style="font-size:11.0pt">Decrease window below current usage<o:p></o:p></span></li></ol>
</li><li class="MsoListParagraph" style="margin-left:0cm;mso-list:l1 level1 lfo2"><span style="font-size:11.0pt">Loop back to step 2<o:p></o:p></span></li></ol>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Job accepting:<o:p></o:p></span></p>
<ol style="margin-top:0cm" start="1" type="1">
<li class="MsoListParagraph" style="margin-left:0cm;mso-list:l2 level1 lfo3"><span style="font-size:11.0pt">Get current used window-percentage<o:p></o:p></span></li><li class="MsoListParagraph" style="margin-left:0cm;mso-list:l2 level1 lfo3"><span style="font-size:11.0pt">If window-usage < window-size<o:p></o:p></span>
<ol style="margin-top:0cm" start="1" type="a">
<li class="MsoListParagraph" style="margin-left:0cm;mso-list:l2 level2 lfo3"><span style="font-size:11.0pt">Register function if necessary<o:p></o:p></span></li><li class="MsoListParagraph" style="margin-left:0cm;mso-list:l2 level2 lfo3"><span style="font-size:11.0pt">Accept job<o:p></o:p></span></li><li class="MsoListParagraph" style="margin-left:0cm;mso-list:l2 level2 lfo3"><span style="font-size:11.0pt">summarize window-usage (the job will update asynchronously when finished)<o:p></o:p></span></li></ol>
</li><li class="MsoListParagraph" style="margin-left:0cm;mso-list:l2 level1 lfo3"><span style="font-size:11.0pt">Else<o:p></o:p></span>
<ol style="margin-top:0cm" start="1" type="a">
<li class="MsoListParagraph" style="margin-left:0cm;mso-list:l2 level2 lfo3"><span style="font-size:11.0pt">Deregister function if necessary<o:p></o:p></span></li></ol>
</li><li class="MsoListParagraph" style="margin-left:0cm;mso-list:l2 level1 lfo3"><span style="font-size:11.0pt">Loop back to step 1<o:p></o:p></span></li></ol>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">The magic numbers used are subject for further discussion and algorithm tweaking.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Weighting of jobs<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">------------------------<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Now different jobs take different amounts of resources so we would need some simple estimation about that. This could be tuned in the future. For the start I’d propose something simple like this:<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Cost_job = 5 + 5 * size of inventory<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">In the future this could be improved to estimate the costs based on historical data of the individual jobs.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Sensors<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">----------<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Further different ways of deployment will have different needs about the sensors. E.g. the load and ram sensors which utilize load1 and memfree won’t work in a kubernetes based deployments as they assume the
executor is located exclusively on a VM. In order to mitigate I’d like to have some generic sensor interface where we also could put a cgroups sensor into which checks resource usage according to the cgroup limit (which is what we need for a kubernetes hosted
zuul). We also could put a filesystem sensor in which monitors if there is enough local storage. For hooking this into the algorithm I think we could start with a single function<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">def isStatusOk() -> bool<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Exposing the data<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">-------------------------<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">The window-usage and window-size values could also be exported to statsd. This could enable autoscaling of the number of executors in deployments supporting that.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">What are your thoughts about that?<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Kind regards<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Tobi<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">--</span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><b><span lang="EN-US" style="font-size:11.0pt">BMW Car IT GmbH</span></b><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Tobias Henkel</span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Spezialist Entwicklung<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Moosacher Straße 86<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">80809 München<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"> <o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Tel.: </span><span lang="DE" style="font-size:11.0pt">+49 89 189311-48<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Fax: +49 89 189311-20</span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Mail: <a href="mailto:tobias.henkel@bmw.de">tobias.henkel@bmw.de</a></span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Web: </span><span style="font-size:11.0pt"><a href="http://www.bmw-carit.de/"><span lang="EN-US">http://www.bmw-carit.de</span></a><o:p></o:p></span></p>
<p class="MsoNormal"><span lang="DE" style="font-size:11.0pt">-----------------------------------------------------------------------------<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="DE" style="font-size:11.0pt">BMW Car IT GmbH<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="DE" style="font-size:11.0pt">Geschäftsführer: Kai-Uwe Balszuweit<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="DE" style="font-size:11.0pt">und Christian Salzmann<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="DE" style="font-size:11.0pt">Sitz und Registergericht: München HRB 134810<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">-----------------------------------------------------------------------------</span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
</div>
</body>
</html>