<html>
  <head>
    <meta content="text/html; charset=ISO-8859-1"
      http-equiv="Content-Type">
  </head>
  <body bgcolor="#FFFFFF" text="#000000">
    <div class="moz-cite-prefix"><br>
      On 01/03/2014 11:38 PM, Xu (Simon) Chen wrote:<br>
    </div>
    <blockquote
cite="mid:CAGNyuhWtZvXCv_Rk1=6pcpeFkgA3oAGTR3fodqK48uaNvSer8A@mail.gmail.com"
      type="cite">
      <div dir="ltr">
        <div class="gmail_quote">
          <div class="HOEnZb">
            <div class="h5">
              <div dir="ltr">
                <div class="gmail_quote">
                  <div>
                    <div>
                      <div dir="ltr">
                        <div class="gmail_quote">
                          <div>
                            <div>
                              <div dir="ltr">
                                <div class="gmail_quote">
                                  <div>
                                    <div>
                                      <div dir="ltr">
                                        <div class="gmail_quote">
                                          <div>
                                            <div>
                                              <div dir="ltr">
                                                <div class="gmail_quote">
                                                  <div>
                                                    <div>
                                                      <div dir="ltr">
                                                        <div
                                                          class="gmail_quote">
                                                          <div>
                                                          <div>
                                                          <div dir="ltr">
                                                          <div
                                                          class="gmail_quote">
                                                          <div>
                                                          <div>
                                                          <div dir="ltr">
                                                          <div
                                                          class="gmail_quote">
                                                          <div>
                                                          <div>
                                                          <div dir="ltr">
                                                          <div
                                                          class="gmail_quote">
                                                          <div>
                                                          <div>
                                                          <div dir="ltr">
                                                          <div
                                                          class="gmail_quote">
                                                          <div dir="ltr">Hi
                                                          folks,
                                                          <div><br>
                                                          </div>
                                                          <div>I am
                                                          having trouble
                                                          with using
                                                          memcache as
                                                          the keystone
                                                          token backend.
                                                          I have three
                                                          keystone nodes
                                                          running
                                                          active/active.
                                                          Each is
                                                          running
                                                          keystone on
                                                          apache (for
                                                          kerberos
                                                          auth). I
                                                          recently
                                                          switched from
                                                          using sql
                                                          backend to
                                                          memcache,
                                                          while have
                                                          memcached
                                                          running on all
                                                          three of the
                                                          keystone
                                                          nodes. <br>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                        </div>
                                                      </div>
                                                    </div>
                                                  </div>
                                                </div>
                                              </div>
                                            </div>
                                          </div>
                                        </div>
                                      </div>
                                    </div>
                                  </div>
                                </div>
                              </div>
                            </div>
                          </div>
                        </div>
                      </div>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </div>
        </div>
      </div>
    </blockquote>
    <br>
    This triggers a memory of htere being something wonky with
    Greenthreads,  the threading override in Eventlet, and Memcached. 
    But you said Apache, so I think that you are not running with
    greenthreads?<br>
    <br>
    There are numerous things out there about apache and memcached
    performance.  For example, one article talks about filling up
    partitions, etc.<br>
    <br>
    <br>
    <blockquote
cite="mid:CAGNyuhWtZvXCv_Rk1=6pcpeFkgA3oAGTR3fodqK48uaNvSer8A@mail.gmail.com"
      type="cite">
      <div dir="ltr">
        <div class="gmail_quote">
          <div class="HOEnZb">
            <div class="h5">
              <div dir="ltr">
                <div class="gmail_quote">
                  <div>
                    <div>
                      <div dir="ltr">
                        <div class="gmail_quote">
                          <div>
                            <div>
                              <div dir="ltr">
                                <div class="gmail_quote">
                                  <div>
                                    <div>
                                      <div dir="ltr">
                                        <div class="gmail_quote">
                                          <div>
                                            <div>
                                              <div dir="ltr">
                                                <div class="gmail_quote">
                                                  <div>
                                                    <div>
                                                      <div dir="ltr">
                                                        <div
                                                          class="gmail_quote">
                                                          <div>
                                                          <div>
                                                          <div dir="ltr">
                                                          <div
                                                          class="gmail_quote">
                                                          <div>
                                                          <div>
                                                          <div dir="ltr">
                                                          <div
                                                          class="gmail_quote">
                                                          <div>
                                                          <div>
                                                          <div dir="ltr">
                                                          <div
                                                          class="gmail_quote">
                                                          <div>
                                                          <div>
                                                          <div dir="ltr">
                                                          <div
                                                          class="gmail_quote">
                                                          <div dir="ltr">
                                                          <div><br>
                                                          </div>
                                                          <div>This
                                                          setup would
                                                          run well for a
                                                          while, but
                                                          then apache
                                                          would start to
                                                          hog CPUs, and
                                                          memcached
                                                          would increase
                                                          to 30% or so.
                                                          I tried to
                                                          increase
                                                          memcached
                                                          cluster from 3
                                                          to 6 nodes,
                                                          but in general
                                                          the
                                                          performance is
                                                          much worse
                                                          compared to
                                                          sql backend.</div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                        </div>
                                                      </div>
                                                    </div>
                                                  </div>
                                                </div>
                                              </div>
                                            </div>
                                          </div>
                                        </div>
                                      </div>
                                    </div>
                                  </div>
                                </div>
                              </div>
                            </div>
                          </div>
                        </div>
                      </div>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </div>
        </div>
      </div>
    </blockquote>
    Probably due to the need for replication.  In order to keep it
    anywhere close to in sync,  it is going to require some non-trivial
    subset of fully connectedness.<br>
    <br>
    <br>
    <br>
    <blockquote
cite="mid:CAGNyuhWtZvXCv_Rk1=6pcpeFkgA3oAGTR3fodqK48uaNvSer8A@mail.gmail.com"
      type="cite">
      <div dir="ltr">
        <div class="gmail_quote">
          <div class="HOEnZb">
            <div class="h5">
              <div dir="ltr">
                <div class="gmail_quote">
                  <div>
                    <div>
                      <div dir="ltr">
                        <div class="gmail_quote">
                          <div>
                            <div>
                              <div dir="ltr">
                                <div class="gmail_quote">
                                  <div>
                                    <div>
                                      <div dir="ltr">
                                        <div class="gmail_quote">
                                          <div>
                                            <div>
                                              <div dir="ltr">
                                                <div class="gmail_quote">
                                                  <div>
                                                    <div>
                                                      <div dir="ltr">
                                                        <div
                                                          class="gmail_quote">
                                                          <div>
                                                          <div>
                                                          <div dir="ltr">
                                                          <div
                                                          class="gmail_quote">
                                                          <div>
                                                          <div>
                                                          <div dir="ltr">
                                                          <div
                                                          class="gmail_quote">
                                                          <div>
                                                          <div>
                                                          <div dir="ltr">
                                                          <div
                                                          class="gmail_quote">
                                                          <div>
                                                          <div>
                                                          <div dir="ltr">
                                                          <div
                                                          class="gmail_quote">
                                                          <div dir="ltr">
                                                          <div><br>
                                                          </div>
                                                          <div>Any
                                                          ideas?</div>
                                                          <div><br>
                                                          </div>
                                                          <div>Thanks.</div>
                                                          <span><font
                                                          color="#888888">
                                                          <div>-Simon</div>
                                                          </font></span></div>
                                                          </div>
                                                          <br>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          <br>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          <br>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          <br>
                                                          </div>
                                                          </div>
                                                          </div>
                                                        </div>
                                                        <br>
                                                      </div>
                                                    </div>
                                                  </div>
                                                </div>
                                                <br>
                                              </div>
                                            </div>
                                          </div>
                                        </div>
                                        <br>
                                      </div>
                                    </div>
                                  </div>
                                </div>
                                <br>
                              </div>
                            </div>
                          </div>
                        </div>
                        <br>
                      </div>
                    </div>
                  </div>
                </div>
                <br>
              </div>
            </div>
          </div>
        </div>
        <br>
      </div>
      <br>
      <fieldset class="mimeAttachmentHeader"></fieldset>
      <br>
      <pre wrap="">_______________________________________________
Mailing list: <a class="moz-txt-link-freetext" href="http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack">http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack</a>
Post to     : <a class="moz-txt-link-abbreviated" href="mailto:openstack@lists.openstack.org">openstack@lists.openstack.org</a>
Unsubscribe : <a class="moz-txt-link-freetext" href="http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack">http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack</a>
</pre>
    </blockquote>
    <br>
  </body>
</html>