<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="ProgId" content="Word.Document">
<meta name="Generator" content="Microsoft Word 12">
<meta name="Originator" content="Microsoft Word 12">
<link rel="File-List" href="cid:filelist.xml@01CC89BA.7369D5E0"><!--[if gte mso 9]><xml>
<o:OfficeDocumentSettings>
<o:AllowPNG/>
<o:DoNotRelyOnCSS/>
<o:TargetScreenSize>1024x768</o:TargetScreenSize>
</o:OfficeDocumentSettings>
</xml><![endif]--><!--[if gte mso 9]><xml>
<w:WordDocument>
<w:SpellingState>Clean</w:SpellingState>
<w:TrackMoves/>
<w:TrackFormatting/>
<w:EnvelopeVis/>
<w:ValidateAgainstSchemas/>
<w:SaveIfXMLInvalid>false</w:SaveIfXMLInvalid>
<w:IgnoreMixedContent>false</w:IgnoreMixedContent>
<w:AlwaysShowPlaceholderText>false</w:AlwaysShowPlaceholderText>
<w:DoNotPromoteQF/>
<w:LidThemeOther>EN-GB</w:LidThemeOther>
<w:LidThemeAsian>ZH-CN</w:LidThemeAsian>
<w:LidThemeComplexScript>X-NONE</w:LidThemeComplexScript>
<w:Compatibility>
<w:DoNotExpandShiftReturn/>
<w:BreakWrappedTables/>
<w:SplitPgBreakAndParaMark/>
<w:DontVertAlignCellWithSp/>
<w:DontBreakConstrainedForcedTables/>
<w:DontVertAlignInTxbx/>
<w:Word11KerningPairs/>
<w:CachedColBalance/>
<w:UseFELayout/>
</w:Compatibility>
<w:BrowserLevel>MicrosoftInternetExplorer4</w:BrowserLevel>
<m:mathPr>
<m:mathFont m:val="Cambria Math"/>
<m:brkBin m:val="before"/>
<m:brkBinSub m:val="--"/>
<m:smallFrac m:val="off"/>
<m:dispDef/>
<m:lMargin m:val="0"/>
<m:rMargin m:val="0"/>
<m:defJc m:val="centerGroup"/>
<m:wrapIndent m:val="1440"/>
<m:intLim m:val="subSup"/>
<m:naryLim m:val="undOvr"/>
</m:mathPr></w:WordDocument>
</xml><![endif]--><!--[if gte mso 9]><xml>
<w:LatentStyles DefLockedState="false" DefUnhideWhenUsed="true"
DefSemiHidden="true" DefQFormat="false" DefPriority="99"
LatentStyleCount="267">
<w:LsdException Locked="false" Priority="0" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="Normal"/>
<w:LsdException Locked="false" Priority="9" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="heading 1"/>
<w:LsdException Locked="false" Priority="9" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="heading 2"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 3"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 4"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 5"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 6"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 7"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 8"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 9"/>
<w:LsdException Locked="false" Priority="39" Name="toc 1"/>
<w:LsdException Locked="false" Priority="39" Name="toc 2"/>
<w:LsdException Locked="false" Priority="39" Name="toc 3"/>
<w:LsdException Locked="false" Priority="39" Name="toc 4"/>
<w:LsdException Locked="false" Priority="39" Name="toc 5"/>
<w:LsdException Locked="false" Priority="39" Name="toc 6"/>
<w:LsdException Locked="false" Priority="39" Name="toc 7"/>
<w:LsdException Locked="false" Priority="39" Name="toc 8"/>
<w:LsdException Locked="false" Priority="39" Name="toc 9"/>
<w:LsdException Locked="false" Priority="35" QFormat="true" Name="caption"/>
<w:LsdException Locked="false" Priority="10" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="Title"/>
<w:LsdException Locked="false" Priority="1" Name="Default Paragraph Font"/>
<w:LsdException Locked="false" Priority="11" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="Subtitle"/>
<w:LsdException Locked="false" Priority="22" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="Strong"/>
<w:LsdException Locked="false" Priority="20" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="Emphasis"/>
<w:LsdException Locked="false" Priority="59" SemiHidden="false"
UnhideWhenUsed="false" Name="Table Grid"/>
<w:LsdException Locked="false" UnhideWhenUsed="false" Name="Placeholder Text"/>
<w:LsdException Locked="false" Priority="1" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="No Spacing"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Shading"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false"
UnhideWhenUsed="false" Name="Light List"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Grid"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 1"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 2"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 1"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 2"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 1"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 2"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 3"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false"
UnhideWhenUsed="false" Name="Dark List"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Shading"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful List"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Grid"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Shading Accent 1"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false"
UnhideWhenUsed="false" Name="Light List Accent 1"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Grid Accent 1"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 1 Accent 1"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 2 Accent 1"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 1 Accent 1"/>
<w:LsdException Locked="false" UnhideWhenUsed="false" Name="Revision"/>
<w:LsdException Locked="false" Priority="34" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="List Paragraph"/>
<w:LsdException Locked="false" Priority="29" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="Quote"/>
<w:LsdException Locked="false" Priority="30" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="Intense Quote"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 2 Accent 1"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 1 Accent 1"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 2 Accent 1"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 3 Accent 1"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false"
UnhideWhenUsed="false" Name="Dark List Accent 1"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Shading Accent 1"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful List Accent 1"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Grid Accent 1"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Shading Accent 2"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false"
UnhideWhenUsed="false" Name="Light List Accent 2"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Grid Accent 2"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 1 Accent 2"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 2 Accent 2"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 1 Accent 2"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 2 Accent 2"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 1 Accent 2"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 2 Accent 2"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 3 Accent 2"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false"
UnhideWhenUsed="false" Name="Dark List Accent 2"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Shading Accent 2"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful List Accent 2"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Grid Accent 2"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Shading Accent 3"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false"
UnhideWhenUsed="false" Name="Light List Accent 3"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Grid Accent 3"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 1 Accent 3"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 2 Accent 3"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 1 Accent 3"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 2 Accent 3"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 1 Accent 3"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 2 Accent 3"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 3 Accent 3"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false"
UnhideWhenUsed="false" Name="Dark List Accent 3"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Shading Accent 3"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful List Accent 3"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Grid Accent 3"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Shading Accent 4"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false"
UnhideWhenUsed="false" Name="Light List Accent 4"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Grid Accent 4"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 1 Accent 4"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 2 Accent 4"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 1 Accent 4"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 2 Accent 4"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 1 Accent 4"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 2 Accent 4"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 3 Accent 4"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false"
UnhideWhenUsed="false" Name="Dark List Accent 4"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Shading Accent 4"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful List Accent 4"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Grid Accent 4"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Shading Accent 5"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false"
UnhideWhenUsed="false" Name="Light List Accent 5"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Grid Accent 5"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 1 Accent 5"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 2 Accent 5"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 1 Accent 5"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 2 Accent 5"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 1 Accent 5"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 2 Accent 5"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 3 Accent 5"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false"
UnhideWhenUsed="false" Name="Dark List Accent 5"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Shading Accent 5"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful List Accent 5"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Grid Accent 5"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Shading Accent 6"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false"
UnhideWhenUsed="false" Name="Light List Accent 6"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false"
UnhideWhenUsed="false" Name="Light Grid Accent 6"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 1 Accent 6"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Shading 2 Accent 6"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 1 Accent 6"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium List 2 Accent 6"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 1 Accent 6"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 2 Accent 6"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false"
UnhideWhenUsed="false" Name="Medium Grid 3 Accent 6"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false"
UnhideWhenUsed="false" Name="Dark List Accent 6"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Shading Accent 6"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful List Accent 6"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false"
UnhideWhenUsed="false" Name="Colorful Grid Accent 6"/>
<w:LsdException Locked="false" Priority="19" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="Subtle Emphasis"/>
<w:LsdException Locked="false" Priority="21" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="Intense Emphasis"/>
<w:LsdException Locked="false" Priority="31" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="Subtle Reference"/>
<w:LsdException Locked="false" Priority="32" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="Intense Reference"/>
<w:LsdException Locked="false" Priority="33" SemiHidden="false"
UnhideWhenUsed="false" QFormat="true" Name="Book Title"/>
<w:LsdException Locked="false" Priority="37" Name="Bibliography"/>
<w:LsdException Locked="false" Priority="39" QFormat="true" Name="TOC Heading"/>
</w:LatentStyles>
</xml><![endif]--><style>
<!--
/* Font Definitions */
@font-face
{font-family:Helvetica;
panose-1:2 11 6 4 2 2 2 2 2 4;
mso-font-charset:0;
mso-generic-font-family:swiss;
mso-font-pitch:variable;
mso-font-signature:536881799 -2147483648 8 0 511 0;}
@font-face
{font-family:SimSun;
panose-1:2 1 6 0 3 1 1 1 1 1;
mso-font-alt:\5B8B\4F53;
mso-font-charset:134;
mso-generic-font-family:auto;
mso-font-pitch:variable;
mso-font-signature:3 135135232 16 0 262145 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;
mso-font-charset:1;
mso-generic-font-family:roman;
mso-font-format:other;
mso-font-pitch:variable;
mso-font-signature:0 0 0 0 0 0;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;
mso-font-charset:0;
mso-generic-font-family:swiss;
mso-font-pitch:variable;
mso-font-signature:-1610611985 1073750139 0 0 159 0;}
@font-face
{font-family:Tahoma;
panose-1:2 11 6 4 3 5 4 4 2 4;
mso-font-charset:0;
mso-generic-font-family:swiss;
mso-font-pitch:variable;
mso-font-signature:1627400839 -2147483648 8 0 66047 0;}
@font-face
{font-family:"\@SimSun";
panose-1:2 1 6 0 3 1 1 1 1 1;
mso-font-charset:134;
mso-generic-font-family:auto;
mso-font-pitch:variable;
mso-font-signature:3 135135232 16 0 262145 0;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{mso-style-unhide:no;
mso-style-qformat:yes;
mso-style-parent:"";
margin:0cm;
margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:12.0pt;
font-family:"Times New Roman","serif";
mso-fareast-font-family:SimSun;}
h2
{mso-style-priority:9;
mso-style-unhide:no;
mso-style-qformat:yes;
mso-style-link:"Heading 2 Char";
mso-margin-top-alt:auto;
margin-right:0cm;
mso-margin-bottom-alt:auto;
margin-left:0cm;
mso-pagination:widow-orphan;
mso-outline-level:2;
font-size:18.0pt;
font-family:"Times New Roman","serif";
font-weight:bold;}
a:link, span.MsoHyperlink
{mso-style-noshow:yes;
mso-style-priority:99;
color:blue;
text-decoration:underline;
text-underline:single;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-noshow:yes;
mso-style-priority:99;
color:purple;
text-decoration:underline;
text-underline:single;}
p
{mso-style-noshow:yes;
mso-style-priority:99;
mso-margin-top-alt:auto;
margin-right:0cm;
mso-margin-bottom-alt:auto;
margin-left:0cm;
mso-pagination:widow-orphan;
font-size:12.0pt;
font-family:"Times New Roman","serif";
mso-fareast-font-family:SimSun;}
span.Heading2Char
{mso-style-name:"Heading 2 Char";
mso-style-noshow:yes;
mso-style-priority:9;
mso-style-unhide:no;
mso-style-locked:yes;
mso-style-link:"Heading 2";
mso-ansi-font-size:13.0pt;
mso-bidi-font-size:13.0pt;
font-family:"Cambria","serif";
mso-ascii-font-family:Cambria;
mso-fareast-font-family:SimSun;
mso-hansi-font-family:Cambria;
mso-bidi-font-family:"Times New Roman";
color:#4F81BD;
font-weight:bold;}
span.EmailStyle19
{mso-style-type:personal-reply;
mso-style-noshow:yes;
mso-style-unhide:no;
mso-ansi-font-size:11.0pt;
mso-bidi-font-size:11.0pt;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-fareast-font-family:SimSun;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:"Times New Roman";
color:#1F497D;}
span.SpellE
{mso-style-name:"";
mso-spl-e:yes;}
.MsoChpDefault
{mso-style-type:export-only;
mso-default-props:yes;
mso-ascii-font-family:Calibri;
mso-fareast-font-family:SimSun;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:"Times New Roman";}
@page WordSection1
{size:612.0pt 792.0pt;
margin:72.0pt 72.0pt 72.0pt 72.0pt;
mso-header-margin:36.0pt;
mso-footer-margin:36.0pt;
mso-paper-source:0;}
div.WordSection1
{page:WordSection1;}
-->
</style><!--[if gte mso 10]>
<style>
/* Style Definitions */
table.MsoNormalTable
{mso-style-name:"Table Normal";
mso-tstyle-rowband-size:0;
mso-tstyle-colband-size:0;
mso-style-noshow:yes;
mso-style-priority:99;
mso-style-qformat:yes;
mso-style-parent:"";
mso-padding-alt:0cm 5.4pt 0cm 5.4pt;
mso-para-margin:0cm;
mso-para-margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:11.0pt;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;}
</style>
<![endif]--><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="EN-GB" link="blue" vlink="purple" style="tab-interval:36.0pt">
<div class="WordSection1">
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D">If you are on a Battery Backed Unit raid controller, then its generally
safe to disable barriers for journal <span class="SpellE">filesystems</span>.<span style="mso-spacerun:yes">
</span>If your doing soft raid, <span class="SpellE">jbod</span>, single disk arrays or
<span class="SpellE">cheaped</span> out and did not get a BBU then you may want to enable barriers for
<span class="SpellE">filesystem</span> consistency.<span style="mso-spacerun:yes">
</span><span style="mso-spacerun:yes"> </span><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D">For raid cards with a BBU then set your
<span class="SpellE">io</span> scheduler to <span class="SpellE">noop</span>, and disable barriers.<span style="mso-spacerun:yes">
</span>The raid card does its own re-ordering of <span class="SpellE">io</span> operations, the OS has an incomplete picture of the true drive geometry.
<span style="mso-spacerun:yes"> </span>The raid card is emulating one disk geometry which could be an array of 2 – 100+ disks.<span style="mso-spacerun:yes">
</span>The OS simply can not make good judgment calls on how best to schedule <span class="SpellE">
io</span> to different parts of the disk because its built around the assumption of a single spinning disk.<span style="mso-spacerun:yes">
</span>This is also true for if a write has made it safely non persistent cache (<span class="SpellE">ie</span> disk cache),
<span style="mso-spacerun:yes"> </span>to a persistent cache (<span class="SpellE">ie</span> the battery in your raid card) or persistent storage (that array of disks) .<span style="mso-spacerun:yes">
</span><span style="mso-spacerun:yes"> </span><span style="mso-spacerun:yes"> </span>This is a failure of the Raid card <-> OS interface.<span style="mso-spacerun:yes">
</span>There simply is not the richness to say (signal write is ok if on platter or persistent cache not okay in disk cache) or<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D">Enabling barriers effectively turns all writes into Write-Through operations,
so the write goes straight to the disk platter and you get little performance benefit from the raid card (which hurts a lot in terms of lost
<span class="SpellE">iops</span>). <span style="mso-spacerun:yes"> </span>If the BBU looses charge/fails
<span style="mso-spacerun:yes"> </span>then the raid controller downgrades to Write-Through (<span class="SpellE">vs</span> Write-Backed) operation.<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D">BBU
<span style="mso-spacerun:yes"> </span>raid controllers disable disk caches, as these are not safe in event of power loss, and do not provide any benefit over the raid card cache.<span style="mso-spacerun:yes">
</span><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D"><span style="mso-spacerun:yes"> </span><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D">In the context of swift,
<span class="SpellE">hdfs</span> and other highly replicated <span class="SpellE">
datastores</span>, I run them in <span class="SpellE">jbod</span> or raid-0 + <span class="SpellE">
nobarrier</span> , <span class="SpellE">noatime</span>, <span class="SpellE">nodiratime</span> with a
<span class="SpellE">filesystem</span> aligned to the geometry of underlying storage*
<span style="mso-spacerun:yes"> </span>etc to squeeze as much performance as possible out of the raw storage.<span style="mso-spacerun:yes">
</span>Let the application layer deal with redundancy of data across the network, if a machine /disk dies … so what, you have N other copies of that data elsewhere on the network.<span style="mso-spacerun:yes">
</span>A bit of storage is lost … do consider how many nodes can be down at any time when operating these sorts of clusters Big
<span class="SpellE">boxen</span> with lots of storage may seem attractive from a density perspective until you loose one and 25% of your storage capacity with it … many smaller baskets …<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D">For network level data consistency
<span style="mso-spacerun:yes"> </span>swift should have a <span style="mso-spacerun:yes">
</span>data scrubber (periodic process to read and compare checksums of replicated blocks), I have not checked if this is implemented or on the roadmap.<span style="mso-spacerun:yes">
</span>I would be very surprised if this was not a part of swift.<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D">*you can hint to the
<span class="SpellE">fs</span> layer how to offset block writes by specifying a stride width which is the number of data carrying disks in the array and the block size typically the default is 64k for raid arrays<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";mso-bidi-font-family:
"Times New Roman";color:#1F497D"><o:p> </o:p></span></font></p>
<div style="border:none;border-left:solid blue 1.5pt;padding:0cm 0cm 0cm 4.0pt">
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><font size="2" face="Tahoma"><span lang="EN-US" style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:
"Times New Roman";mso-ansi-language:EN-US;font-weight:bold">From:</span></font></b><font size="2" face="Tahoma"><span lang="EN-US" style="font-size:10.0pt;font-family:"Tahoma","sans-serif";
mso-fareast-font-family:"Times New Roman";mso-ansi-language:EN-US">
openstack-operators-bounces@lists.openstack.org [mailto:openstack-operators-bounces@lists.openstack.org]
<b><span style="font-weight:bold">On Behalf Of </span></b>Cole Crawford<br>
<b><span style="font-weight:bold">Sent:</span></b> 13 October 2011 13:51<br>
<b><span style="font-weight:bold">To:</span></b> openstack-operators@lists.openstack.org<br>
<b><span style="font-weight:bold">Subject:</span></b> Re: [Openstack-operators] XFS documentation seems to conflict with recommendations in Swift<o:p></o:p></span></font></p>
</div>
</div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
<p style="margin:0cm;margin-bottom:.0001pt"><font size="2" face="Arial"><span style="font-size:10.0pt;font-family:"Arial","sans-serif"">generally mounting with -o nobarrier is a bad idea (ext4 or xfs), unless you have disks that do not have write caches. don't
follow that<o:p></o:p></span></font></p>
<p style="margin:0cm;margin-bottom:.0001pt"><font size="2" face="Arial"><span style="font-size:10.0pt;font-family:"Arial","sans-serif"">recommendation, or for example - fsync won't work which is something swift relies upon.<o:p></o:p></span></font></p>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
</div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt">On Thu, Oct 13, 2011 at 9:18 AM, Marcelo Martins <<a href="mailto:btorch-os@zeroaccess.org">btorch-os@zeroaccess.org</a>> wrote:<o:p></o:p></span></font></p>
<div>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt">Hi Jonathan,<o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt">I guess that will depend on how your storage nodes are configured (hardware wise). The reason why it's recommended is because the storage drives are actually attached
to a controller that has RiW cache enabled. <o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
</div>
<div style="mso-element:para-border-div;border:none;border-bottom:solid #AAAAAA 1.0pt;
mso-border-bottom-alt:solid #AAAAAA .75pt;padding:0cm 0cm 2.0pt 0cm">
<h2 style="mso-margin-top-alt:0cm;margin-right:0cm;margin-bottom:7.2pt;
margin-left:0cm;mso-line-height-alt:14.25pt;border:none;mso-border-bottom-alt:
solid #AAAAAA .75pt;padding:0cm;mso-padding-alt:0cm 0cm 2.0pt 0cm;background-color:
initial">
<b style="mso-ansi-font-weight:normal;mso-bidi-font-weight:normal"><font size="4" color="black" face="Arial"><span style="font-size:14.5pt;font-family:"Arial","sans-serif";
color:black;font-weight:normal">Q. Should barriers be enabled with storage which has
a persistent write cache?<o:p></o:p></span></font></b></h2>
</div>
<div style="margin-top:4.8pt;margin-bottom:6.0pt">
<p class="MsoNormal" style="line-height:18.0pt;background:white"><font size="2" face="Arial"><span style="font-size:10.0pt;font-family:"Arial","sans-serif"">Many hardware RAID have a persistent write cache which preserves it across power failure, interface
resets, system crashes, etc. Using write barriers in this instance is not recommended and will in fact lower performance. Therefore, it is recommended to turn off the barrier support and mount the filesystem with "nobarrier". But take care about the hard disk
write cache, which should be off.<o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
</div>
<div>
<div>
<div>
<p class="MsoNormal"><font size="4" color="black" face="Helvetica"><span style="font-size:13.5pt;font-family:"Helvetica","sans-serif";color:black">Marcelo Martins<o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="4" color="black" face="Helvetica"><span style="font-size:13.5pt;font-family:"Helvetica","sans-serif";color:black">Openstack-swift<o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="4" color="black" face="Helvetica"><span style="font-size:13.5pt;font-family:"Helvetica","sans-serif";color:black"><a href="mailto:btorch-os@zeroaccess.org" target="_blank">btorch-os@zeroaccess.org</a><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="4" color="black" face="Helvetica"><span style="font-size:13.5pt;font-family:"Helvetica","sans-serif";color:black"><o:p> </o:p></span></font></p>
</div>
<div>
<div>
<p class="MsoNormal"><font size="4" color="black" face="Helvetica"><span style="font-size:13.5pt;font-family:"Helvetica","sans-serif";color:black">“Knowledge is the wings on which our aspirations take flight and soar. When it comes to surfing and life if you
know what to do you can do it. If you desire anything become educated about it and succeed. “<o:p></o:p></span></font></p>
</div>
</div>
<div>
<p class="MsoNormal"><font size="4" color="black" face="Helvetica"><span style="font-size:13.5pt;font-family:"Helvetica","sans-serif";color:black"><o:p> </o:p></span></font></p>
</div>
</div>
<p class="MsoNormal" style="margin-bottom:12.0pt"><font size="3" face="Times New Roman"><span style="font-size:12.0pt"><o:p> </o:p></span></font></p>
</div>
<div>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
<div>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt">On Oct 12, 2011, at 10:08 AM, Jonathan Simms wrote:<o:p></o:p></span></font></p>
</div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><br style="mso-special-character:line-break">
<![if !supportLineBreakNewLine]><br style="mso-special-character:line-break">
<![endif]><o:p></o:p></span></font></p>
<div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt">Hello all,<br>
<br>
I'm in the middle of a 120T Swift deployment, and I've had some<br>
concerns about the backing filesystem. I formatted everything with<br>
ext4 with 1024b inodes (for storing xattrs), but the process took so<br>
long that I'm now looking at XFS again. In particular, this concerns<br>
me <a href="http://xfs.org/index.php/XFS_FAQ#Write_barrier_support" target="_blank">
http://xfs.org/index.php/XFS_FAQ#Write_barrier_support</a>.<br>
<br>
In the swift documentation, it's recommended to mount the filesystems<br>
w/ 'nobarrier', but it would seem to me that this would leave the data<br>
open to corruption in the case of a crash. AFAIK, swift doesn't do<br>
checksumming (and checksum checking) of stored data (after it is<br>
written), which would mean that any data corruption would silently get<br>
passed back to the users.<br>
<br>
Now, I haven't had operational experience running XFS in production,<br>
I've mainly used ZFS, JFS, and ext{3,4}. Are there any recommendations<br>
for using XFS safely in production?<br>
_______________________________________________<br>
Openstack-operators mailing list<br>
<a href="mailto:Openstack-operators@lists.openstack.org" target="_blank">Openstack-operators@lists.openstack.org</a><br>
<a href="http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-operators" target="_blank">http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-operators</a><o:p></o:p></span></font></p>
</div>
</div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
</div>
</div>
</div>
<p class="MsoNormal" style="margin-bottom:12.0pt"><font size="3" face="Times New Roman"><span style="font-size:12.0pt"><br>
_______________________________________________<br>
Openstack-operators mailing list<br>
<a href="mailto:Openstack-operators@lists.openstack.org">Openstack-operators@lists.openstack.org</a><br>
<a href="http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-operators" target="_blank">http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-operators</a><o:p></o:p></span></font></p>
</div>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:
12.0pt"><o:p> </o:p></span></font></p>
</div>
</div>
<br>
<hr>
<font face="Arial" color="Gray" size="1">Sophos Limited, The Pentagon, Abingdon Science Park, Abingdon, OX14 3YP, United Kingdom.<br>
Company Reg No 2096520. VAT Reg No GB 991 2418 08.<br>
</font>
</body>
</html>