<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="ProgId" content="Word.Document">
<meta name="Generator" content="Microsoft Word 14">
<meta name="Originator" content="Microsoft Word 14">
<link rel="File-List" href="cid:filelist.xml@01CCEBEC.EBB0BF20"><link rel="Edit-Time-Data" href="cid:editdata.mso"><!--[if !mso]><style>v\:* {behavior:url(#default#VML);}
o\:* {behavior:url(#default#VML);}
w\:* {behavior:url(#default#VML);}
.shape {behavior:url(#default#VML);}
</style><![endif]--><!--[if gte mso 9]><xml>
<o:OfficeDocumentSettings>
<o:AllowPNG/>
<o:DoNotRelyOnCSS/>
</o:OfficeDocumentSettings>
</xml><![endif]--><!--[if gte mso 9]><xml>
<w:WordDocument>
<w:Zoom>120</w:Zoom>
<w:SpellingState>Clean</w:SpellingState>
<w:TrackMoves/>
<w:TrackFormatting/>
<w:EnvelopeVis/>
<w:ValidateAgainstSchemas/>
<w:SaveIfXMLInvalid>false</w:SaveIfXMLInvalid>
<w:IgnoreMixedContent>false</w:IgnoreMixedContent>
<w:AlwaysShowPlaceholderText>false</w:AlwaysShowPlaceholderText>
<w:DoNotPromoteQF/>
<w:LidThemeOther>EN-US</w:LidThemeOther>
<w:LidThemeAsian>X-NONE</w:LidThemeAsian>
<w:LidThemeComplexScript>X-NONE</w:LidThemeComplexScript>
<w:Compatibility>
<w:DoNotExpandShiftReturn/>
<w:BreakWrappedTables/>
<w:SplitPgBreakAndParaMark/>
<w:EnableOpenTypeKerning/>
</w:Compatibility>
<w:BrowserLevel>MicrosoftInternetExplorer4</w:BrowserLevel>
<m:mathPr>
<m:mathFont m:val="Cambria Math"/>
<m:brkBin m:val="before"/>
<m:brkBinSub m:val="--"/>
<m:smallFrac m:val="off"/>
<m:dispDef/>
<m:lMargin m:val="0"/>
<m:rMargin m:val="0"/>
<m:defJc m:val="centerGroup"/>
<m:wrapIndent m:val="1440"/>
<m:intLim m:val="subSup"/>
<m:naryLim m:val="undOvr"/>
</m:mathPr></w:WordDocument>
</xml><![endif]--><!--[if gte mso 9]><xml>
<w:LatentStyles DefLockedState="false" DefUnhideWhenUsed="true" DefSemiHidden="true" DefQFormat="false" DefPriority="99" LatentStyleCount="267">
<w:LsdException Locked="false" Priority="0" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Normal"/>
<w:LsdException Locked="false" Priority="9" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="heading 1"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 2"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 3"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 4"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 5"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 6"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 7"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 8"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 9"/>
<w:LsdException Locked="false" Priority="39" Name="toc 1"/>
<w:LsdException Locked="false" Priority="39" Name="toc 2"/>
<w:LsdException Locked="false" Priority="39" Name="toc 3"/>
<w:LsdException Locked="false" Priority="39" Name="toc 4"/>
<w:LsdException Locked="false" Priority="39" Name="toc 5"/>
<w:LsdException Locked="false" Priority="39" Name="toc 6"/>
<w:LsdException Locked="false" Priority="39" Name="toc 7"/>
<w:LsdException Locked="false" Priority="39" Name="toc 8"/>
<w:LsdException Locked="false" Priority="39" Name="toc 9"/>
<w:LsdException Locked="false" Priority="35" QFormat="true" Name="caption"/>
<w:LsdException Locked="false" Priority="10" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Title"/>
<w:LsdException Locked="false" Priority="1" Name="Default Paragraph Font"/>
<w:LsdException Locked="false" Priority="11" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Subtitle"/>
<w:LsdException Locked="false" Priority="22" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Strong"/>
<w:LsdException Locked="false" Priority="20" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Emphasis"/>
<w:LsdException Locked="false" Priority="59" SemiHidden="false" UnhideWhenUsed="false" Name="Table Grid"/>
<w:LsdException Locked="false" UnhideWhenUsed="false" Name="Placeholder Text"/>
<w:LsdException Locked="false" Priority="1" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="No Spacing"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 1"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 1"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 1"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 1"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 1"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 1"/>
<w:LsdException Locked="false" UnhideWhenUsed="false" Name="Revision"/>
<w:LsdException Locked="false" Priority="34" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="List Paragraph"/>
<w:LsdException Locked="false" Priority="29" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Quote"/>
<w:LsdException Locked="false" Priority="30" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Intense Quote"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 1"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 1"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 1"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 1"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 1"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 1"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 1"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 1"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 2"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 2"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 2"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 2"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 2"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 2"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 2"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 2"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 2"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 2"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 2"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 2"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 2"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 2"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 3"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 3"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 3"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 3"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 3"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 3"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 3"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 3"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 3"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 3"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 3"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 3"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 3"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 3"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 4"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 4"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 4"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 4"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 4"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 4"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 4"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 4"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 4"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 4"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 4"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 4"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 4"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 4"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 5"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 5"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 5"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 5"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 5"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 5"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 5"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 5"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 5"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 5"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 5"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 5"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 5"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 5"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 6"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 6"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 6"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 6"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 6"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 6"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 6"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 6"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 6"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 6"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 6"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 6"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 6"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 6"/>
<w:LsdException Locked="false" Priority="19" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Subtle Emphasis"/>
<w:LsdException Locked="false" Priority="21" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Intense Emphasis"/>
<w:LsdException Locked="false" Priority="31" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Subtle Reference"/>
<w:LsdException Locked="false" Priority="32" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Intense Reference"/>
<w:LsdException Locked="false" Priority="33" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Book Title"/>
<w:LsdException Locked="false" Priority="37" Name="Bibliography"/>
<w:LsdException Locked="false" Priority="39" QFormat="true" Name="TOC Heading"/>
</w:LatentStyles>
</xml><![endif]--><style><!--
/* Font Definitions */
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;
mso-font-alt:"Times New Roman";
mso-font-charset:0;
mso-generic-font-family:swiss;
mso-font-pitch:variable;
mso-font-signature:-520092929 1073786111 9 0 415 0;}
@font-face
{font-family:Tahoma;
panose-1:2 11 6 4 3 5 4 4 2 4;
mso-font-alt:Verdana;
mso-font-charset:0;
mso-generic-font-family:swiss;
mso-font-pitch:variable;
mso-font-signature:-520081665 -1073717157 41 0 66047 0;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{mso-style-unhide:no;
mso-style-qformat:yes;
mso-style-parent:"";
margin:0in;
margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:11.0pt;
font-family:"Calibri","sans-serif";
mso-fareast-font-family:Calibri;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;
text-underline:single;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-noshow:yes;
mso-style-priority:99;
color:purple;
text-decoration:underline;
text-underline:single;}
p
{mso-style-noshow:yes;
mso-style-priority:99;
margin:0in;
margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:12.0pt;
font-family:"Times New Roman","serif";
mso-fareast-font-family:Calibri;}
p.MsoAcetate, li.MsoAcetate, div.MsoAcetate
{mso-style-noshow:yes;
mso-style-priority:99;
mso-style-link:"Balloon Text Char";
margin:0in;
margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:8.0pt;
font-family:"Tahoma","sans-serif";
mso-fareast-font-family:Calibri;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-noshow:yes;
mso-style-priority:34;
mso-style-unhide:no;
mso-style-qformat:yes;
margin-top:0in;
margin-right:0in;
margin-bottom:0in;
margin-left:.5in;
margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:11.0pt;
font-family:"Calibri","sans-serif";
mso-fareast-font-family:Calibri;}
span.BalloonTextChar
{mso-style-name:"Balloon Text Char";
mso-style-noshow:yes;
mso-style-priority:99;
mso-style-unhide:no;
mso-style-locked:yes;
mso-style-link:"Balloon Text";
mso-ansi-font-size:8.0pt;
mso-bidi-font-size:8.0pt;
font-family:"Tahoma","sans-serif";
mso-ascii-font-family:Tahoma;
mso-fareast-font-family:Calibri;
mso-hansi-font-family:Tahoma;
mso-bidi-font-family:Tahoma;}
p.msochpdefault, li.msochpdefault, div.msochpdefault
{mso-style-name:msochpdefault;
mso-style-noshow:yes;
mso-style-priority:99;
mso-style-unhide:no;
margin:0in;
margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:10.0pt;
font-family:"Calibri","sans-serif";
mso-fareast-font-family:Calibri;}
span.balloontextchar0
{mso-style-name:balloontextchar;
mso-style-unhide:no;
font-family:"Tahoma","sans-serif";
mso-ascii-font-family:Tahoma;
mso-hansi-font-family:Tahoma;
mso-bidi-font-family:Tahoma;}
span.emailstyle18
{mso-style-name:emailstyle18;
mso-style-unhide:no;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:windowtext;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.emailstyle19
{mso-style-name:emailstyle19;
mso-style-unhide:no;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:#1F497D;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.emailstyle22
{mso-style-name:emailstyle22;
mso-style-unhide:no;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:#1F497D;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.balloontextchar00
{mso-style-name:balloontextchar0;
mso-style-unhide:no;
font-family:"Tahoma","sans-serif";
mso-ascii-font-family:Tahoma;
mso-hansi-font-family:Tahoma;
mso-bidi-font-family:Tahoma;}
span.emailstyle26
{mso-style-name:emailstyle26;
mso-style-unhide:no;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:#1F497D;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.emailstyle28
{mso-style-name:emailstyle28;
mso-style-unhide:no;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:#1F497D;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.EmailStyle29
{mso-style-type:personal;
mso-style-noshow:yes;
mso-style-unhide:no;
mso-ansi-font-size:10.0pt;
mso-bidi-font-size:10.0pt;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:#1F497D;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.EmailStyle30
{mso-style-type:personal-reply;
mso-style-noshow:yes;
mso-style-unhide:no;
mso-ansi-font-size:10.0pt;
mso-bidi-font-size:10.0pt;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:#1F497D;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
.MsoChpDefault
{mso-style-type:export-only;
mso-default-props:yes;
font-size:10.0pt;
mso-ansi-font-size:10.0pt;
mso-bidi-font-size:10.0pt;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;
mso-header-margin:.5in;
mso-footer-margin:.5in;
mso-paper-source:0;}
div.WordSection1
{page:WordSection1;}
--></style><!--[if gte mso 10]><style>/* Style Definitions */
table.MsoNormalTable
{mso-style-name:"Table Normal";
mso-tstyle-rowband-size:0;
mso-tstyle-colband-size:0;
mso-style-noshow:yes;
mso-style-priority:99;
mso-style-parent:"";
mso-padding-alt:0in 5.4pt 0in 5.4pt;
mso-para-margin:0in;
mso-para-margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:10.0pt;
font-family:"Calibri","sans-serif";}
</style><![endif]--><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="EN-US" link="blue" vlink="purple" style="tab-interval:.5in">
<div class="WordSection1">
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Ugh, small but important correction below<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal"><b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";font-weight:bold">From:</span></font></b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">
nvmewin-bounces@lists.openfabrics.org [mailto:nvmewin-bounces@lists.openfabrics.org]
<b><span style="font-weight:bold">On Behalf Of </span></b>Luse, Paul E<br>
<b><span style="font-weight:bold">Sent:</span></b> Wednesday, February 15, 2012 2:16 PM<br>
<b><span style="font-weight:bold">To:</span></b> Greg de Valois; nvmewin@lists.openfabrics.org<br>
<b><span style="font-weight:bold">Subject:</span></b> Re: [nvmewin] Question on Processor to MSI vector mapping<o:p></o:p></span></font></p>
</div>
</div>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:11.0pt"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">The only difference between what we’re doing and what you get with either the storport API or the ‘learning method’ I mention is the locality of
the SQ memory when the IO is submitted. <span style="mso-spacerun:yes"> </span>With what we’re doing now, we know that the submission core is local to SQ mem.<span style="mso-spacerun:yes">
</span>We don’t have that guarantee otherwise.<span style="mso-spacerun:yes"> </span>
Its negligible I believe which is why I’m proposing we move to the learning mode (very small cod change, ‘ve made it and tested).<span style="mso-spacerun:yes">
</span>I prefer learning mode over the storport API because it works regardless of OS dependencies, APIC modes, etc.<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">As an example of the tiny SQ mem difference, take this scenario<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Core 0:<span style="mso-spacerun:yes">
</span>SQ0 and CQ0 are allocated local to core 0.<span style="mso-spacerun:yes">
</span><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Core 1:<span style="mso-spacerun:yes">
</span>SQ1 and CQ1 are allocated local to core 1.<span style="mso-spacerun:yes">
</span><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Core 2:<span style="mso-spacerun:yes">
</span>SQ2 and CQ2 are allocated local to core 2.<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">In our current implementation if we get an IO coming in on Core 1:<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">We submit it to SQ1 (local mem), when we created CQ we knew the vector so we also know that this IO will complete on CQ1<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Using the storport API, if we get an IO coming in Core 1:<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">There’s no assurance that we’ll be using SQ1.<span style="mso-spacerun:yes">
</span>If the API tells us that the vector is 2, for example, we’d then choose to submit on *<b style="mso-bidi-font-weight:normal"><span style="font-weight:bold;mso-bidi-font-weight:normal">SQ2*</span></b> which is local to core 2, not core 1.<span style="mso-spacerun:yes">
</span>On the completion side, everything is just as optimal.<span style="mso-spacerun:yes">
</span>The IO completes on core 2 which both SQ/CQ for core2 are local to core 2<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Does that help?<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Thx<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Paul<span style="mso-spacerun:yes">
</span><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-outline-level:1"><b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";font-weight:bold">From:</span></font></b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">
Greg de Valois <a href="mailto:[mailto:Greg.deValois@sandisk.com]">[mailto:Greg.deValois@sandisk.com]</a>
<br>
<b><span style="font-weight:bold">Sent:</span></b> Wednesday, February 15, 2012 2:03 PM<br>
<b><span style="font-weight:bold">To:</span></b> Luse, Paul E; <a href="mailto:nvmewin@lists.openfabrics.org">
nvmewin@lists.openfabrics.org</a><br>
<b><span style="font-weight:bold">Subject:</span></b> RE: Question on Processor to MSI vector mapping<o:p></o:p></span></font></p>
</div>
</div>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:11.0pt"><o:p> </o:p></span></font></p>
<div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">I'm not understanding why it matters which core has been assigned which vector,
as we have IO associated with a vector, put onto a queue associated with that vector, and the controller completes the IO using that vector.<o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black"> <o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">Why should the driver and controller care what the mapping of vectors to processors
is?<o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black"> <o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">Thanks, Greg<o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black"> <o:p></o:p></span></font></p>
</div>
<div id="divRpF210046">
<div class="MsoNormal" align="center" style="text-align:center"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">
<hr size="3" width="100%" align="center">
</span></font></div>
<p class="MsoNormal" style="margin-bottom:12.0pt;mso-outline-level:1"><b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black;font-weight:bold">From:</span></font></b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">
Luse, Paul E [paul.e.luse@intel.com]<br>
<b><span style="font-weight:bold">Sent:</span></b> Wednesday, February 15, 2012 12:46 PM<br>
<b><span style="font-weight:bold">To:</span></b> Greg de Valois; <a href="mailto:nvmewin@lists.openfabrics.org">
nvmewin@lists.openfabrics.org</a><br>
<b><span style="font-weight:bold">Subject:</span></b> RE: Question on Processor to MSI vector mapping<o:p></o:p></span></font></p>
</div>
<div>
<div>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">I think the point you’re missing (or I’m not understanding) is
<i><span style="font-style:italic">when the driver creates the CQ it tells the device which MSIX vector to use for that CQ</span></i>. We don’t know that at the time we create the CQ. All we know when we create the CQ is the number of vectors and the vector
numbers but we don’t know which one is associated with which core. The API only provides that info on a per IO basis. So, everything you describe is correct however the downside is (not that big of a deal but as compared to what we’re doing now) that because
we don’t know the mapping when we create the SQ/CQ pair we may be placing the IO on a different core than the SQ is associated with because the API told us about a vector that is associated with a different Q pair than that we setup for the current core.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Feel free to call me if that would be easier – 480 554 3688</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Thx</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Paul</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-outline-level:1"><b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black;font-weight:bold">From:</span></font></b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">
Greg de Valois <a href="mailto:[mailto:Greg.deValois@sandisk.com]">[mailto:Greg.deValois@sandisk.com]</a>
<br>
<b><span style="font-weight:bold">Sent:</span></b> Wednesday, February 15, 2012 1:40 PM<br>
<b><span style="font-weight:bold">To:</span></b> Luse, Paul E; <a href="mailto:nvmewin@lists.openfabrics.org">
nvmewin@lists.openfabrics.org</a><br>
<b><span style="font-weight:bold">Subject:</span></b> RE: Question on Processor to MSI vector mapping</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
</div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">My understanding is that Windows assigns the number of MSI vectors requested (if possible) by the INF to an adapter,
each of which is associated with one of the available processors. The driver sets up a queue pair for each of those vectors, and when an IO comes, it is put on the queue associated with the vector that is returned from the API call. This vector is one that
has been assigned to the current executing core. As long as that API works correctly, the driver doesn't have to do anything more to assure that the IO is submitted and completed on that vector.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">Is there some part of those statements that isn't correct?</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">I'm a little confused here.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">Greg</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
</div>
<div id="divRpF257901">
<div class="MsoNormal" align="center" style="text-align:center"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">
<hr size="3" width="100%" align="center">
</span></font></div>
<p class="MsoNormal" style="margin-bottom:12.0pt;mso-outline-level:1"><b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black;font-weight:bold">From:</span></font></b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">
Luse, Paul E [paul.e.luse@intel.com]<br>
<b><span style="font-weight:bold">Sent:</span></b> Wednesday, February 15, 2012 11:14 AM<br>
<b><span style="font-weight:bold">To:</span></b> Greg de Valois; <a href="mailto:nvmewin@lists.openfabrics.org">
nvmewin@lists.openfabrics.org</a><br>
<b><span style="font-weight:bold">Subject:</span></b> RE: Question on Processor to MSI vector mapping</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<div>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Because we create the SQ and CQ before the first IO and we have to provide the CQ vector when created. So, for example lets say we have 3 cores
and an SQ/CQ pair all numbered the same as the core #. When we create them we arbitrarily give the CQ a vector to complete on. When the first OI comes lets say its on core 1 and the stoport PA tells us the vector we should expect it on is 3, we’d need to
submit on SQ3 but we’re on core 1. There are ways around this of course, we could do some creative things in passiveInit wrt the creation of the CQ’s and submitting test IOs, re-creating, etc., but those things I don’t think are worth the complexity.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Make sense?</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Thx</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Paul</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-outline-level:1"><b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black;font-weight:bold">From:</span></font></b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">
Greg de Valois <a href="mailto:[mailto:Greg.deValois@sandisk.com]">[mailto:Greg.deValois@sandisk.com]</a>
<br>
<b><span style="font-weight:bold">Sent:</span></b> Wednesday, February 15, 2012 12:06 PM<br>
<b><span style="font-weight:bold">To:</span></b> Luse, Paul E; <a href="mailto:nvmewin@lists.openfabrics.org">
nvmewin@lists.openfabrics.org</a><br>
<b><span style="font-weight:bold">Subject:</span></b> RE: Question on Processor to MSI vector mapping</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
</div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">Paul:</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">Thanks for the reply. I'm afraid I'm not quite following you: why does using the API provided by Storport to get the
MSI vector associated with the current request imply that you're submitting on a different core than completing? The core you're executing on is the one assigned the vector that the API gives you, you put it on the submission queue that has been assigned to
that vector, and it's completed on the completion queue for that vector as well.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">What am I missing here?</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">Greg</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
</div>
<div id="divRpF346993">
<div class="MsoNormal" align="center" style="text-align:center"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">
<hr size="3" width="100%" align="center">
</span></font></div>
<p class="MsoNormal" style="margin-bottom:12.0pt;mso-outline-level:1"><b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black;font-weight:bold">From:</span></font></b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">
Luse, Paul E [paul.e.luse@intel.com]<br>
<b><span style="font-weight:bold">Sent:</span></b> Wednesday, February 15, 2012 10:57 AM<br>
<b><span style="font-weight:bold">To:</span></b> Greg de Valois; <a href="mailto:nvmewin@lists.openfabrics.org">
nvmewin@lists.openfabrics.org</a><br>
<b><span style="font-weight:bold">Subject:</span></b> RE: Question on Processor to MSI vector mapping</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<div>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Hi Greg-</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Thanks for the question, it’s a good one! The reasoning was mainly because it was fairly straightforward and had one benefit over the method you
mention – we could assure NUMA optimization and vector matching for both SQ and CQ. There are many, many ways that one could approach this problem and we discussed a few as part of the dev of this driver and then individually discussed experiences with various
methods with other dev teams at our respective companies.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">If one uses the method you mention below, we’d create our SQ/CQ per core NUMA optimized and then be submitting on a different core than we complete
on however we’d still be optimizing the completion side. The more I thought about this the more I realize its actually not buying us much of anything over using the Msft API due to the effects of CPU cache and the fact that the SQ access on submission is
a write and not a read. I also heard from various other folks that they didn’t find the API below to be accurate all of the time, I can’t say from experience that I’ve seen this.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">That said, I will likely be proposing an alternate method in the near future so I’ll go ahead and propose it now since you brought up the subject:</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Proposal: no long decompose the MSI address to populate the mapping table. Instead, start off with a 1:1 mapping and ‘learn and update’ the mapping
table on the completion side. Would still avoid the storport API because I don’t think it adds value over the learned method and requires us to use the DPC steering option which I’ve witnessed to have unpredictable side effects. I do plan on following up
with Msft (and have already had several internal discussions at Intel with other storport devs) on exactly how some of these optimizations within storport so we can better gauge whether they’re a good fit for us or now.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Pro: This will *<b><span style="font-weight:bold">always</span></b>* work whereas the method we have no does not work for APIC logical mode. I
prefer a simple “one size fits all” solution every time over a 2 path solution even if one path is slightly optimized. It makes the driver more maintainable and gives us less variables in debug (right now we don’t even store whether we found the APIC in phy
or logical mode so during debug you don’t really know).</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Con: SQ mem will be on a different core than the submitting thread but I don’t believe this is a measurable issue. Certainly can perform some experiments
to check though</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Other thoughts?</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Thx</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Paul</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-outline-level:1"><b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black;font-weight:bold">From:</span></font></b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">
<a href="mailto:nvmewin-bounces@lists.openfabrics.org">nvmewin-bounces@lists.openfabrics.org</a>
<a href="mailto:[mailto:nvmewin-bounces@lists.openfabrics.org]">[mailto:nvmewin-bounces@lists.openfabrics.org]</a>
<b><span style="font-weight:bold">On Behalf Of </span></b>Greg de Valois<br>
<b><span style="font-weight:bold">Sent:</span></b> Tuesday, February 14, 2012 5:02 PM<br>
<b><span style="font-weight:bold">To:</span></b> <a href="mailto:nvmewin@lists.openfabrics.org">
nvmewin@lists.openfabrics.org</a><br>
<b><span style="font-weight:bold">Subject:</span></b> [nvmewin] Question on Processor to MSI vector mapping</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
</div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<div>
<div>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:#1F497D">All:</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
</div>
<div>
<div>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:#1F497D">I'm wondering if anyone can explain to me the reasoning behind the processor to MSI vector translation that is being
done by the driver, instead of using the vector returned from StorPortGetStartIoPerfParams for each IO? Are there cases where this doesn't work properly?</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:#1F497D">Thanks,
</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:#1F497D">Greg de Valois</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:#1F497D">SanDisk</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
</div>
</div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<div class="MsoNormal" align="center" style="text-align:center"><font size="3" color="black" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:black">
<hr size="3" width="100%" align="center">
</span></font></div>
<p class="MsoNormal" style="margin-bottom:12.0pt"><font size="2" color="gray" face="Arial"><span style="font-size:10.0pt;font-family:"Arial","sans-serif";color:gray">PLEASE NOTE: The information contained in this electronic mail message is intended only for
the use of the designated recipient(s) named above. If the reader of this message is not the intended recipient, you are hereby notified that you have received this message in error and that any review, dissemination, distribution, or copying of this message
is strictly prohibited. If you have received this communication in error, please notify the sender by telephone or e-mail (as shown above) immediately and destroy any and all copies of this message in your possession (whether hard copies or electronically
stored copies).</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</body>
</html>