<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<meta name="ProgId" content="Word.Document">
<meta name="Generator" content="Microsoft Word 14">
<meta name="Originator" content="Microsoft Word 14">
<link rel="File-List" href="cid:filelist.xml@01CD0E69.D0055DE0"><link rel="Edit-Time-Data" href="cid:editdata.mso"><!--[if !mso]><style>v\:* {behavior:url(#default#VML);}
o\:* {behavior:url(#default#VML);}
w\:* {behavior:url(#default#VML);}
.shape {behavior:url(#default#VML);}
</style><![endif]--><!--[if gte mso 9]><xml>
<o:OfficeDocumentSettings>
<o:AllowPNG/>
<o:DoNotRelyOnCSS/>
</o:OfficeDocumentSettings>
</xml><![endif]--><!--[if gte mso 9]><xml>
<w:WordDocument>
<w:Zoom>120</w:Zoom>
<w:SpellingState>Clean</w:SpellingState>
<w:TrackMoves/>
<w:TrackFormatting/>
<w:EnvelopeVis/>
<w:ValidateAgainstSchemas/>
<w:SaveIfXMLInvalid>false</w:SaveIfXMLInvalid>
<w:IgnoreMixedContent>false</w:IgnoreMixedContent>
<w:AlwaysShowPlaceholderText>false</w:AlwaysShowPlaceholderText>
<w:DoNotPromoteQF/>
<w:LidThemeOther>EN-US</w:LidThemeOther>
<w:LidThemeAsian>X-NONE</w:LidThemeAsian>
<w:LidThemeComplexScript>X-NONE</w:LidThemeComplexScript>
<w:Compatibility>
<w:DoNotExpandShiftReturn/>
<w:BreakWrappedTables/>
<w:SnapToGridInCell/>
<w:WrapTextWithPunct/>
<w:UseAsianBreakRules/>
<w:DontGrowAutofit/>
<w:SplitPgBreakAndParaMark/>
<w:EnableOpenTypeKerning/>
<w:DontFlipMirrorIndents/>
<w:OverrideTableStyleHps/>
</w:Compatibility>
<w:BrowserLevel>MicrosoftInternetExplorer4</w:BrowserLevel>
<m:mathPr>
<m:mathFont m:val="Cambria Math"/>
<m:brkBin m:val="before"/>
<m:brkBinSub m:val="--"/>
<m:smallFrac m:val="off"/>
<m:dispDef/>
<m:lMargin m:val="0"/>
<m:rMargin m:val="0"/>
<m:defJc m:val="centerGroup"/>
<m:wrapIndent m:val="1440"/>
<m:intLim m:val="subSup"/>
<m:naryLim m:val="undOvr"/>
</m:mathPr></w:WordDocument>
</xml><![endif]--><!--[if gte mso 9]><xml>
<w:LatentStyles DefLockedState="false" DefUnhideWhenUsed="true" DefSemiHidden="true" DefQFormat="false" DefPriority="99" LatentStyleCount="267">
<w:LsdException Locked="false" Priority="0" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Normal"/>
<w:LsdException Locked="false" Priority="9" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="heading 1"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 2"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 3"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 4"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 5"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 6"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 7"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 8"/>
<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 9"/>
<w:LsdException Locked="false" Priority="39" Name="toc 1"/>
<w:LsdException Locked="false" Priority="39" Name="toc 2"/>
<w:LsdException Locked="false" Priority="39" Name="toc 3"/>
<w:LsdException Locked="false" Priority="39" Name="toc 4"/>
<w:LsdException Locked="false" Priority="39" Name="toc 5"/>
<w:LsdException Locked="false" Priority="39" Name="toc 6"/>
<w:LsdException Locked="false" Priority="39" Name="toc 7"/>
<w:LsdException Locked="false" Priority="39" Name="toc 8"/>
<w:LsdException Locked="false" Priority="39" Name="toc 9"/>
<w:LsdException Locked="false" Priority="35" QFormat="true" Name="caption"/>
<w:LsdException Locked="false" Priority="10" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Title"/>
<w:LsdException Locked="false" Priority="1" Name="Default Paragraph Font"/>
<w:LsdException Locked="false" Priority="11" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Subtitle"/>
<w:LsdException Locked="false" Priority="22" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Strong"/>
<w:LsdException Locked="false" Priority="20" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Emphasis"/>
<w:LsdException Locked="false" Priority="59" SemiHidden="false" UnhideWhenUsed="false" Name="Table Grid"/>
<w:LsdException Locked="false" UnhideWhenUsed="false" Name="Placeholder Text"/>
<w:LsdException Locked="false" Priority="1" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="No Spacing"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 1"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 1"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 1"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 1"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 1"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 1"/>
<w:LsdException Locked="false" UnhideWhenUsed="false" Name="Revision"/>
<w:LsdException Locked="false" Priority="34" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="List Paragraph"/>
<w:LsdException Locked="false" Priority="29" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Quote"/>
<w:LsdException Locked="false" Priority="30" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Intense Quote"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 1"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 1"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 1"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 1"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 1"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 1"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 1"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 1"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 2"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 2"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 2"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 2"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 2"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 2"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 2"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 2"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 2"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 2"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 2"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 2"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 2"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 2"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 3"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 3"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 3"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 3"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 3"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 3"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 3"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 3"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 3"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 3"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 3"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 3"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 3"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 3"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 4"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 4"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 4"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 4"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 4"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 4"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 4"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 4"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 4"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 4"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 4"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 4"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 4"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 4"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 5"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 5"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 5"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 5"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 5"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 5"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 5"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 5"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 5"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 5"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 5"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 5"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 5"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 5"/>
<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 6"/>
<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 6"/>
<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 6"/>
<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 6"/>
<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 6"/>
<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 6"/>
<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 6"/>
<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 6"/>
<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 6"/>
<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 6"/>
<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 6"/>
<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 6"/>
<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 6"/>
<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 6"/>
<w:LsdException Locked="false" Priority="19" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Subtle Emphasis"/>
<w:LsdException Locked="false" Priority="21" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Intense Emphasis"/>
<w:LsdException Locked="false" Priority="31" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Subtle Reference"/>
<w:LsdException Locked="false" Priority="32" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Intense Reference"/>
<w:LsdException Locked="false" Priority="33" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Book Title"/>
<w:LsdException Locked="false" Priority="37" Name="Bibliography"/>
<w:LsdException Locked="false" Priority="39" QFormat="true" Name="TOC Heading"/>
</w:LatentStyles>
</xml><![endif]--><style><!--
/* Font Definitions */
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;
mso-font-alt:Haettenschweiler;
mso-font-charset:2;
mso-generic-font-family:auto;
mso-font-pitch:variable;
mso-font-signature:0 268435456 0 0 -2147483648 0;}
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;
mso-font-alt:Haettenschweiler;
mso-font-charset:2;
mso-generic-font-family:auto;
mso-font-pitch:variable;
mso-font-signature:0 268435456 0 0 -2147483648 0;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;
mso-font-alt:"Century Gothic";
mso-font-charset:0;
mso-generic-font-family:swiss;
mso-font-pitch:variable;
mso-font-signature:-520092929 1073786111 9 0 415 0;}
@font-face
{font-family:Tahoma;
panose-1:2 11 6 4 3 5 4 4 2 4;
mso-font-alt:Verdana;
mso-font-charset:0;
mso-generic-font-family:swiss;
mso-font-pitch:variable;
mso-font-signature:-520081665 -1073717157 41 0 66047 0;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{mso-style-unhide:no;
mso-style-qformat:yes;
mso-style-parent:"";
margin:0in;
margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:11.0pt;
font-family:"Calibri","sans-serif";
mso-fareast-font-family:Calibri;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;
text-underline:single;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-noshow:yes;
mso-style-priority:99;
color:purple;
text-decoration:underline;
text-underline:single;}
p.MsoAcetate, li.MsoAcetate, div.MsoAcetate
{mso-style-noshow:yes;
mso-style-priority:99;
mso-style-link:"Balloon Text Char";
margin:0in;
margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:8.0pt;
font-family:"Tahoma","sans-serif";
mso-fareast-font-family:Calibri;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
mso-style-unhide:no;
mso-style-qformat:yes;
margin-top:0in;
margin-right:0in;
margin-bottom:0in;
margin-left:.5in;
margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:11.0pt;
font-family:"Calibri","sans-serif";
mso-fareast-font-family:Calibri;}
span.BalloonTextChar
{mso-style-name:"Balloon Text Char";
mso-style-noshow:yes;
mso-style-priority:99;
mso-style-unhide:no;
mso-style-locked:yes;
mso-style-link:"Balloon Text";
mso-ansi-font-size:8.0pt;
mso-bidi-font-size:8.0pt;
font-family:"Tahoma","sans-serif";
mso-ascii-font-family:Tahoma;
mso-fareast-font-family:Calibri;
mso-hansi-font-family:Tahoma;
mso-bidi-font-family:Tahoma;}
p.msochpdefault, li.msochpdefault, div.msochpdefault
{mso-style-name:msochpdefault;
mso-style-unhide:no;
mso-margin-top-alt:auto;
margin-right:0in;
mso-margin-bottom-alt:auto;
margin-left:0in;
mso-pagination:widow-orphan;
font-size:10.0pt;
font-family:"Calibri","sans-serif";
mso-fareast-font-family:Calibri;}
span.balloontextchar0
{mso-style-name:balloontextchar;
mso-style-unhide:no;
font-family:"Tahoma","sans-serif";
mso-ascii-font-family:Tahoma;
mso-hansi-font-family:Tahoma;
mso-bidi-font-family:Tahoma;}
span.emailstyle20
{mso-style-name:emailstyle20;
mso-style-unhide:no;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:windowtext;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.emailstyle21
{mso-style-name:emailstyle21;
mso-style-unhide:no;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:#1F497D;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.emailstyle22
{mso-style-name:emailstyle22;
mso-style-unhide:no;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:#1F497D;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.emailstyle23
{mso-style-name:emailstyle23;
mso-style-unhide:no;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:#1F497D;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.EmailStyle26
{mso-style-type:personal;
mso-style-noshow:yes;
mso-style-unhide:no;
mso-ansi-font-size:10.0pt;
mso-bidi-font-size:10.0pt;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:#1F497D;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.EmailStyle27
{mso-style-type:personal;
mso-style-noshow:yes;
mso-style-unhide:no;
mso-ansi-font-size:10.0pt;
mso-bidi-font-size:10.0pt;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:#1F497D;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.EmailStyle28
{mso-style-type:personal-reply;
mso-style-noshow:yes;
mso-style-unhide:no;
mso-ansi-font-size:10.0pt;
mso-bidi-font-size:10.0pt;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;
color:#1F497D;
mso-text-animation:none;
font-weight:normal;
font-style:normal;
text-decoration:none;
text-underline:none;
text-decoration:none;
text-line-through:none;}
span.SpellE
{mso-style-name:"";
mso-spl-e:yes;}
.MsoChpDefault
{mso-style-type:export-only;
mso-default-props:yes;
font-size:10.0pt;
mso-ansi-font-size:10.0pt;
mso-bidi-font-size:10.0pt;
font-family:"Calibri","sans-serif";
mso-ascii-font-family:Calibri;
mso-hansi-font-family:Calibri;
mso-bidi-font-family:Calibri;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;
mso-header-margin:.5in;
mso-footer-margin:.5in;
mso-paper-source:0;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:1748576883;
mso-list-type:hybrid;
mso-list-template-ids:-199843352 -901881866 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l0:level1
{mso-level-start-at:0;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:.75in;
text-indent:-.25in;
font-family:"Calibri","sans-serif";
mso-fareast-font-family:Calibri;}
@list l0:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:1.25in;
text-indent:-.25in;
font-family:"Courier New";}
@list l0:level3
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:1.75in;
text-indent:-.25in;
font-family:Wingdings;}
@list l0:level4
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:2.25in;
text-indent:-.25in;
font-family:Symbol;}
@list l0:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:2.75in;
text-indent:-.25in;
font-family:"Courier New";}
@list l0:level6
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:3.25in;
text-indent:-.25in;
font-family:Wingdings;}
@list l0:level7
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:3.75in;
text-indent:-.25in;
font-family:Symbol;}
@list l0:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:4.25in;
text-indent:-.25in;
font-family:"Courier New";}
@list l0:level9
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:4.75in;
text-indent:-.25in;
font-family:Wingdings;}
ol
{margin-bottom:0in;}
ul
{margin-bottom:0in;}
--></style><!--[if gte mso 10]><style>/* Style Definitions */
table.MsoNormalTable
{mso-style-name:"Table Normal";
mso-tstyle-rowband-size:0;
mso-tstyle-colband-size:0;
mso-style-noshow:yes;
mso-style-priority:99;
mso-style-parent:"";
mso-padding-alt:0in 5.4pt 0in 5.4pt;
mso-para-margin:0in;
mso-para-margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:10.0pt;
font-family:"Calibri","sans-serif";}
</style><![endif]--><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="EN-US" link="blue" vlink="purple" style="tab-interval:.5in">
<div class="WordSection1">
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">OK, all tested again and working (same tests as before).<span style="mso-spacerun:yes">
</span>Pw is ofanvme123 and you’ll need to update your <span class="SpellE">qemu</span> for the learning stuff to work (drier will still work but with only 1 queue)<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Below is the list of changes to the patch.<span style="mso-spacerun:yes">
</span>I’ll schedule a call to review but don’t plan on reviewing the changes one by one (we can of course if folks would prefer but there are a lot of small changes and it will take a while plus distract from the main functional differences) instead I’ll walk
through the <span class="SpellE">init</span> path for learning and subsequent IO submission/completion.<span style="mso-spacerun:yes">
</span>I’ll have a live debugger session going to demonstrate the changes in action as needed.<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">nvmStd.h</span></font></span><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:.75in;text-indent:-.25in;mso-list:l0 level1 lfo2">
<![if !supportLists]><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">
</span></font></span></span></font><![endif]><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D">Removed elements no longer needed
<o:p></o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">nvmeStd.c</span></font></span><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:.75in;text-indent:-.25in;mso-list:l0 level1 lfo2">
<![if !supportLists]><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">
</span></font></span></span></font><![endif]><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D">Removed setting the learning mode queue size, we don’t re-allocate now<o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:.75in;text-indent:-.25in;mso-list:l0 level1 lfo2">
<![if !supportLists]><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">
</span></font></span></span></font><![endif]><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D">Changes throughout for
<span class="SpellE">NVMeDriverFatalError</span>() where I removed the last <span class="SpellE">
parm</span> (there were more callers that didn’t need it than did)<o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:.75in;text-indent:-.25in;mso-list:l0 level1 lfo2">
<![if !supportLists]><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">
</span></font></span></span></font><![endif]><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D">In the DPC/ISR, no longer update the QID in the CT, instead only update the vector info in the CT, the QI and the MMT
<span class="SpellE">structs</span><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:.75in;text-indent:-.25in;mso-list:l0 level1 lfo2">
<![if !supportLists]><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">
</span></font></span></span></font><![endif]><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D">In the DPC/ISR,, removed support for sync commands.<span style="mso-spacerun:yes">
</span>Added as part of re-using the delete queues functions but ran into too many complexities making it not worth it.<span style="mso-spacerun:yes">
</span>Instead, the delete queues are part of the <span class="SpellE">init</span> state machine and on shutdown we simply do an EN transition to delete the queues now<o:p></o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">nvmeStat.c</span></font></span><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:.75in;text-indent:-.25in;mso-list:l0 level1 lfo2">
<![if !supportLists]><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">
</span></font></span></span></font><![endif]><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D">Changes for
<span class="SpellE">NVMeDriverFatalError</span>() calls mentioned <span class="SpellE">
earler</span><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:.75in;text-indent:-.25in;mso-list:l0 level1 lfo2">
<![if !supportLists]><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">
</span></font></span></span></font><![endif]><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D">In
<span class="SpellE">NVMeRunningWaitOnReSetupQueues</span>(), we’re now called as part of the
<span class="SpellE">init</span> state machine for each queue so the call to delete
<span class="SpellE">Cpl</span> queues is conditional on the sub queues having been deleted already.<span style="mso-spacerun:yes">
</span>Also no longer clear the <span class="SpellE">allocateQueue</span> flag since we’re no longer freeing/reallocating memory<o:p></o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Nvmeio.c</span></font></span><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:.75in;text-indent:-.25in;mso-list:l0 level1 lfo2">
<![if !supportLists]><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">
</span></font></span></span></font><![endif]><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D">Removed sync support<o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:.75in"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Nvmeinit.c</span></font></span><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D"><span style="mso-tab-count:1">
</span><o:p></o:p></span></font></p>
<p class="MsoNormal" style="text-indent:.5in"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">-In the
<span class="SpellE">nvmeInitcallback</span>() added the state <span class="SpellE">
NVMeWaitOnReSetupQueues</span> to handle completion of the queue deletion phase, once all are delete we change next state to recreate queues<o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">
</span></font></span></span></font><![endif]><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D">Changes to
<span class="SpellE">NVMeDeleteCplQueues</span>() and <span class="SpellE">NVMeDeleteSubQueues</span>() to call for one deletion and count on state machine to delete all queues (was sync on all queues)<o:p></o:p></span></font></p>
<p class="MsoListParagraph"><span class="SpellE"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">NVMeNormalShutdown</span></font></span><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D">()
<span class="SpellE">cals</span> for device reset to delete queues instead of trying to delete the queues individually synchronously or via a state machine that would need to be coded<o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">
</span></font></span></span></font><![endif]><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D">Removed learning support from
<span class="SpellE">NVMeAllocIoQueues</span>(), no longer needed<o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:.75in"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-outline-level:1"><b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";font-weight:bold">From:</span></font></b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">
<a href="mailto:nvmewin-bounces@lists.openfabrics.org">nvmewin-bounces@lists.openfabrics.org</a>
<a href="mailto:[mailto:nvmewin-bounces@lists.openfabrics.org]">[mailto:nvmewin-bounces@lists.openfabrics.org]</a>
<b><span style="font-weight:bold">On Behalf Of </span></b>Luse, Paul E<br>
<b><span style="font-weight:bold">Sent:</span></b> Tuesday, March 27, 2012 9:06 AM<br>
<b><span style="font-weight:bold">To:</span></b> Chang, Alex; <a href="mailto:nvmewin@lists.openfabrics.org">
nvmewin@lists.openfabrics.org</a><br>
<b><span style="font-weight:bold">Subject:</span></b> Re: [<span class="SpellE">nvmewin</span>] Learning Mode Patch for Review - UPDATED<o:p></o:p></span></font></p>
</div>
</div>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:11.0pt"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Agreed and implemented as such
</span></font><font size="2" color="#1f497d" face="Wingdings"><span style="font-size:10.0pt;font-family:Wingdings;mso-ascii-font-family:Calibri;mso-hansi-font-family:Calibri;color:#1F497D;mso-char-type:symbol;mso-symbol-font-family:Wingdings"><span style="mso-char-type:symbol;mso-symbol-font-family:Wingdings">J</span></span></font><font size="2" color="#1f497d"><span style="font-size:10.0pt;color:#1F497D"><span style="mso-spacerun:yes">
</span>I should be able to test first thing Thu morn.<span style="mso-spacerun:yes">
</span>Will provide an update afterwards and the latest patch as well.<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Thx<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Paul<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-outline-level:1"><b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";font-weight:bold">From:</span></font></b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">
Chang, Alex </span></font><a href="mailto:[mailto:Alex.Chang@idt.com]"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">[mailto:Alex.Chang@idt.com]</span></font></a><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">
<br>
<b><span style="font-weight:bold">Sent:</span></b> Tuesday, March 27, 2012 8:42 AM<br>
<b><span style="font-weight:bold">To:</span></b> Luse, Paul E; </span></font><a href="mailto:nvmewin@lists.openfabrics.org"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">nvmewin@lists.openfabrics.org</span></font></a><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman""><br>
<b><span style="font-weight:bold">Subject:</span></b> RE: Learning Mode Patch for Review - UPDATED<o:p></o:p></span></font></p>
</div>
</div>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:11.0pt"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="blue" face="Arial"><span style="font-size:10.0pt;font-family:"Arial","sans-serif";mso-fareast-font-family:"Times New Roman";color:blue">Hi Paul,</span></font><font size="3" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman""><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman""> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="blue" face="Arial"><span style="font-size:10.0pt;font-family:"Arial","sans-serif";mso-fareast-font-family:"Times New Roman";color:blue">Yes, that's what I meant. The sole purpose of learning mode you added is to find
out the mappings between vectors and cores. If we allocate queue memory based on NUMA locality for each core, and then delete/re-create queues after learning is completed, the NUMA locality is still being covered without re-allocating the queue memory.</span></font><font size="3" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman""><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman""> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="blue" face="Arial"><span style="font-size:10.0pt;font-family:"Arial","sans-serif";mso-fareast-font-family:"Times New Roman";color:blue">Thanks,</span></font><font size="3" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman""><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="blue" face="Arial"><span style="font-size:10.0pt;font-family:"Arial","sans-serif";mso-fareast-font-family:"Times New Roman";color:blue">Alex</span></font><font size="3" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman""><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman""><o:p> </o:p></span></font></p>
<div class="MsoNormal" align="center" style="text-align:center"><font size="3" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman"">
<hr size="3" width="100%" align="center">
</span></font></div>
<p class="MsoNormal" style="margin-bottom:12.0pt;mso-outline-level:1"><b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";font-weight:bold">From:</span></font></b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">
Luse, Paul E </span></font><a href="mailto:[mailto:paul.e.luse@intel.com]"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">[mailto:paul.e.luse@intel.com]</span></font></a><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">
<br>
<b><span style="font-weight:bold">Sent:</span></b> Monday, March 26, 2012 7:18 PM<br>
<b><span style="font-weight:bold">To:</span></b> Chang, Alex; </span></font><a href="mailto:nvmewin@lists.openfabrics.org"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">nvmewin@lists.openfabrics.org</span></font></a><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman""><br>
<b><span style="font-weight:bold">Subject:</span></b> RE: Learning Mode Patch for Review - UPDATED</span></font><font size="3" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman""><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">I guess we could do that, keep the queue-core association and change just the IV for the CQ, is that what you’re suggesting?<span style="mso-spacerun:yes">
</span>So, for example, below shows what mapping might look like before, then after learning with how I have it coded
<span class="SpellE">vs</span> what you are saying.<span style="mso-spacerun:yes">
</span>If I understand you correctly I do like that way better, <span class="SpellE">
pls</span> take a look below and confirm for me. <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">BEFORE<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">Core<span style="mso-spacerun:yes">
</span><span style="mso-tab-count:1"> </span><span style="mso-spacerun:yes"> </span>QP<span style="mso-tab-count:1">
</span>MSIX<span style="mso-tab-count:1"> </span>NUMA<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">0<span style="mso-tab-count:1">
</span>1<span style="mso-tab-count:1"> </span>1<span style="mso-tab-count:1">
</span>0<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">1<span style="mso-tab-count:1">
</span>2<span style="mso-tab-count:1"> </span>2<span style="mso-tab-count:1">
</span>1<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">2<span style="mso-tab-count:1">
</span>3<span style="mso-tab-count:1"> </span>3<span style="mso-tab-count:1">
</span>2<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">WHAT I’M DOING<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">Core<span style="mso-spacerun:yes">
</span><span style="mso-tab-count:1"> </span><span style="mso-spacerun:yes"> </span>QP<span style="mso-tab-count:1">
</span>MSIX<span style="mso-tab-count:1"> </span>NUMA<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">0<span style="mso-tab-count:1">
</span>8<span style="mso-tab-count:1"> </span>8<span style="mso-tab-count:1">
</span>0<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">1<span style="mso-tab-count:1">
</span>1<span style="mso-tab-count:1"> </span>1<span style="mso-tab-count:1">
</span>1<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">2<span style="mso-tab-count:1">
</span>2<span style="mso-tab-count:1"> </span>2<span style="mso-tab-count:1">
</span>2<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">WHAT YOU ARE SUGGESTING<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">Core<span style="mso-spacerun:yes">
</span><span style="mso-tab-count:1"> </span><span style="mso-spacerun:yes"> </span>QP<span style="mso-tab-count:1">
</span>MSIX<span style="mso-tab-count:1"> </span>NUMA<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">0<span style="mso-tab-count:1">
</span>1<span style="mso-tab-count:1"> </span>8<span style="mso-tab-count:1">
</span>0<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">1<span style="mso-tab-count:1">
</span>2<span style="mso-tab-count:1"> </span>1<span style="mso-tab-count:1">
</span>1<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt">2<span style="mso-tab-count:1">
</span>3<span style="mso-tab-count:1"> </span>2<span style="mso-tab-count:1">
</span>2<o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"><o:p> </o:p></span></font></p>
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-outline-level:1"><b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";font-weight:bold">From:</span></font></b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">
Chang, Alex </span></font><a href="mailto:[mailto:Alex.Chang@idt.com]"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">[mailto:Alex.Chang@idt.com]</span></font></a><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">
<br>
<b><span style="font-weight:bold">Sent:</span></b> Monday, March 26, 2012 6:41 PM<br>
<b><span style="font-weight:bold">To:</span></b> Luse, Paul E; </span></font><a href="mailto:nvmewin@lists.openfabrics.org"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">nvmewin@lists.openfabrics.org</span></font></a><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman""><br>
<b><span style="font-weight:bold">Subject:</span></b> RE: Learning Mode Patch for Review - UPDATED<o:p></o:p></span></font></p>
</div>
</div>
<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:11.0pt"><o:p> </o:p></span></font></p>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">Hi Paul,
<o:p></o:p></span></font></p>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black"><o:p> </o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">In the changes of
<span class="SpellE">nvmestat.c</span>, you mentioned:<o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><span class="SpellE"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D">NVMeRunningWaitOnReSetupQueues</span></font></span><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D">(). Delete
the queues (frees the <span class="SpellE">mem</span> also), set the flag to indicate that we need to allocate them again and then jump back to the state where we allocate them.</span></font><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black"><o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">I am not sure it's necessary to free the allocated IO queue memory. Isn't
it enough to adjust the mapping between cores and queues after re-creating the queue?<o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black"><o:p> </o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">Thanks,<o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">Alex<o:p></o:p></span></font></p>
</div>
<div>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black"><o:p> </o:p></span></font></p>
</div>
<div>
<div>
<p class="MsoNormal" style="text-indent:-.25in"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black"><o:p> </o:p></span></font></p>
</div>
<div>
<p class="MsoNormal" style="text-indent:-.25in"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black"><o:p> </o:p></span></font></p>
</div>
<div>
<div class="MsoNormal" align="center" style="text-align:center"><font size="3" color="black" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:black">
<hr size="3" width="100%" align="center">
</span></font></div>
<div id="divRpF874637">
<p class="MsoNormal" style="margin-bottom:12.0pt;mso-outline-level:1"><b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black;font-weight:bold">From:</span></font></b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">
</span></font><a href="mailto:nvmewin-bounces@lists.openfabrics.org"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">nvmewin-bounces@lists.openfabrics.org</span></font></a><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black">
[nvmewin-bounces@lists.openfabrics.org] on behalf of Luse, Paul E [paul.e.luse@intel.com]<br>
<b><span style="font-weight:bold">Sent:</span></b> Monday, March 26, 2012 5:52 PM<br>
<b><span style="font-weight:bold">To:</span></b> </span></font><a href="mailto:nvmewin@lists.openfabrics.org"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">nvmewin@lists.openfabrics.org</span></font></a><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";color:black"><br>
<b><span style="font-weight:bold">Subject:</span></b> Re: [<span class="SpellE">nvmewin</span>] Learning Mode Patch for Review - UPDATED</span></font><font size="3" color="black" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:black"><o:p></o:p></span></font></p>
</div>
<div>
<div>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Quick testing update: ran into a QEMU bug that I’ve since fixed and enabled me to continue testing. There will be a few small tweaks to the package
I sent out but nothing major. I have to head to the OFA developer workshop for the next two days so will pick up on this again Thu. I won’t schedule the review until I get everything 100% tested.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D">Thx<br>
Paul</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-outline-level:1"><b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black;font-weight:bold">From:</span></font></b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">
Luse, Paul E <br>
<b><span style="font-weight:bold">Sent:</span></b> Friday, March 23, 2012 4:25 PM<br>
<b><span style="font-weight:bold">To:</span></b> </span></font><a href="mailto:nvmewin@lists.openfabrics.org"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif"">nvmewin@lists.openfabrics.org</span></font></a><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black"><br>
<b><span style="font-weight:bold">Subject:</span></b> Learning Mode Patch for Review - UPDATED</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
</div>
</div>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Here are the changes in addition to the ones below to accommodate the deletion/recreation of queues
as part of learning mode. Looks like a lot only because I touched a lot of functions for cleanup in a few areas
</span></font><font size="3" color="#1f497d" face="Wingdings"><span style="font-size:12.0pt;font-family:Wingdings;color:#1F497D">J</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">I’m not totally done testing, I only have the final queue deletion to step through and am having a
small challenge w/QEMU and my proto HW doesn’t support deleting queues, I’ll get it figured out soon though. I’ll setup a review for late next week, if anyone would like more time let me know – there’s no blazing hurry on this but I do think we want this
to be our first release (so we’re actually NUMA optimized)</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Pw is ofanvme123</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeStd.h</span></font></span><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><span class="SpellE"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Up’d</span></font></span><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">
the poll mode timeout from 1 to 3 and renamed it as <span class="SpellE">its</span> now used for more than just
<span class="SpellE">crashdump</span>, it’s a generic polling retry value</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">New error states and also I repurposed this from an
<span class="SpellE">init</span> state machine specific set of errors to general errors that may throw the driver into a ‘shutdown state’ as well. The shutdown routine was using this
<span class="SpellE">enum</span> and the driver state to indicate shutdown before however the names/usages were all related to the
<span class="SpellE">init</span> state machine which I think could be confusing for new folks reading the code</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Added 2 new
<span class="SpellE">init</span> states, learning and re-setting up the queues</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">A few fairly obvious changes to support logic changes in the C code described later</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Fixed the device state timeout counter, was a UCHAR and not big enough to handle our timeout counter</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:0in"><span class="SpellE"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeInit.h</span></font></span><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> New
<span class="SpellE">parm</span> for the <span class="SpellE">alloc</span> queue function, need to tell it how many entries on the queue as we don’t need large queues for learning and it’s a waste of
<span class="SpellE">alloc</span>/free larger than needed contiguous chunks of memory if we used one size</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeStd.c</span></font></span><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">By default I set the # of queues to create for learning to the previously defined minimum for the driver</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Multiple changes where calling
<span class="SpellE">NVMeAllocQueues</span>() to pass the new <span class="SpellE">
parm</span>, # of queue entries</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Multiple changes where I renamed
<span class="SpellE">StartState</span> to <span class="SpellE">DriveState</span> to reflect how we’re using it anyway</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">In
<span class="SpellE">NVMeInitialize</span>() removed the <span class="SpellE">init</span> of
<span class="SpellE">num</span> Q’s allocated elements in queue info, its overwritten later regardless</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Some refactoring in both the ISR and DPC around the logic where we decide whether we need to call
<span class="SpellE">storportNotification</span> or not, with the new condition added (whether caller wants synchronous IO or not) it was too confusing to try and work it into the previous set of conditions</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">This is where we clear the
<span class="SpellE">srbExt</span> synchronous flag which, for sync IO, tells the caller (more on this later) that the IO has now completed</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Fixed the
<span class="SpellE">passiveInit</span> timeout handler, it was relying on the time drive state machine thread to switch states, if it hung then this would never happen so we’d spin forever. So, made
<span class="SpellE">passiveInit</span> do the time based timing and removed from the
<span class="SpellE">init</span> machine (don’t need to time in both places)</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Removed the
<span class="SpellE">startiolock</span> from the learning mode condition in the ISR/DPC, not needed until we implement
<span class="SpellE">concurrent_channels</span></span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeStat.c</span></font></span><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Multiple changes where I renamed
<span class="SpellE">StartState</span> to <span class="SpellE">DriveState</span> to reflect how we’re using it anyway</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Addition of 2 new states in
<span class="SpellE">NVMeRunning</span>()</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">State handler added:
<span class="SpellE">NVMeRunningWaitOnLearnMapping</span>(). If we’re in sub-optimal conditions to begin with or if we’ve already done this state, we skip the learning and re-setup queues states. Also, if no namespace exists there’s no way to learn so we
just skip <span class="SpellE">init</span> learning and go non-NUMA optimized. The existing learning logic will pick the right queue and adjust the mapping tables (so we still learn as the host sends IOs) we just won’t
<span class="SpellE">resetup</span> the queues. The completion side will go back to this state if we have more IOs to learn. FLUSH is a mandatory command (and doesn’t require data xfer) so it was the logical choice here however I made it a read of 1 block
in case IHV’s have issues with flush (either not supporting or not wanting a flush on every boot for whatever reason).</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">State handler added:
<span class="SpellE">NVMeRunningWaitOnReSetupQueues</span>(). Delete the queues (frees the
<span class="SpellE">mem</span> also), set the flag to indicate that we need to allocate them again and then jump back to the state where we allocate them. Learning won’t happen again because the learning state handler will recognize that
<span class="SpellE">its</span> been completed once</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Moved timeout check to passive
<span class="SpellE">init</span> thread</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeSnti.c</span></font></span><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">renamed
<span class="SpellE">StartState</span> to <span class="SpellE">DriveState</span> to reflect how we’re using it anyway</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeIO.c</span></font></span><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">changes here support being able to do sync IO. The shutdown routines, which I reused for learning mode
<span class="SpellE">resetup</span>, needed sync IO but hadn’t added it so this is technically a change that was needed anyway</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">moved the
<span class="SpellE">NVMeIssueCmd</span>() ret value check to after the call (was in the wrong place)</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">after we issue the command, we handle polling for
<span class="SpellE">crashdump</span> same as before (where there are no INTs)</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">after we handle
<span class="SpellE">crashdump</span>, added we poll here for sync IO to complete but because INTs are enabled, we don’t poll by calling the ISR, we just wait for the ISR to fire and clear the flag in the
<span class="SpellE">srbExt</span> that’s set if the IO is sync</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeInit.h</span></font></span><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">function proto change for new
<span class="SpellE">parm</span></span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeInit.c</span></font></span><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><span class="SpellE"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">NVMeAllocQueues</span></font></span><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">()
changed to accept a <span class="SpellE">parm</span> for # of entries to <span class="SpellE">
alloc</span></span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Multiple changes where I renamed
<span class="SpellE">StartState</span> to <span class="SpellE">DriveState</span> to reflect how we’re using it anyway</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">New callback:
<span class="SpellE">NVMeDeleteQueueCallback</span>. Simply decrements the # of deleted queues if no error</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">In the
<span class="SpellE">init</span> callback for the <span class="SpellE">NVMeWaitOnIoSQ</span> state, we now check how many cores we’ve learned and skip learning if done. Otherwise we move to the learning state</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">In the
<span class="SpellE">init</span> callback, new case for <span class="SpellE">NVMeWaitOnLearnMapping</span>: If we have more cores to learn, stay in this state, otherwise move onto remapping/setting up queues. There’s no new completion state for
<span class="SpellE">resttting</span> up the queues, things will then end in the <span class="SpellE">
NVMeWaitOnIoSQ</span></span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Added missing default handler</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">In both delete queues functions, changes to support making them synchronous</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">In
<span class="SpellE">NVMeAllocIoQueues</span>() we set the number of queues we want based on learning mode or not</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:0in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Sources</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">-</span></font><font size="1" color="#1f497d" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:#1F497D">
</span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Removed QEMU define as we only used it for some asserts that are no longer needed w/learning mode</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:0in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Also throughout you’ll find a new call
<span class="SpellE">NVMeDriverFatalError</span>() that replaces a bunch of places where we were setting and error code, setting a state and logging an error. There’s one
<span class="SpellE">parm</span> in here to indicate whether <span class="SpellE">
its</span> being called from the <span class="SpellE">init</span> state machine or not so it
<span class="SpellE">nows</span> whether to fire off the arbiter. That way this single call can be used anywhere to flag a fatal error, log it in the system log and change the
<span class="SpellE">DriverState</span> to avoid accepting new IO. This is clearly only for cases when we want to stop accepting IO or simply cannot.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<div class="MsoNormal" align="center" style="text-align:center"><font size="3" color="black" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:black">
<hr size="3" width="100%" align="center">
</span></font></div>
<p class="MsoNormal" style="margin-bottom:12.0pt;mso-outline-level:1"><b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black;font-weight:bold">From:</span></font></b><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">
</span></font><a href="mailto:nvmewin-bounces@lists.openfabrics.org" target="_blank"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif"">nvmewin-bounces@lists.openfabrics.org</span></font></a><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">
</span></font><a href="mailto:[mailto:nvmewin-bounces@lists.openfabrics.org]" target="_blank"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif"">[mailto:nvmewin-bounces@lists.openfabrics.org]</span></font></a><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">
<b><span style="font-weight:bold">On Behalf Of </span></b>Luse, Paul E<br>
<b><span style="font-weight:bold">Sent:</span></b> Friday, March 16, 2012 9:42 AM<br>
<b><span style="font-weight:bold">To:</span></b> </span></font><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif"">nvmewin@lists.openfabrics.org</span></font></a><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black"><br>
<b><span style="font-weight:bold">Subject:</span></b> [<span class="SpellE">nvmewin</span>] ***UNCHECKED*** Learning Mode Patch for Review<br>
<b><span style="font-weight:bold">Importance:</span></b> High</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><i><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black;font-style:italic">Note that I merged Alex’s patch in here as it’s still pending with Ray but is only a half-dozen lines of code so rather than have
Ray create to tags and do 2 merges I just rolled them together so Ray can just process one patch. Normally wouldn’t recommend this but the pending patch is so small it just makes sense.</span></font></i><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">Password is ofanvme123. I can schedule a call to walk through any of this if anyone would like.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><b><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black;font-weight:bold">Learning Mode:</span></font></b><font size="2" color="black"><span style="font-size:10.0pt;color:black"> Pretty easy, its only enabled
if we’re in an optimal <span class="SpellE">config</span> <span class="SpellE">wrt</span> cores/queues/vectors. Assume we have N processors, it works like this…. On startup the first N IOs will be sent to queue # 1..N sequentially. Each queue is created
with a matching MSI ID so in this manner we assure to hit every queue and every message ID by incrementing the queue # for the first N IOs regardless of the submitting core. On the completion side we simply look at the core that we completed on and update
the table for the completing core such that the correct queues are used the next time an IO submits to this core.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><b><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black;font-weight:bold">Testing:
</span></font></b><font size="2" color="black"><span style="font-size:10.0pt;color:black">(in all cases I confirmed functionality<b><span style="font-weight:bold">
</span></b>and via <span class="SpellE">iometer</span> and<b><span style="font-weight:bold">
</span></b><span class="SpellE">xperf</span> I confirmed core load balancing)</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Chatham (DT with 8 cores): Win8 Server public beta, 2008R2</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">QEMU (configured for 2 cores): Win7-64</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Tools: The standard stuff: format, stress,
<span class="SpellE">iometer</span>, SCSI compliance, shutdown/restart</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><b><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black;font-weight:bold">Changes:</span></font></b><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">nvmeStd.h</span></font></span><font size="2" color="black"><span style="font-size:10.0pt;color:black">:
</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Comment changes as needed</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Removed the
<span class="SpellE">logcalmode</span> element from the <span class="SpellE">msgTable</span> as
<span class="SpellE">its</span> no longer used</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Removed DBG directive from
<span class="SpellE">procNum</span> element, <span class="SpellE">its</span> now required for learning mode</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Removed proto for
<span class="SpellE">NVMeMsixMapCores</span>, not used anymore (combined MSI and MSIX routines, more later)</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:0in"><span class="SpellE"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">nvmeIo.c</span></font></span><font size="2" color="black"><span style="font-size:10.0pt;color:black">:</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Removed DBG directive from
<span class="SpellE">procNum</span> element, <span class="SpellE">its</span> now required for learning mode</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:0in"><span class="SpellE"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">nvmeInit.c</span></font></span><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Removed
<span class="SpellE">NVMeSwap</span>(),<span class="SpellE">NVMeQuickSort</span>(),<span class="SpellE">NVMeMsixMapCores</span>() as they’re no longer used</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Changes to
<span class="SpellE">NVMeMsiMapCores</span>():</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" color="black" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";color:black">o</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Set the initial core table
<span class="SpellE">MsgID</span> to the <span class="SpellE">CplQ</span> number, this is needed so that we can determine the
<span class="SpellE">CplQ</span> from the <span class="SpellE">MsgId</span> during learning mode while in the DPC/ISR</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" color="black" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";color:black">o</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">The other changes are just code simplifications
</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Changed
<span class="SpellE">NVMeCompleteResMapTbl</span>() to use <span class="SpellE">NVMeMsiMapCores</span>() for either MSI or MSIX</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Changed NVMeMapCore2Queue():</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" color="black" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";color:black">o</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">We now check if we’re in learning mode, if not then we simply look up the queue
<span class="SpellE">num</span> from the core table</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" color="black" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";color:black">o</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">If we’re in learning mode (based on a simple count of how many cores we’ve learned
<span class="SpellE">vs</span> total available cores), then we use the core number that we’re on (that we’re learning) + 1, the +1 is because all queue numbers are core+1 by our convention</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Change in
<span class="SpellE">NVMeAllocIoQueues</span>() to effectively disable learning mode if we only have 1 queue (it makes no sense and actually causes problems for learning mode if we don’t do this). We disable it by pretending that we’ve already learned all
the cores</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><span class="SpellE"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">nvmeStd.c</span></font></span><font size="2" color="black"><span style="font-size:10.0pt;color:black">:</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">In
<span class="SpellE">NVMePassiveInitialize</span>(), disable learning mode is we’re sharing one MSI over multiple queues, same reasons as when we
<span class="SpellE">hae</span> one queue</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">In
<span class="SpellE">NVMeInitialize</span>():</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" color="black" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";color:black">o</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Enable DPC
<span class="SpellE">perf</span> opt per Msft recommendation to steer <span class="SpellE">
stoport</span> completion DPCs back to the submitting core</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">In
<span class="SpellE">NVMeStartIo</span>() – bug fix unrelated to learning mode but I found it while debugging learning mode (via BSOD). You can’t return FALSE from this function per MSDN docs. Always return TRUE</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">In
<span class="SpellE">IoCompletionDpcRoutine</span>() and same changes in the ISR when its enabled for completions instead:</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" color="black" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";color:black">o</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Merged Alex’s
<span class="SpellE">bugfixes</span> in with my changes</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" color="black" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";color:black">o</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Removed the DBG related code for checking core affiliation</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" color="black" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";color:black">o</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Where we decide which queues to check, I set a Boolean to determine if we’re learning or not to FALSE if we’re in shared mode because we disable learning mode during
<span class="SpellE">init</span> if that’s the case</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" color="black" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";color:black">o</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">If we’re not shared, the learning Boolean is set based on how many cores we’ve learned and whether the
<span class="SpellE">MsgId</span> is >0 as <span class="SpellE">MsgId</span> 0 is admin and we exclude that from learning mode</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" color="black" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";color:black">o</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">If we’re not learning then, we only search the queue specified in the MMT</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" color="black" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";color:black">o</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">If we are learning, we know the queue # is the same as the
<span class="SpellE">MsdId</span> because we <span class="SpellE">init’d</span> it that way</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" color="black" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";color:black">o</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">The ‘learning’ happens in a new
<span class="SpellE">coniditonal</span> just after we determine we have an <span class="SpellE">
srbExt</span>. It works as follows:</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.5in;text-indent:-.25in"><font size="2" color="black" face="Wingdings"><span style="font-size:10.0pt;font-family:Wingdings;color:black">§</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Grab the
<span class="SpellE">startIO</span> lock as we’re sharing the core table with <span class="SpellE">
startIO</span> and during learning mode we’re not yet assured that start/complete are on the same core. Note the lock is only taken on IOs during learning mode (the first few IOs)</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.5in;text-indent:-.25in"><font size="2" color="black" face="Wingdings"><span style="font-size:10.0pt;font-family:Wingdings;color:black">§</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Lookup the CT entry for the core that we’re completing on and set its queue numbers to queue number that was associated with the IO that just completed. This assures that
the next lookup in the table for this core # will complete on this, the same, core.</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="margin-left:1.5in;text-indent:-.25in"><font size="2" color="black" face="Wingdings"><span style="font-size:10.0pt;font-family:Wingdings;color:black">§</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Increment our learning counter which will direct the next IO to the next core</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font size="2" color="black"><span style="font-size:10.0pt;color:black">Unrelated changes to
<span class="SpellE">NVMeProcessIoctl</span>(): A few changes were made here as the routine assumed every IOCTL we’d get would be a PT IOCTL making it difficult for venders to add additional private IOCTLs. Just moved things around a bit as we had to add one
to our product specific code. No other changes here other than placing IOCTL specific code in the correct case block</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black">-</span></font><font size="1" color="black" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";color:black">
</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:10.0pt;color:black"> </span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">____________________________________</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";color:black">Paul Luse<br>
Sr. Staff Engineer<br>
PCG Server Software Engineering <br>
Desk: 480.554.3688, Mobile: 480.334.4630</span></font><font color="black"><span style="color:black"><o:p></o:p></span></font></p>
<p class="MsoNormal"><font size="2" color="black" face="Calibri"><span style="font-size:11.0pt;color:black"> <o:p></o:p></span></font></p>
</div>
</div>
</div>
</div>
</div>
</div>
</body>
</html>