
<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">

<head>

<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">

<meta name="ProgId" content="Word.Document">

<meta name="Generator" content="Microsoft Word 14">

<meta name="Originator" content="Microsoft Word 14">

<link rel="File-List" href="cid:filelist.xml@01CD0B79.477E72B0"><link rel="Edit-Time-Data" href="cid:editdata.mso"><!--[if !mso]><style>v\:* {behavior:url(#default#VML);}

o\:* {behavior:url(#default#VML);}

w\:* {behavior:url(#default#VML);}

.shape {behavior:url(#default#VML);}

</style><![endif]--><!--[if gte mso 9]><xml>

<o:OfficeDocumentSettings>

<o:AllowPNG/>

<o:DoNotRelyOnCSS/>

</o:OfficeDocumentSettings>

</xml><![endif]--><!--[if gte mso 9]><xml>

<w:WordDocument>

<w:Zoom>120</w:Zoom>

<w:SpellingState>Clean</w:SpellingState>

<w:TrackMoves/>

<w:TrackFormatting/>

<w:EnvelopeVis/>

<w:PunctuationKerning/>

<w:ValidateAgainstSchemas/>

<w:SaveIfXMLInvalid>false</w:SaveIfXMLInvalid>

<w:IgnoreMixedContent>false</w:IgnoreMixedContent>

<w:AlwaysShowPlaceholderText>false</w:AlwaysShowPlaceholderText>

<w:DoNotPromoteQF/>

<w:LidThemeOther>EN-US</w:LidThemeOther>

<w:LidThemeAsian>X-NONE</w:LidThemeAsian>

<w:LidThemeComplexScript>X-NONE</w:LidThemeComplexScript>

<w:Compatibility>

<w:DoNotExpandShiftReturn/>

<w:BreakWrappedTables/>

<w:SnapToGridInCell/>

<w:WrapTextWithPunct/>

<w:UseAsianBreakRules/>

<w:DontGrowAutofit/>

<w:SplitPgBreakAndParaMark/>

<w:EnableOpenTypeKerning/>

<w:DontFlipMirrorIndents/>

<w:OverrideTableStyleHps/>

</w:Compatibility>

<w:BrowserLevel>MicrosoftInternetExplorer4</w:BrowserLevel>

<m:mathPr>

<m:mathFont m:val="Cambria Math"/>

<m:brkBin m:val="before"/>

<m:brkBinSub m:val="--"/>

<m:smallFrac m:val="off"/>

<m:dispDef/>

<m:lMargin m:val="0"/>

<m:rMargin m:val="0"/>

<m:defJc m:val="centerGroup"/>

<m:wrapIndent m:val="1440"/>

<m:intLim m:val="subSup"/>

<m:naryLim m:val="undOvr"/>

</m:mathPr></w:WordDocument>

</xml><![endif]--><!--[if gte mso 9]><xml>

<w:LatentStyles DefLockedState="false" DefUnhideWhenUsed="true" DefSemiHidden="true" DefQFormat="false" DefPriority="99" LatentStyleCount="267">

<w:LsdException Locked="false" Priority="0" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Normal"/>

<w:LsdException Locked="false" Priority="9" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="heading 1"/>

<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 2"/>

<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 3"/>

<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 4"/>

<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 5"/>

<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 6"/>

<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 7"/>

<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 8"/>

<w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 9"/>

<w:LsdException Locked="false" Priority="39" Name="toc 1"/>

<w:LsdException Locked="false" Priority="39" Name="toc 2"/>

<w:LsdException Locked="false" Priority="39" Name="toc 3"/>

<w:LsdException Locked="false" Priority="39" Name="toc 4"/>

<w:LsdException Locked="false" Priority="39" Name="toc 5"/>

<w:LsdException Locked="false" Priority="39" Name="toc 6"/>

<w:LsdException Locked="false" Priority="39" Name="toc 7"/>

<w:LsdException Locked="false" Priority="39" Name="toc 8"/>

<w:LsdException Locked="false" Priority="39" Name="toc 9"/>

<w:LsdException Locked="false" Priority="35" QFormat="true" Name="caption"/>

<w:LsdException Locked="false" Priority="10" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Title"/>

<w:LsdException Locked="false" Priority="1" Name="Default Paragraph Font"/>

<w:LsdException Locked="false" Priority="11" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Subtitle"/>

<w:LsdException Locked="false" Priority="22" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Strong"/>

<w:LsdException Locked="false" Priority="20" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Emphasis"/>

<w:LsdException Locked="false" Priority="59" SemiHidden="false" UnhideWhenUsed="false" Name="Table Grid"/>

<w:LsdException Locked="false" UnhideWhenUsed="false" Name="Placeholder Text"/>

<w:LsdException Locked="false" Priority="1" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="No Spacing"/>

<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading"/>

<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List"/>

<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid"/>

<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1"/>

<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2"/>

<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1"/>

<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2"/>

<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1"/>

<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2"/>

<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3"/>

<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List"/>

<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading"/>

<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List"/>

<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid"/>

<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 1"/>

<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 1"/>

<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 1"/>

<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 1"/>

<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 1"/>

<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 1"/>

<w:LsdException Locked="false" UnhideWhenUsed="false" Name="Revision"/>

<w:LsdException Locked="false" Priority="34" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="List Paragraph"/>

<w:LsdException Locked="false" Priority="29" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Quote"/>

<w:LsdException Locked="false" Priority="30" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Intense Quote"/>

<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 1"/>

<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 1"/>

<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 1"/>

<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 1"/>

<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 1"/>

<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 1"/>

<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 1"/>

<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 1"/>

<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 2"/>

<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 2"/>

<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 2"/>

<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 2"/>

<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 2"/>

<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 2"/>

<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 2"/>

<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 2"/>

<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 2"/>

<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 2"/>

<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 2"/>

<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 2"/>

<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 2"/>

<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 2"/>

<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 3"/>

<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 3"/>

<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 3"/>

<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 3"/>

<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 3"/>

<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 3"/>

<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 3"/>

<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 3"/>

<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 3"/>

<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 3"/>

<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 3"/>

<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 3"/>

<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 3"/>

<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 3"/>

<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 4"/>

<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 4"/>

<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 4"/>

<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 4"/>

<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 4"/>

<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 4"/>

<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 4"/>

<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 4"/>

<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 4"/>

<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 4"/>

<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 4"/>

<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 4"/>

<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 4"/>

<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 4"/>

<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 5"/>

<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 5"/>

<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 5"/>

<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 5"/>

<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 5"/>

<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 5"/>

<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 5"/>

<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 5"/>

<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 5"/>

<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 5"/>

<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 5"/>

<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 5"/>

<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 5"/>

<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 5"/>

<w:LsdException Locked="false" Priority="60" SemiHidden="false" UnhideWhenUsed="false" Name="Light Shading Accent 6"/>

<w:LsdException Locked="false" Priority="61" SemiHidden="false" UnhideWhenUsed="false" Name="Light List Accent 6"/>

<w:LsdException Locked="false" Priority="62" SemiHidden="false" UnhideWhenUsed="false" Name="Light Grid Accent 6"/>

<w:LsdException Locked="false" Priority="63" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 1 Accent 6"/>

<w:LsdException Locked="false" Priority="64" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Shading 2 Accent 6"/>

<w:LsdException Locked="false" Priority="65" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 1 Accent 6"/>

<w:LsdException Locked="false" Priority="66" SemiHidden="false" UnhideWhenUsed="false" Name="Medium List 2 Accent 6"/>

<w:LsdException Locked="false" Priority="67" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 1 Accent 6"/>

<w:LsdException Locked="false" Priority="68" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 2 Accent 6"/>

<w:LsdException Locked="false" Priority="69" SemiHidden="false" UnhideWhenUsed="false" Name="Medium Grid 3 Accent 6"/>

<w:LsdException Locked="false" Priority="70" SemiHidden="false" UnhideWhenUsed="false" Name="Dark List Accent 6"/>

<w:LsdException Locked="false" Priority="71" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Shading Accent 6"/>

<w:LsdException Locked="false" Priority="72" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful List Accent 6"/>

<w:LsdException Locked="false" Priority="73" SemiHidden="false" UnhideWhenUsed="false" Name="Colorful Grid Accent 6"/>

<w:LsdException Locked="false" Priority="19" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Subtle Emphasis"/>

<w:LsdException Locked="false" Priority="21" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Intense Emphasis"/>

<w:LsdException Locked="false" Priority="31" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Subtle Reference"/>

<w:LsdException Locked="false" Priority="32" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Intense Reference"/>

<w:LsdException Locked="false" Priority="33" SemiHidden="false" UnhideWhenUsed="false" QFormat="true" Name="Book Title"/>

<w:LsdException Locked="false" Priority="37" Name="Bibliography"/>

<w:LsdException Locked="false" Priority="39" QFormat="true" Name="TOC Heading"/>

</w:LatentStyles>

</xml><![endif]--><style><!--

/* Font Definitions */

@font-face

        {font-family:Wingdings;

        panose-1:5 0 0 0 0 0 0 0 0 0;

        mso-font-alt:"SymbolProp BT";

        mso-font-charset:2;

        mso-generic-font-family:auto;

        mso-font-pitch:variable;

        mso-font-signature:0 268435456 0 0 -2147483648 0;}

@font-face

        {font-family:Wingdings;

        panose-1:5 0 0 0 0 0 0 0 0 0;

        mso-font-alt:"SymbolProp BT";

        mso-font-charset:2;

        mso-generic-font-family:auto;

        mso-font-pitch:variable;

        mso-font-signature:0 268435456 0 0 -2147483648 0;}

@font-face

        {font-family:Calibri;

        panose-1:2 15 5 2 2 2 4 3 2 4;

        mso-font-alt:"Times New Roman";

        mso-font-charset:0;

        mso-generic-font-family:swiss;

        mso-font-pitch:variable;

        mso-font-signature:-520092929 1073786111 9 0 415 0;}

@font-face

        {font-family:Tahoma;

        panose-1:2 11 6 4 3 5 4 4 2 4;

        mso-font-alt:Tahoma;

        mso-font-charset:0;

        mso-generic-font-family:swiss;

        mso-font-pitch:variable;

        mso-font-signature:-520081665 -1073717157 41 0 66047 0;}

/* Style Definitions */

p.MsoNormal, li.MsoNormal, div.MsoNormal

        {mso-style-unhide:no;

        mso-style-qformat:yes;

        mso-style-parent:"";

        margin:0in;

        margin-bottom:.0001pt;

        mso-pagination:widow-orphan;

        font-size:11.0pt;

        font-family:"Calibri","sans-serif";

        mso-fareast-font-family:Calibri;

        mso-bidi-font-family:"Times New Roman";}

a:link, span.MsoHyperlink

        {mso-style-noshow:yes;

        mso-style-priority:99;

        color:blue;

        text-decoration:underline;

        text-underline:single;}

a:visited, span.MsoHyperlinkFollowed

        {mso-style-noshow:yes;

        mso-style-priority:99;

        color:purple;

        text-decoration:underline;

        text-underline:single;}

p.MsoAcetate, li.MsoAcetate, div.MsoAcetate

        {mso-style-noshow:yes;

        mso-style-priority:99;

        mso-style-link:"Balloon Text Char";

        margin:0in;

        margin-bottom:.0001pt;

        mso-pagination:widow-orphan;

        font-size:8.0pt;

        font-family:"Tahoma","sans-serif";

        mso-fareast-font-family:Calibri;}

p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph

        {mso-style-priority:34;

        mso-style-unhide:no;

        mso-style-qformat:yes;

        margin-top:0in;

        margin-right:0in;

        margin-bottom:0in;

        margin-left:.5in;

        margin-bottom:.0001pt;

        mso-pagination:widow-orphan;

        font-size:11.0pt;

        font-family:"Calibri","sans-serif";

        mso-fareast-font-family:Calibri;

        mso-bidi-font-family:"Times New Roman";}

span.BalloonTextChar

        {mso-style-name:"Balloon Text Char";

        mso-style-noshow:yes;

        mso-style-priority:99;

        mso-style-unhide:no;

        mso-style-locked:yes;

        mso-style-link:"Balloon Text";

        mso-ansi-font-size:8.0pt;

        mso-bidi-font-size:8.0pt;

        font-family:"Tahoma","sans-serif";

        mso-ascii-font-family:Tahoma;

        mso-hansi-font-family:Tahoma;

        mso-bidi-font-family:Tahoma;}

span.EmailStyle20

        {mso-style-type:personal;

        mso-style-noshow:yes;

        mso-style-unhide:no;

        mso-ansi-font-size:10.0pt;

        mso-bidi-font-size:10.0pt;

        font-family:"Calibri","sans-serif";

        mso-ascii-font-family:Calibri;

        mso-hansi-font-family:Calibri;

        mso-bidi-font-family:Calibri;

        color:windowtext;

        mso-text-animation:none;

        font-weight:normal;

        font-style:normal;

        text-decoration:none;

        text-underline:none;

        text-decoration:none;

        text-line-through:none;}

span.EmailStyle21

        {mso-style-type:personal;

        mso-style-noshow:yes;

        mso-style-unhide:no;

        mso-ansi-font-size:10.0pt;

        mso-bidi-font-size:10.0pt;

        font-family:"Calibri","sans-serif";

        mso-ascii-font-family:Calibri;

        mso-hansi-font-family:Calibri;

        mso-bidi-font-family:Calibri;

        color:#1F497D;

        mso-text-animation:none;

        font-weight:normal;

        font-style:normal;

        text-decoration:none;

        text-underline:none;

        text-decoration:none;

        text-line-through:none;}

span.EmailStyle22

        {mso-style-type:personal;

        mso-style-noshow:yes;

        mso-style-unhide:no;

        mso-ansi-font-size:10.0pt;

        mso-bidi-font-size:10.0pt;

        font-family:"Calibri","sans-serif";

        mso-ascii-font-family:Calibri;

        mso-hansi-font-family:Calibri;

        mso-bidi-font-family:Calibri;

        color:#1F497D;

        mso-text-animation:none;

        font-weight:normal;

        font-style:normal;

        text-decoration:none;

        text-underline:none;

        text-decoration:none;

        text-line-through:none;}

span.EmailStyle23

        {mso-style-type:personal-reply;

        mso-style-noshow:yes;

        mso-style-unhide:no;

        mso-ansi-font-size:10.0pt;

        mso-bidi-font-size:10.0pt;

        font-family:"Calibri","sans-serif";

        mso-ascii-font-family:Calibri;

        mso-hansi-font-family:Calibri;

        mso-bidi-font-family:Calibri;

        color:#1F497D;

        font-weight:normal;

        font-style:normal;

        text-decoration:none;

        text-underline:none;

        text-decoration:none;

        text-line-through:none;}

.MsoChpDefault

        {mso-style-type:export-only;

        mso-default-props:yes;

        font-size:10.0pt;

        mso-ansi-font-size:10.0pt;

        mso-bidi-font-size:10.0pt;

        font-family:"Calibri","sans-serif";

        mso-ascii-font-family:Calibri;

        mso-fareast-font-family:Calibri;

        mso-hansi-font-family:Calibri;

        mso-bidi-font-family:Calibri;}

@page WordSection1

        {size:8.5in 11.0in;

        margin:1.0in 1.0in 1.0in 1.0in;

        mso-header-margin:.5in;

        mso-footer-margin:.5in;

        mso-paper-source:0;}

div.WordSection1

        {page:WordSection1;}

/* List Definitions */

@list l0

        {mso-list-id:1191450303;

        mso-list-type:hybrid;

        mso-list-template-ids:-887951328 1490686764 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}

@list l0:level1

        {mso-level-start-at:0;

        mso-level-number-format:bullet;

        mso-level-text:-;

        mso-level-tab-stop:none;

        mso-level-number-position:left;

        text-indent:-.25in;

        font-family:"Times New Roman","serif";

        mso-fareast-font-family:Calibri;}

@list l0:level2

        {mso-level-number-format:bullet;

        mso-level-text:o;

        mso-level-tab-stop:none;

        mso-level-number-position:left;

        text-indent:-.25in;

        font-family:"Courier New";}

@list l0:level3

        {mso-level-number-format:bullet;

        mso-level-text:\F0A7;

        mso-level-tab-stop:none;

        mso-level-number-position:left;

        text-indent:-.25in;

        font-family:Wingdings;}

@list l0:level4

        {mso-level-number-format:bullet;

        mso-level-text:\F0B7;

        mso-level-tab-stop:none;

        mso-level-number-position:left;

        text-indent:-.25in;

        font-family:Symbol;}

@list l0:level5

        {mso-level-number-format:bullet;

        mso-level-text:o;

        mso-level-tab-stop:none;

        mso-level-number-position:left;

        text-indent:-.25in;

        font-family:"Courier New";}

@list l0:level6

        {mso-level-number-format:bullet;

        mso-level-text:\F0A7;

        mso-level-tab-stop:none;

        mso-level-number-position:left;

        text-indent:-.25in;

        font-family:Wingdings;}

@list l0:level7

        {mso-level-number-format:bullet;

        mso-level-text:\F0B7;

        mso-level-tab-stop:none;

        mso-level-number-position:left;

        text-indent:-.25in;

        font-family:Symbol;}

@list l0:level8

        {mso-level-number-format:bullet;

        mso-level-text:o;

        mso-level-tab-stop:none;

        mso-level-number-position:left;

        text-indent:-.25in;

        font-family:"Courier New";}

@list l0:level9

        {mso-level-number-format:bullet;

        mso-level-text:\F0A7;

        mso-level-tab-stop:none;

        mso-level-number-position:left;

        text-indent:-.25in;

        font-family:Wingdings;}

ol

        {margin-bottom:0in;}

ul

        {margin-bottom:0in;}

--></style><!--[if gte mso 10]><style>/* Style Definitions */

table.MsoNormalTable

        {mso-style-name:"Table Normal";

        mso-tstyle-rowband-size:0;

        mso-tstyle-colband-size:0;

        mso-style-noshow:yes;

        mso-style-priority:99;

        mso-style-parent:"";

        mso-padding-alt:0in 5.4pt 0in 5.4pt;

        mso-para-margin:0in;

        mso-para-margin-bottom:.0001pt;

        mso-pagination:widow-orphan;

        font-size:10.0pt;

        font-family:"Calibri","sans-serif";}

</style><![endif]--><!--[if gte mso 9]><xml>

<o:shapedefaults v:ext="edit" spidmax="1026" />

</xml><![endif]--><!--[if gte mso 9]><xml>

<o:shapelayout v:ext="edit">

<o:idmap v:ext="edit" data="1" />

</o:shapelayout></xml><![endif]-->

</head>

<body lang="EN-US" link="blue" vlink="purple" style="tab-interval:.5in">

<div class="WordSection1">

<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri;color:#1F497D">Quick testing update:<span style="mso-spacerun:yes"> 

</span>ran into a QEMU bug that I’ve since fixed and enabled me to continue testing.<span style="mso-spacerun:yes"> 

</span>There will be a few small tweaks to the package I sent out but nothing major.<span style="mso-spacerun:yes"> 

</span>I have to head to the OFA developer workshop for the next two days so will pick up on this again Thu.<span style="mso-spacerun:yes"> 

</span>I won’t schedule the review until I get everything 100% tested.<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri;color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri;color:#1F497D">Thx<br>

Paul<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="2" color="#1f497d" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri;color:#1F497D"><o:p> </o:p></span></font></p>

<div>

<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">

<p class="MsoNormal"><b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";font-weight:bold">From:</span></font></b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman"">

 Luse, Paul E <br>

<b><span style="font-weight:bold">Sent:</span></b> Friday, March 23, 2012 4:25 PM<br>

<b><span style="font-weight:bold">To:</span></b> nvmewin@lists.openfabrics.org<br>

<b><span style="font-weight:bold">Subject:</span></b> Learning Mode Patch for Review - UPDATED<o:p></o:p></span></font></p>

</div>

</div>

<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:11.0pt"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Here are the changes in addition to the ones below to accommodate the deletion/recreation of queues

 as part of learning mode.<span style="mso-spacerun:yes">  </span>Looks like a lot only because I touched a lot of functions for cleanup in a few areas

</span></font><font size="3" color="#1f497d" face="Wingdings"><span style="font-size:12.0pt;font-family:Wingdings;mso-ascii-font-family:"Times New Roman";mso-hansi-font-family:"Times New Roman";color:#1F497D;mso-char-type:symbol;mso-symbol-font-family:Wingdings"><span style="mso-char-type:symbol;mso-symbol-font-family:Wingdings">J</span></span></font><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">I’m not totally done testing, I only have the final queue deletion to step through and am having a

 small challenge w/QEMU and my proto HW doesn’t support deleting queues, I’ll get it figured out soon though.<span style="mso-spacerun:yes"> 

</span>I’ll setup a review for late next week, if anyone would like more time let me know – there’s no blazing hurry on this but I do think we want this to be our first release (so we’re actually NUMA optimized)<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Pw is ofanvme123<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeStd.h<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Up’d the poll mode timeout from 1 to 3 and renamed it as its now used for more

 than just crashdump, it’s a generic polling retry value<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">New error states and also I repurposed this from an init state machine specific

 set of errors to general errors that may throw the driver into a ‘shutdown state’ as well.<span style="mso-spacerun:yes"> 

</span>The shutdown routine was using this enum and the driver state to indicate shutdown before however the names/usages were all related to the init state machine which I think could be confusing for new folks reading the code<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Added 2 new init states, learning and re-setting up the queues<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">A few fairly obvious changes to support logic changes in the C code described

 later<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Fixed the device state timeout counter, was a UCHAR and not big enough to handle

 our timeout counter<o:p></o:p></span></font></p>

<p class="MsoListParagraph"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:0in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeInit.h<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><span style="mso-spacerun:yes"> </span>New parm for the alloc queue function,

 need to tell it how many entries on the queue as we don’t need large queues for learning and it’s a waste of alloc/free larger than needed contiguous chunks of memory if we used one size<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeStd.c<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">By default I set the # of queues to create for learning to the previously defined

 minimum for the driver<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Multiple changes where calling NVMeAllocQueues() to pass the new parm, # of

 queue entries<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Multiple changes where I renamed StartState to DriveState to reflect how we’re

 using it anyway<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">In NVMeInitialize() removed the init of num Q’s allocated elements in queue

 info, its overwritten later regardless<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Some refactoring in both the ISR and DPC around the logic where we decide whether

 we need to call storportNotification or not, with the new condition added (whether caller wants synchronous IO or not) it was too confusing to try and work it into the previous set of conditions<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">This is where we clear the srbExt synchronous flag which, for sync IO, tells

 the caller (more on this later) that the IO has now completed<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Fixed the passiveInit timeout handler, it was relying on the time drive state

 machine thread to switch states, if it hung then this would never happen so we’d spin forever.<span style="mso-spacerun:yes"> 

</span>So, made passiveInit do the time based timing and removed from the init machine (don’t need to time in both places)<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Removed the startiolock from the learning mode condition in the ISR/DPC, not

 needed until we implement concurrent_channels<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeStat.c<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Multiple changes where I renamed StartState to DriveState to reflect how we’re

 using it anyway<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Addition of 2 new states in NVMeRunning()<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">State handler added: NVMeRunningWaitOnLearnMapping().<span style="mso-spacerun:yes"> 

</span>If we’re in sub-optimal conditions to begin with or if we’ve already done this state, we skip the learning and re-setup queues states.<span style="mso-spacerun:yes"> 

</span>Also, if no namespace exists there’s no way to learn so we just skip init learning and go non-NUMA optimized.<span style="mso-spacerun:yes"> 

</span>The existing learning logic will pick the right queue and adjust the mapping tables (so we still learn as the host sends IOs) we just won’t resetup the queues.<span style="mso-spacerun:yes"> 

</span>The completion side will go back to this state if we have more IOs to learn.<span style="mso-spacerun:yes"> 

</span>FLUSH is a mandatory command (and doesn’t require data xfer) so it was the logical choice here however I made it a read of 1 block in case IHV’s have issues with flush (either not supporting or not wanting a flush on every boot for whatever reason).<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">State handler added: NVMeRunningWaitOnReSetupQueues().<span style="mso-spacerun:yes"> 

</span>Delete the queues (frees the mem also), set the flag to indicate that we need to allocate them again and then jump back to the state where we allocate them.<span style="mso-spacerun:yes"> 

</span>Learning won’t happen again because the learning state handler will recognize that its been completed once<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Moved timeout check to passive init thread<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeSnti.c<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">renamed StartState to DriveState to reflect how we’re using it anyway<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeIO.c<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">changes here support being able to do sync IO.<span style="mso-spacerun:yes"> 

</span>The shutdown routines, which I reused for learning mode resetup, needed sync IO but hadn’t added it so this is technically a change that was needed anyway<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">moved the NVMeIssueCmd() ret value check to after the call (was in the wrong

 place)<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">after we issue the command, we handle polling for crashdump same as before

 (where there are no INTs)<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">after we handle crashdump, added we poll here for sync IO to complete but because

 INTs are enabled, we don’t poll by calling the ISR, we just wait for the ISR to fire and clear the flag in the srbExt that’s set if the IO is sync<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeInit.h<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">function proto change for new parm<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">nvmeInit.c<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">NVMeAllocQueues() changed to accept a parm for # of entries to alloc<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Multiple changes where I renamed StartState to DriveState to reflect how we’re

 using it anyway<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">New callback: NVMeDeleteQueueCallback.<span style="mso-spacerun:yes"> 

</span>Simply decrements the # of deleted queues if no error<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">In the init callback for the NVMeWaitOnIoSQ state, we now check how many cores

 we’ve learned and skip learning<span style="mso-spacerun:yes">  </span>if done.<span style="mso-spacerun:yes"> 

</span>Otherwise we move to the learning state<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">In the init callback, new case for NVMeWaitOnLearnMapping:<span style="mso-spacerun:yes"> 

</span>If we have more cores to learn, stay in this state, otherwise move onto remapping/setting up queues.<span style="mso-spacerun:yes"> 

</span>There’s no new completion state for resttting up the queues, things will then end in the NVMeWaitOnIoSQ<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Added missing default handler<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">In both delete queues functions, changes to support making them synchronous<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">In NVMeAllocIoQueues() we set the number of queues we want based on learning

 mode or not<o:p></o:p></span></font></p>

<p class="MsoListParagraph"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:0in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Sources<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l0 level1 lfo2"><![if !supportLists]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Times New Roman";color:#1F497D"><span style="mso-list:Ignore">-<font size="1" face="Times New Roman"><span style="font:7.0pt "Times New Roman"">       

</span></font></span></span></font><![endif]><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Removed QEMU define as we only used it for some asserts that are no longer

 needed w/learning mode<o:p></o:p></span></font></p>

<p class="MsoListParagraph"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:0in"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D">Also throughout you’ll find a new call NVMeDriverFatalError() that replaces

 a bunch of places where we were setting and error code, setting a state and logging an error.<span style="mso-spacerun:yes"> 

</span>There’s one parm in here to indicate whether its being called from the init state machine or not so it nows whether to fire off the arbiter.<span style="mso-spacerun:yes"> 

</span>That way this single call can be used anywhere to flag a fatal error, log it in the system log and change the DriverState to avoid accepting new IO.<span style="mso-spacerun:yes"> 

</span>This is clearly only for cases when we want to stop accepting IO or simply cannot.<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="3" color="#1f497d" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif";color:#1F497D"><o:p> </o:p></span></font></p>

<div class="MsoNormal" align="center" style="text-align:center"><font size="3" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif"">

<hr size="3" width="100%" align="center">

</span></font></div>

<p class="MsoNormal" style="margin-bottom:12.0pt;mso-outline-level:1"><b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";font-weight:bold">From:</span></font></b><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif"">

</span></font><a href="mailto:nvmewin-bounces@lists.openfabrics.org"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif"">nvmewin-bounces@lists.openfabrics.org</span></font></a><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif"">

</span></font><a href="mailto:[mailto:nvmewin-bounces@lists.openfabrics.org]"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif"">[mailto:nvmewin-bounces@lists.openfabrics.org]</span></font></a><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif"">

<b><span style="font-weight:bold">On Behalf Of </span></b>Luse, Paul E<br>

<b><span style="font-weight:bold">Sent:</span></b> Friday, March 16, 2012 9:42 AM<br>

<b><span style="font-weight:bold">To:</span></b> </span></font><a href="mailto:nvmewin@lists.openfabrics.org"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif"">nvmewin@lists.openfabrics.org</span></font></a><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif""><br>

<b><span style="font-weight:bold">Subject:</span></b> [nvmewin] ***UNCHECKED*** Learning Mode Patch for Review<br>

<b><span style="font-weight:bold">Importance:</span></b> High</span></font><font size="3" face="Times New Roman"><span style="font-size:12.0pt;font-family:"Times New Roman","serif""><o:p></o:p></span></font></p>

<p class="MsoNormal"><i style="mso-bidi-font-style:normal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri;font-style:italic;mso-bidi-font-style:normal">Note that I merged Alex’s patch in here as it’s still pending with

 Ray but is only a half-dozen lines of code so rather than have Ray create to tags and do 2 merges I just rolled them together so Ray can just process one patch.<span style="mso-spacerun:yes"> 

</span>Normally wouldn’t recommend this but the pending patch is so small it just makes sense.<o:p></o:p></span></font></i></p>

<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Password is ofanvme123.<span style="mso-spacerun:yes"> 

</span>I can schedule a call to walk through any of this if anyone would like.<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><b style="mso-bidi-font-weight:normal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri;font-weight:bold;mso-bidi-font-weight:normal">Learning Mode:</span></font></b><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri"><span style="mso-spacerun:yes"> 

</span>Pretty easy, its only enabled if we’re in an optimal config wrt cores/queues/vectors.<span style="mso-spacerun:yes"> 

</span>Assume we have N processors, it works like this…. On startup the first N IOs will be sent to queue # 1..N sequentially.<span style="mso-spacerun:yes"> 

</span>Each queue is created with a matching MSI ID so in this manner we assure to hit every queue and every message ID by incrementing the queue # for the first N IOs regardless of the submitting core.<span style="mso-spacerun:yes"> 

</span>On the completion side we simply look at the core that we completed on and update the table for the completing core such that the correct queues are used the next time an IO submits to this core.<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><b style="mso-bidi-font-weight:normal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri;font-weight:bold;mso-bidi-font-weight:normal">Testing:

</span></font></b><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">(in all cases I confirmed functionality<b style="mso-bidi-font-weight:normal"><span style="font-weight:bold;mso-bidi-font-weight:normal">

</span></b>and via iometer and<b style="mso-bidi-font-weight:normal"><span style="font-weight:bold;mso-bidi-font-weight:normal">

</span></b>xperf I confirmed core load balancing)<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Chatham (DT with 8 cores):<span style="mso-spacerun:yes"> 

</span>Win8 Server public beta, 2008R2<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">QEMU (configured for 2 cores): Win7-64<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Tools:<span style="mso-spacerun:yes"> 

</span>The standard stuff:<span style="mso-spacerun:yes">  </span>format, stress, iometer, SCSI compliance, shutdown/restart<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><b style="mso-bidi-font-weight:normal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri;font-weight:bold;mso-bidi-font-weight:normal">Changes:<o:p></o:p></span></font></b></p>

<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">nvmeStd.h:

<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Comment changes as needed<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Removed the logcalmode element from the msgTable as its no longer used<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Removed DBG directive from procNum element, its now required for learning mode<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Removed proto for NVMeMsixMapCores, not used anymore (combined MSI and MSIX routines, more later)<o:p></o:p></span></font></p>

<p class="MsoListParagraph"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri"><o:p> </o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:0in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">nvmeIo.c:<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Removed DBG directive from procNum element, its now required for learning mode<o:p></o:p></span></font></p>

<p class="MsoListParagraph"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri"><o:p> </o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:0in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">nvmeInit.c<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Removed NVMeSwap(),NVMeQuickSort(),NVMeMsixMapCores() as they’re no longer used<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Changes to NVMeMsiMapCores():<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";mso-fareast-font-family:"Courier New"">o</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Courier New"">  

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Set the initial core table MsgID to the CplQ number, this is needed so that we can determine the CplQ from the MsgId during learning mode while in the DPC/ISR<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";mso-fareast-font-family:"Courier New"">o</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Courier New"">  

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">The other changes are just code simplifications

<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Changed NVMeCompleteResMapTbl() to use NVMeMsiMapCores() for either MSI or MSIX<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Changed NVMeMapCore2Queue():<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";mso-fareast-font-family:"Courier New"">o</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Courier New"">  

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">We now check if we’re in learning mode, if not then we simply look up the queue num from the core table<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";mso-fareast-font-family:"Courier New"">o</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Courier New"">  

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">If we’re in learning mode (based on a simple count of how many cores we’ve learned vs total available cores), then we use the core number that we’re on (that we’re learning)

 + 1, the +1 is because all queue numbers are core+1 by our convention<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Change in NVMeAllocIoQueues() to effectively disable learning mode if we only have 1 queue (it makes no sense and actually causes problems for learning mode if we don’t

 do this).<span style="mso-spacerun:yes">  </span>We disable it by pretending that we’ve already learned all the cores<o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">nvmeStd.c:<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">In NVMePassiveInitialize(), disable learning mode is we’re sharing one MSI over multiple queues, same reasons as when we hae one queue<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">In NVMeInitialize():<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";mso-fareast-font-family:"Courier New"">o</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Courier New"">  

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Enable DPC perf opt per Msft recommendation to steer stoport completion DPCs back to the submitting core<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">In NVMeStartIo() – bug fix unrelated to learning mode but I found it while debugging learning mode (via BSOD).<span style="mso-spacerun:yes"> 

</span>You can’t return FALSE from this function per MSDN docs.<span style="mso-spacerun:yes"> 

</span>Always return TRUE<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">In IoCompletionDpcRoutine() and same changes in the ISR when its enabled for completions instead:<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";mso-fareast-font-family:"Courier New"">o</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Courier New"">  

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Merged Alex’s bugfixes in with my changes<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";mso-fareast-font-family:"Courier New"">o</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Courier New"">  

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Removed the DBG related code for checking core affiliation<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";mso-fareast-font-family:"Courier New"">o</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Courier New"">  

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Where we decide which queues to check, I set a Boolean to determine if we’re learning or not to FALSE if we’re in shared mode because we disable learning mode during init

 if that’s the case<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";mso-fareast-font-family:"Courier New"">o</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Courier New"">  

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">If we’re not shared, the learning Boolean is set based on how many cores we’ve learned and whether the MsgId is >0 as MsgId 0 is admin and we exclude that from learning

 mode<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";mso-fareast-font-family:"Courier New"">o</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Courier New"">  

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">If we’re not learning then, we only search the queue specified in the MMT<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";mso-fareast-font-family:"Courier New"">o</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Courier New"">  

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">If we are learning, we know the queue # is the same as the MsdId because we init’d it that way<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in"><font size="2" face="Courier New"><span style="font-size:10.0pt;font-family:"Courier New";mso-fareast-font-family:"Courier New"">o</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:"Courier New"">  

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">The ‘learning’ happens in a new coniditonal just after we determine we have an srbExt.<span style="mso-spacerun:yes"> 

</span>It works as follows:<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.5in;text-indent:-.25in"><font size="2" face="Wingdings"><span style="font-size:10.0pt;font-family:Wingdings;mso-fareast-font-family:Wingdings;mso-bidi-font-family:Wingdings">§</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:Wingdings"> 

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Grab the startIO lock as we’re sharing the core table with startIO and during learning mode we’re not yet assured that start/complete are on the same core.<span style="mso-spacerun:yes"> 

</span>Note the lock is only taken on IOs during learning mode (the first few IOs)<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.5in;text-indent:-.25in"><font size="2" face="Wingdings"><span style="font-size:10.0pt;font-family:Wingdings;mso-fareast-font-family:Wingdings;mso-bidi-font-family:Wingdings">§</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:Wingdings"> 

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Lookup the CT entry for the core that we’re completing on and set its queue numbers to queue number that was associated with the IO that just completed.<span style="mso-spacerun:yes"> 

</span>This assures that the next lookup in the table for this core # will complete on this, the same, core.<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="margin-left:1.5in;text-indent:-.25in"><font size="2" face="Wingdings"><span style="font-size:10.0pt;font-family:Wingdings;mso-fareast-font-family:Wingdings;mso-bidi-font-family:Wingdings">§</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif";mso-fareast-font-family:Wingdings"> 

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Increment our learning counter which will direct the next IO to the next core<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">Unrelated changes to NVMeProcessIoctl(): A few changes were made here as the routine assumed every IOCTL we’d get would be a PT IOCTL making it difficult for venders to

 add additional private IOCTLs.<span style="mso-spacerun:yes">  </span>Just moved things around a bit as we had to add one to our product specific code.<span style="mso-spacerun:yes"> 

</span>No other changes here other than placing IOCTL specific code in the correct case block<o:p></o:p></span></font></p>

<p class="MsoListParagraph" style="text-indent:-.25in"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri">-</span></font><font size="1" face="Times New Roman"><span style="font-size:7.0pt;font-family:"Times New Roman","serif"">        

</span></font><font size="2"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri"><o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:10.0pt;mso-bidi-font-family:Calibri"><o:p> </o:p></span></font></p>

<p class="MsoNormal"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";mso-no-proof:yes">____________________________________</span></font><font size="3" face="Arial"><span style="font-size:12.0pt;font-family:"Arial","sans-serif";mso-fareast-font-family:"Times New Roman";mso-no-proof:yes"><o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="2" face="Tahoma"><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif";mso-fareast-font-family:"Times New Roman";mso-no-proof:yes">Paul Luse<br>

Sr. Staff Engineer<br>

PCG Server Software Engineering <br>

Desk: 480.554.3688, Mobile: 480.334.4630</span></font><font size="3" face="Arial"><span style="font-size:12.0pt;font-family:"Arial","sans-serif";mso-fareast-font-family:"Times New Roman";mso-no-proof:yes"><o:p></o:p></span></font></p>

<p class="MsoNormal"><font size="2" face="Calibri"><span style="font-size:11.0pt;mso-bidi-font-family:Calibri"><o:p> </o:p></span></font></p>

</div>

</body>

</html>