<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<!--[if !mso]><style>v\:* {behavior:url(#default#VML);}
o\:* {behavior:url(#default#VML);}
w\:* {behavior:url(#default#VML);}
.shape {behavior:url(#default#VML);}
</style><![endif]--><style><!--
/* Font Definitions */
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Tahoma;
panose-1:2 11 6 4 3 5 4 4 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:purple;
text-decoration:underline;}
p
{mso-style-priority:99;
mso-margin-top-alt:auto;
margin-right:0in;
mso-margin-bottom-alt:auto;
margin-left:0in;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0in;
margin-right:0in;
margin-bottom:0in;
margin-left:.5in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
span.EmailStyle18
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:#002060;
font-weight:normal;
font-style:normal;}
.MsoChpDefault
{mso-style-type:export-only;
font-family:"Calibri",sans-serif;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:774863098;
mso-list-type:hybrid;
mso-list-template-ids:-1660524186 67698689 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l0:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l0:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l0:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l0:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l0:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l0:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l0:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l0:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l0:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l1
{mso-list-id:1055392016;
mso-list-type:hybrid;
mso-list-template-ids:1520450250 67698703 67698713 67698715 67698703 67698713 67698715 67698703 67698713 67698715;}
@list l1:level1
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1:level2
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1:level3
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l1:level4
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1:level5
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1:level6
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l1:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l2
{mso-list-id:1175922401;
mso-list-type:hybrid;
mso-list-template-ids:2131514790 67698689 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l2:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l2:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l2:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l2:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l2:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l2:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l2:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l2:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l2:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l3
{mso-list-id:1953440707;
mso-list-type:hybrid;
mso-list-template-ids:1281929776 67698703 67698713 67698715 67698703 67698713 67698715 67698703 67698713 67698715;}
@list l3:level1
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l3:level2
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l3:level3
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l3:level4
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l3:level5
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l3:level6
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l3:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l3:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l3:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l4
{mso-list-id:2044861544;
mso-list-type:hybrid;
mso-list-template-ids:-39801166 67698689 67698713 67698715 67698703 67698713 67698715 67698703 67698713 67698715;}
@list l4:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l4:level2
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l4:level3
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l4:level4
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l4:level5
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l4:level6
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l4:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l4:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l4:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
ol
{margin-bottom:0in;}
ul
{margin-bottom:0in;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="EN-US" link="blue" vlink="purple">
<div class="WordSection1">
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060;background:yellow;mso-highlight:yellow">All reviewers,</span><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">The concurrent channels patch from Google has been out for review for just over a month. I extended the time to review to provide companies ample time to review/test.
<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Intel has still found the same issues as before with the latest zipped source code sent from Google…<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><o:p> </o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l4 level1 lfo4"><![if !supportLists]><span style="font-size:11.0pt;font-family:Symbol;color:#002060"><span style="mso-list:Ignore">·<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Not on the latest branch (missing namespace management code), and<o:p></o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l4 level1 lfo4"><![if !supportLists]><span style="font-size:11.0pt;font-family:Symbol;color:#002060"><span style="mso-list:Ignore">·<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Missing call to StorPortGetStartIoPerfParams() to get the STARTIO_PERFORMANCE_PARAMETERS in order to identify the current MSI-x vector number (in order
to get the correct SQ on which to submit the command).<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">In lieu of this, I would like to propose two options concerning our mid-year release:<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><o:p> </o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l1 level1 lfo5"><![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><span style="mso-list:Ignore">1.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Move forward with the mid-year release and hold off on resolving the concurrent channels patch from Google until after the release, or<o:p></o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l1 level1 lfo5"><![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><span style="mso-list:Ignore">2.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Resolve the current concurrent channels patch first, and then release (this will delay our targeted end of August release timeframe).<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Please cast your vote for which option you choose. Note that Intel is in favor of option 1.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><o:p> </o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l2 level1 lfo6"><![if !supportLists]><span style="font-size:11.0pt;font-family:Symbol;color:#002060"><span style="mso-list:Ignore">·<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">HGST:
<b>TBD</b><o:p></o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l2 level1 lfo6"><![if !supportLists]><span style="font-size:11.0pt;font-family:Symbol;color:#002060"><span style="mso-list:Ignore">·<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Intel:
<b>Option 1</b><o:p></o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l2 level1 lfo6"><![if !supportLists]><span style="font-size:11.0pt;font-family:Symbol;color:#002060"><span style="mso-list:Ignore">·<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Samsung:
<b>TBD</b><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">If there are other suggestions for the mid-year release, please feel free to suggest.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thank you…<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Ray<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><o:p> </o:p></span></p>
<p class="MsoNormal"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> Iuliu Rus [mailto:iuliur@google.com]
<br>
<b>Sent:</b> Friday, July 08, 2016 12:52 PM<br>
<b>To:</b> Robles, Raymond C <raymond.c.robles@intel.com><br>
<b>Cc:</b> Foster, Carolyn D <carolyn.d.foster@intel.com>; nvmewin <nvmewin@lists.openfabrics.org><br>
<b>Subject:</b> Re: [nvmewin] review comments for patch from Google<o:p></o:p></span></p>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<p class="MsoNormal">Password is nvme<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<p class="MsoNormal">On Fri, Jul 8, 2016 at 12:25 PM, Robles, Raymond C <<a href="mailto:raymond.c.robles@intel.com" target="_blank">raymond.c.robles@intel.com</a>> wrote:<o:p></o:p></p>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-right:0in">
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Hi Iuliu,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">As Carolyn suggested below, you will have to encrypt (password protect) your zip file. Many company
firewalls (i.e. Intel) will block .c/.h files. </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Could you please rezip your source code and add encryption/password protection, and resend? Don’t forget
to send the password as well.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thanks!</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Ray</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> Iuliu Rus [mailto:<a href="mailto:iuliur@google.com" target="_blank">iuliur@google.com</a>]
<br>
<b>Sent:</b> Friday, July 08, 2016 12:23 PM<br>
<b>To:</b> Robles, Raymond C <<a href="mailto:raymond.c.robles@intel.com" target="_blank">raymond.c.robles@intel.com</a>><br>
<b>Cc:</b> Foster, Carolyn D <<a href="mailto:carolyn.d.foster@intel.com" target="_blank">carolyn.d.foster@intel.com</a>>; nvmewin <<a href="mailto:nvmewin@lists.openfabrics.org" target="_blank">nvmewin@lists.openfabrics.org</a>><br>
<b>Subject:</b> Re: [nvmewin] review comments for patch from Google</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">I did, but some of your e-mail servers rejected it, even after changing the extension.<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">On Fri, Jul 8, 2016 at 12:20 PM, Robles, Raymond C <<a href="mailto:raymond.c.robles@intel.com" target="_blank">raymond.c.robles@intel.com</a>> wrote:<o:p></o:p></p>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Hi Iuliu,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Did you ever send out your zipped source code?</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thank you…</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Ray</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> Foster, Carolyn D
<br>
<b>Sent:</b> Wednesday, June 29, 2016 4:45 PM<br>
<b>To:</b> Iuliu Rus <<a href="mailto:iuliur@google.com" target="_blank">iuliur@google.com</a>>; Robles, Raymond C <<a href="mailto:raymond.c.robles@intel.com" target="_blank">raymond.c.robles@intel.com</a>><br>
<b>Cc:</b> nvmewin <<a href="mailto:nvmewin@lists.openfabrics.org" target="_blank">nvmewin@lists.openfabrics.org</a>><br>
<b>Subject:</b> RE: [nvmewin] review comments for patch from Google</span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Hi Iuliu,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Can you please zip your local source files and send it out to the group? You might want to encrypt
it and provide us with the password.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Thanks!</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Carolyn</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><a name="m_5056999964509500280_m_5682067863342812"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></a></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> nvmewin [</span><a href="mailto:nvmewin-bounces@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">mailto:nvmewin-bounces@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">]
<b>On Behalf Of </b>Iuliu Rus<br>
<b>Sent:</b> Wednesday, June 29, 2016 4:20 PM<br>
<b>To:</b> Robles, Raymond C <</span><a href="mailto:raymond.c.robles@intel.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">raymond.c.robles@intel.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Cc:</b> nvmewin <</span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Subject:</b> Re: [nvmewin] review comments for patch from Google</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">I think i don't understand how patching works. Did you sync up your local copy before looking at it? I did not remove those lines, they are still present in my local copy.<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">On Wed, Jun 29, 2016 at 3:44 PM, Robles, Raymond C <<a href="mailto:raymond.c.robles@intel.com" target="_blank">raymond.c.robles@intel.com</a>> wrote:<o:p></o:p></p>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Hi Iuliu,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thanks for the updated patch. After reviewing again, I have some additional questions…</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:Symbol;color:#002060">·</span><span style="font-size:7.0pt;color:#002060">
</span><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">In ProcessIo, the dpc lock is obtained later in the code than originally implemented. Why move the lock acquire down?</span><o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:Symbol;color:#002060">·</span><span style="font-size:7.0pt;color:#002060">
</span><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">In ProcessIo, it appears that you do not call StorPortGetStartIoPerfParams() to obtain the completing MSI-x message number. Getting this MSI-x message number ensures that you
know which interrupt vector will be used to complete this request… and therefore you know on which SQ/CQ pair to post the SQE. Was this left out intentionally? See sample code below:</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.25in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black">if (((DeviceExtension->perfData.Flags & STOR_PERF_INTERRUPT_MESSAGE_RANGES) == STOR_PERF_INTERRUPT_MESSAGE_RANGES) &&</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black">(SrbExtension->pSrb != NULL)) {</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black">Perf.Size = sizeof(STARTIO_PERFORMANCE_PARAMETERS);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black">StorPortGetStartIoPerfParams((PVOID)DeviceExtension,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:2.5in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black">(PSCSI_REQUEST_BLOCK)pSrbExtension->pSrb,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:2.5in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black">&Perf);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black">StorStatus = NVMeMapMsiToQueue(DeviceExtension,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:3.0in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black">Perf.MessageNumber,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:3.0in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black">&SubQueue,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:3.0in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black">&CplQueue);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:Symbol;color:#002060">·</span><span style="font-size:7.0pt;color:#002060">
</span><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">When setting up concurrent channels, I do not see the Storport Perf Opts changes from Suman’s patch (Samsung). The Perf Opts changes replace the old learning mode and drive
the source core interrupt steering in the driver. Without it, the concurrent channels changes will not be optimized correctly. See reference to Perf Opts setup code (directly from OFA source trunk in NVMeInitialize) below:</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Courier New";color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black">if (pAE->ntldrDump == FALSE) {</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> /* Enumerate granted MSI/MSI-X messages if there is any */</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> if (NVMeEnumMsiMessages(pAE) == FALSE) {</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> NVMeFreeBuffers(pAE);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> return (FALSE);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> }</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> perfQueryData.Version = STOR_PERF_VERSION;</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> perfQueryData.Size = sizeof(PERF_CONFIGURATION_DATA);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> Status = StorPortInitializePerfOpts(pAE, TRUE, &perfQueryData);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> ASSERT(STOR_STATUS_SUCCESS == Status);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> if (STOR_STATUS_SUCCESS == Status) {</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> perfData.Version = STOR_PERF_VERSION;</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> perfData.Size = sizeof(PERF_CONFIGURATION_DATA);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> /* Allow optimization of storport DPCs */</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> if (perfQueryData.Flags & STOR_PERF_DPC_REDIRECTION) {</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> perfData.Flags = STOR_PERF_DPC_REDIRECTION;</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> /* Allow optimization of Interrupt Redirection and Group Affinity */</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> if ((perfQueryData.Flags & (STOR_PERF_INTERRUPT_MESSAGE_RANGES | STOR_PERF_ADV_CONFIG_LOCALITY)) ==</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> (STOR_PERF_INTERRUPT_MESSAGE_RANGES | STOR_PERF_ADV_CONFIG_LOCALITY)){</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> perfData.Flags |= STOR_PERF_INTERRUPT_MESSAGE_RANGES;</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> perfData.Flags |= STOR_PERF_ADV_CONFIG_LOCALITY;</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> perfData.FirstRedirectionMessageNumber = 0;</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> perfData.LastRedirectionMessageNumber = (pRMT->NumMsiMsgGranted - 1);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> perfData.MessageTargets = pAE->pArrGrpAff;</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> }</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> }</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> Status = StorPortInitializePerfOpts(pAE, FALSE, &perfData);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> ASSERT(STOR_STATUS_SUCCESS == Status);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> if (STOR_STATUS_SUCCESS == Status){</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> pAE->IsMsiMappingComplete = TRUE;</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> }</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> }</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> /* Call StorPortPassiveInitialization to enable passive init */</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> StorPortEnablePassiveInitialization(pAE, NVMePassiveInitialize);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> return (TRUE);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> } else {</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:1.0in">
<span style="font-size:11.0pt;font-family:"Courier New";color:black"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thank you…</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Ray</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> Iuliu Rus [mailto:</span><a href="mailto:iuliur@google.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">iuliur@google.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">]
<br>
<b>Sent:</b> Wednesday, June 29, 2016 10:56 AM<br>
<b>To:</b> Thomas Freeman <</span><a href="mailto:thomas.freeman@hgst.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">thomas.freeman@hgst.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Cc:</b> Robles, Raymond C <</span><a href="mailto:raymond.c.robles@intel.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">raymond.c.robles@intel.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">>;
nvmewin <</span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Subject:</b> Re: [nvmewin] review comments for patch from Google</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Sorry about Thomas. I resolved that conflict but i think i sent you the patch pre-resolve by mistake. Here is the correct patch. I also triggered the RecoveryDpc from the debugger,
to make sure i have coverage on that. I don't think the normal tests cover it.<o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
</div>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">On Wed, Jun 29, 2016 at 7:25 AM, Thomas Freeman <<a href="mailto:thomas.freeman@hgst.com" target="_blank">thomas.freeman@hgst.com</a>> wrote:<o:p></o:p></p>
</div>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Iuliu,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">It looks like the patch file has an unresolved conflict:</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">+<<<<<<< .mine</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">+ ReleaseDpcSpinLocks(pAE, dpcLocks);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">+ } /* reset the controller */</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">+ if (pAE->NumDpc > MAX_DPC) {</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">+ StorPortFreePool(pHwDeviceExtension, dpcLocks);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">+ }</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">+||||||| .r137</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">+ StorPortReleaseSpinLock(pAE, &startLockhandle);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">+ } /* reset the controller */</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">+=======</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> pAE->DeviceRemovedDuringIO = TRUE;</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">#if (NTDDI_VERSION > NTDDI_WIN7)</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> if (pAE->Timerhandle != NULL) {</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">@@ -2634,6 +2680,7 @@</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> NVMeFreeBuffers(pAE);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> StorPortResume(pAE);</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> } /* reset the controller */</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">+>>>>>>> .r139</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">Tom Freeman</span></b><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">Software Engineer, Device Manager and Driver Development</span></b><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">Western Digital Corporation</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">e.
</span><a href="mailto:Thomas.freeman@hgst.com" target="_blank"><span style="font-size:10.0pt;font-family:"Arial",sans-serif">Thomas.freeman@hgst.com</span></a><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">o.
</span><a href="tel:%2B1-507-322-2311" target="_blank"><span style="font-size:10.0pt;font-family:"Arial",sans-serif">+1-507-322-2311</span></a><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><img border="0" width="150" height="47" style="width:1.5625in;height:.4895in" id="_x0000_i1025" src="cid:image001.jpg@01D1FF8A.272D3960" alt="cid:image001.jpg@01D1D1F7.245A73D0"></span><o:p></o:p></p>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> nvmewin [mailto:</span><a href="mailto:nvmewin-bounces@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin-bounces@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">]
<b>On Behalf Of </b>Iuliu Rus<br>
<b>Sent:</b> Monday, June 27, 2016 11:12 AM<br>
<b>To:</b> Robles, Raymond C <</span><a href="mailto:raymond.c.robles@intel.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">raymond.c.robles@intel.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">></span><o:p></o:p></p>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Cc:</b> nvmewin <<a href="mailto:nvmewin@lists.openfabrics.org" target="_blank">nvmewin@lists.openfabrics.org</a>><br>
<b>Subject:</b> Re: [nvmewin] review comments for patch from Google<o:p></o:p></p>
</div>
</div>
</div>
</div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Done.<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">On Mon, Jun 27, 2016 at 8:52 AM, Robles, Raymond C <<a href="mailto:raymond.c.robles@intel.com" target="_blank">raymond.c.robles@intel.com</a>> wrote:<o:p></o:p></p>
</div>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div style="margin-left:7.5pt;margin-top:7.5pt;margin-right:7.5pt;margin-bottom:7.5pt">
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060;background:yellow">Hi Suman,</span><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thank you for checking on your patches. I was unaware that your latest patch for IOCTL Surprise Removal
was not applied correctly. I’ve just applied that patch again and the trunk should now be at revision 138. Could you please double check to ensure you patch is applied properly and let me know? Thank you!!!</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060;background:yellow">Hi Iuliu,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Sorry about the thrash… but I think we’ve finally got the trunk up to date to the latest (once confirmed
by Suman from Samsung). Once confirmed, please rebase you trunk and add your modifications to that latest revision (138). Thank you!!!</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thanks…<br>
Ray</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> SUMAN PRAKASH B [mailto:</span><a href="mailto:suman.p@samsung.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">suman.p@samsung.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">]
<br>
<b>Sent:</b> Monday, June 27, 2016 1:12 AM<br>
<b>To:</b> Robles, Raymond C <</span><a href="mailto:raymond.c.robles@intel.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">raymond.c.robles@intel.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">>;
Iuliu Rus <</span><a href="mailto:iuliur@google.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">iuliur@google.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Cc:</b> nvmewin <</span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Subject:</b> Re: Re: [nvmewin] review comments for patch from Google</span><o:p></o:p></p>
</div>
</div>
</div>
</div>
<div>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Hi Ray,</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Samsung submitted 3 patches. Revision 134 is first patch, Revision 136 and 137 is second patch.</span><o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">We submitted third patch for Handling device Surprise removal in IOCTL path which was approved by HGST and Intel on 1st June. Please find attached the mail thread for reference.</span><o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">The third patch is not yet checked-in to trunk.</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Thanks,</span><o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Suman</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p>------- <b>Original Message</b> -------<o:p></o:p></p>
<p><b>Sender</b> : Robles, Raymond C<<a href="mailto:raymond.c.robles@intel.com" target="_blank">raymond.c.robles@intel.com</a>><o:p></o:p></p>
<p><b>Date</b> : Jun 27, 2016 12:19 (GMT+05:30)<o:p></o:p></p>
<p><b>Title</b> : Re: [nvmewin] review comments for patch from Google<o:p></o:p></p>
<p> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Hi Iuliu,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">I apologize for the delay in responding. I was performing a manual diff of the last several merges
to complete the investigation. Here is what I found. </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">The latest revision on the OFA source trunk is actually 137 (refer to my screenshot below). I’ve manually
gone back and made sure that all the patches submitted are in this latest revision. I found no errors or issues. The latest trunk is up to date with all of the patches. If you were trying to apply your patch yourself, you cannot. Only myself (OFA co-chair)
and Carolyn Foster (OFA source maintainer) have write permissions to the repo.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Each patch listed below is applied to trunk. The only think you need to insure is that your latest
concurrent channels modifications are based off the 137 revision. If you have any questions, please me know.
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Here is the screenshot of all the patches I’ve checked…</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"><img border="0" width="1082" height="449" style="width:11.2708in;height:4.677in" id="_x0000_i1026" src="cid:image002.png@01D1FF8A.272D3960" alt="cid:image002.png@01D1D1F7.245A73D0"></span><o:p></o:p></p>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thank you…</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Ray</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> nvmewin [mailto:</span><a href="mailto:nvmewin-bounces@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin-bounces@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">]
<b>On Behalf Of </b>Robles, Raymond C<br>
<b>Sent:</b> Friday, June 17, 2016 12:08 PM<br>
<b>To:</b> Iuliu Rus <</span><a href="mailto:iuliur@google.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">iuliur@google.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Cc:</b> nvmewin <</span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Subject:</b> Re: [nvmewin] review comments for patch from Google</span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thanks Iuliu. I’m currently investigating the issue. Will let everyone know when the trunk is updated.
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thank you…</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Ray</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> Iuliu Rus [</span><a href="mailto:iuliur@google.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">mailto:iuliur@google.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">]
<br>
<b>Sent:</b> Friday, June 17, 2016 10:29 AM<br>
<b>To:</b> Robles, Raymond C <</span><a href="mailto:raymond.c.robles@intel.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">raymond.c.robles@intel.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Cc:</b> Thomas Freeman <</span><a href="mailto:thomas.freeman@hgst.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">thomas.freeman@hgst.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">>; nvmewin
<</span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Subject:</b> Re: [nvmewin] review comments for patch from Google</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">I don't see any changes when i try to update. My svn says : completed at revision 137.<o:p></o:p></p>
</div>
</div>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">On Thu, Jun 16, 2016 at 2:42 PM, Robles, Raymond C <<a href="mailto:raymond.c.robles@intel.com" target="_blank">raymond.c.robles@intel.com</a>> wrote:<o:p></o:p></p>
</div>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Hi Iuliu- Please rebase your patch with the latest OFA trunk source. If you do not see any updates
during your rebase, please let me know and I’ll investigate. Thanks!</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> Iuliu Rus [mailto:</span><a href="mailto:iuliur@google.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">iuliur@google.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">]
<br>
<b>Sent:</b> Thursday, June 16, 2016 2:15 PM<br>
<b>To:</b> Robles, Raymond C <</span><a href="mailto:raymond.c.robles@intel.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">raymond.c.robles@intel.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Cc:</b> Thomas Freeman <</span><a href="mailto:thomas.freeman@hgst.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">thomas.freeman@hgst.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">>; nvmewin
<</span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">></span><o:p></o:p></p>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><br>
<b>Subject:</b> Re: [nvmewin] review comments for patch from Google<o:p></o:p></p>
</div>
</div>
</div>
</div>
<div>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Let me know if i have to sync to the latest and re-do the patch.<o:p></o:p></p>
</div>
</div>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">On Thu, Jun 16, 2016 at 2:10 PM, Iuliu Rus <<a href="mailto:iuliur@google.com" target="_blank">iuliur@google.com</a>> wrote:<o:p></o:p></p>
</div>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">We already have a pool of locks - the dpc locks, which is what i'm using in my latest patch. You cannot create spinlocks directly in a storport driver, the only option is to use
either the StartIoLock, the dpc lock, the interrupt lock or one of the MSI locks.<o:p></o:p></p>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">On Thu, Jun 16, 2016 at 2:04 PM, Robles, Raymond C <<a href="mailto:raymond.c.robles@intel.com" target="_blank">raymond.c.robles@intel.com</a>> wrote:<o:p></o:p></p>
</div>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Hello,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Here are my thoughts…
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">In short, I agree with Thomas’s assessment.
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Concurrent channels allows Storport to call StartIo concurrently… and with no locks. Therefore the
locking of critical sections is now the responsibility of the miniport.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Any solution proposed should protect critical resources (cmd id pool, cmd entries, doorbells) with
a lock that is granular enough for our driver, but not at the visibility of the OS or Storport. Acquiring the StartIo lock to protect resources in our driver only is too broad of a lock. My suggestion would be to create a pool of locks (one for every SQ/CQ
pair) that is maintained within our driver. Again, crashdump mode is a non-issue because it is single threaded.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thank you…</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Ray</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> Thomas Freeman [mailto:</span><a href="mailto:thomas.freeman@hgst.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">thomas.freeman@hgst.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">]
<br>
<b>Sent:</b> Thursday, June 16, 2016 1:51 PM<br>
<b>To:</b> Iuliu Rus <</span><a href="mailto:iuliur@google.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">iuliur@google.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Cc:</b> Robles, Raymond C <</span><a href="mailto:raymond.c.robles@intel.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">raymond.c.robles@intel.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">>;
nvmewin <</span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Subject:</b> RE: [nvmewin] review comments for patch from Google</span><o:p></o:p></p>
</div>
</div>
</div>
</div>
<div>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Hi Iuliu,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Here my view, I hope someone else can chime in to verify or correct what I'm saying</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">The purpose of the concurrent channels change is to prevent Storport from acquiring the StartIo lock
before calling Startio. This allows multiple simultaneous calls to StartIo. If you require ProcessIo get the StartIo lock, you are replicating what Storport used to - it defeats the point of concurrent channels.
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Here is my suggestion for what needs to be changed</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">1. There is no need for StartIo Locks - remove the requests to acquire them</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">2. Acquire the Dpc locks in the following situations</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> -ProcessIo, before calling GetCmdEntry - if there is queue sharing</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> -ProcessIo, before calling IssueCmd - always</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> -IoCompletionRoutine, before calling NVMeCompleteCmd - if there is queue sharing</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> -recoveryDpcRoutine - get all of the Dpc locks</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Here is my reasoning</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">-GetCmdEntry and NVMeCompleteCmd both access the FreeQList. With queue sharing, these could run simultaneously
and have a collision when they update the list. </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">-IssueCmd. With queue sharing, different cpu's could attempt to put entries on the queue simultaneously.
But, to make RecoveryDpcRoutine work, this must be protected by the dpcLock whether or not there is queue sharing.
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">I think this will address the deadlock problem I reported also.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Regards,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">Tom Freeman</span></b><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">Software Engineer, Device Manager and Driver Development</span></b><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">Western Digital Corporation</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">e.
</span><a href="mailto:Thomas.freeman@hgst.com" target="_blank"><span style="font-size:10.0pt;font-family:"Arial",sans-serif">Thomas.freeman@hgst.com</span></a><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">o.
</span><a href="tel:%2B1-507-322-2311" target="_blank"><span style="font-size:10.0pt;font-family:"Arial",sans-serif">+1-507-322-2311</span></a><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><img border="0" width="150" height="47" style="width:1.5625in;height:.4895in" id="_x0000_i1027" src="cid:image001.jpg@01D1FF8A.272D3960" alt="cid:image001.jpg@01D1D1F7.245A73D0"></span><o:p></o:p></p>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> Iuliu Rus [</span><a href="mailto:iuliur@google.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">mailto:iuliur@google.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">]
<br>
<b>Sent:</b> Thursday, June 16, 2016 2:42 PM<br>
<b>To:</b> Thomas Freeman <</span><a href="mailto:thomas.freeman@hgst.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">thomas.freeman@hgst.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Cc:</b> Robles, Raymond C <</span><a href="mailto:raymond.c.robles@intel.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">raymond.c.robles@intel.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">>;
nvmewin <</span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Subject:</b> Re: [nvmewin] review comments for patch from Google</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Here's my revised patch for concurrent channels. The synchronization rules are now as follows:<o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">1. Init is single threaded so it doesn't matter if acquirelock is true or false<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">2. Anything called from the DPC has to call ProcessIo with false, since the dpc spinlock is already acquired.<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">3. Anything called from StartIo has to call ProcessIo with true, since now the startio is not automatically acquired.<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Thomas, i think the bug that you found is because the ProcessIo was incorrectly called with TRUE instead of FALSE.<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Let me know what are your thoughts on this.<o:p></o:p></p>
</div>
</div>
</div>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">On Thu, Jun 16, 2016 at 10:41 AM, Iuliu Rus <<a href="mailto:iuliur@google.com" target="_blank">iuliur@google.com</a>> wrote:<o:p></o:p></p>
</div>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Guys, am i OK continuing with the patch on the current code base or do you want me to wait until you pull in some more changes?<o:p></o:p></p>
</div>
</div>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<div>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">On Wed, Jun 15, 2016 at 8:37 AM, Thomas Freeman <<a href="mailto:thomas.freeman@hgst.com" target="_blank">thomas.freeman@hgst.com</a>> wrote:<o:p></o:p></p>
</div>
</div>
</div>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div>
<div>
<div>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Ray,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">It appears that the latest patch file from Iuliu is applied to revision 137. I don’t see Suman’s changes
in that revision.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">As an example, the use of StorPortRequestTimer is not in nvmeStat.c.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">I thought there were some newer revisions than 137, but 137 is the latest I’m able to download.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">137 is also the revision that appears when I go to
</span><a href="https://svn.openfabrics.org/svnrepo/nvmewin/" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">https://svn.openfabrics.org/svnrepo/nvmewin/</span></a><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">SVN is giving me some troubles, so I can’t tell for sure if this is my environment or a server issue.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">Tom Freeman</span></b><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">Software Engineer, Device Manager and Driver Development</span></b><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">Western Digital Corporation</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">e.
</span><a href="mailto:Thomas.freeman@hgst.com" target="_blank"><span style="font-size:10.0pt;font-family:"Arial",sans-serif">Thomas.freeman@hgst.com</span></a><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#1F497D">o.
</span><a href="tel:%2B1-507-322-2311" target="_blank"><span style="font-size:10.0pt;font-family:"Arial",sans-serif">+1-507-322-2311</span></a><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><img border="0" width="150" height="47" style="width:1.5625in;height:.4895in" id="_x0000_i1028" src="cid:image003.jpg@01D1FF8A.272D3960" alt="cid:image003.jpg@01D1D1F7.245A73D0"></span><o:p></o:p></p>
</div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> nvmewin [mailto:</span><a href="mailto:nvmewin-bounces@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin-bounces@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">]
<b>On Behalf Of </b>Robles, Raymond C<br>
<b>Sent:</b> Friday, June 10, 2016 12:23 PM<br>
<b>To:</b> Iuliu Rus <</span><a href="mailto:iuliur@google.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">iuliur@google.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">>;
</span><a href="mailto:suman.p@samsung.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">suman.p@samsung.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"><br>
<b>Cc:</b> nvmewin <</span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">></span><o:p></o:p></p>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><br>
<b>Subject:</b> Re: [nvmewin] review comments for patch from Google<o:p></o:p></p>
</div>
</div>
</div>
</div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Hi Iuliu,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">If you send out the updated patch with the protection in place for both list entries and doorbells,
I can run it on an Intel NVMe SSD (P3700) and see if there is any performance impact (up or down).</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thank you…</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Ray</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> Iuliu Rus [</span><a href="mailto:iuliur@google.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">mailto:iuliur@google.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">]
<br>
<b>Sent:</b> Friday, June 10, 2016 10:17 AM<br>
<b>To:</b> </span><a href="mailto:suman.p@samsung.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">suman.p@samsung.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"><br>
<b>Cc:</b> Robles, Raymond C <</span><a href="mailto:raymond.c.robles@intel.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">raymond.c.robles@intel.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">>;
Judy Brock <</span><a href="mailto:judy.brock@samsung.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">judy.brock@samsung.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">>; nvmewin <</span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">><br>
<b>Subject:</b> Re: RE: review comments for patch from Google</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">we are running this in a virtual machine, and we do see a perf difference with concurrent channels. We did not observe any performance drop with synchronized access to the list
entry.<o:p></o:p></p>
</div>
</div>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">On Fri, Jun 10, 2016 at 7:15 AM, SUMAN PRAKASH B <<a href="mailto:suman.p@samsung.com" target="_blank">suman.p@samsung.com</a>> wrote:<o:p></o:p></p>
</div>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div>
<div>
<div>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Hi Iuliu,</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">As you have the implementation in place, can you let us know if there is any performance improvement when STOR_PERF_CONCURRENT_CHANNELS is set in case of #core and #queues not matching.</span><o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Also, if there is any drop in performance if the list entry and doorbell is protected after setting STOR_PERF_CONCURRENT_CHANNELS.</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Perhaps a bit off topic:</span><o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Even in case when #cores and #queues are same and when STOR_PERF_CONCURRENT_CHANNELS is set, we did not observe any performance gain. Maybe we are hitting the max performance from device even
without setting STOR_PERF_CONCURRENT_CHANNELS. Can you please let us know your observation?</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Thanks,</span><o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Suman</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p>------- <b>Original Message</b> -------<o:p></o:p></p>
<p><b>Sender</b> : Robles, Raymond C<<a href="mailto:raymond.c.robles@intel.com" target="_blank">raymond.c.robles@intel.com</a>><o:p></o:p></p>
<p><b>Date</b> : Jun 10, 2016 08:30 (GMT+05:30)<o:p></o:p></p>
<p><b>Title</b> : RE: review comments for patch from Google<o:p></o:p></p>
</div>
</div>
<div>
<div>
<p> <o:p></o:p></p>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Hi Judy,
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thanks for the additional info. I agree with the assessment. My statement was directed at the general
direction of the solution… and the fact that I think option 1 is not as robust as option 2.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thank you…</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Ray</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> Judy Brock [mailto:</span><a href="mailto:judy.brock@samsung.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">judy.brock@samsung.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">]
<br>
<b>Sent:</b> Thursday, June 09, 2016 5:14 PM<br>
<b>To:</b> Robles, Raymond C <</span><a href="mailto:raymond.c.robles@intel.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">raymond.c.robles@intel.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">>;
Iuliu Rus <</span><a href="mailto:iuliur@google.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">iuliur@google.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">>;
</span><a href="mailto:suman.p@samsung.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">suman.p@samsung.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"><br>
<b>Cc:</b> </span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"><br>
<b>Subject:</b> RE: review comments for patch from Google</span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Hi Ray et al,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Calibri",sans-serif">[Suman wrote] >> Protect the listentry and doorbell before access/update</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> >>Do not set STOR_PERF_CONCURRENT_CHANNELS when number of cores and queues are not same.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;text-indent:.5in">
<span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Ray wrote] >>In the case Suman points out, concurrent channels should still be enabled even if the number of cores and queues do not match. This is especially true for large core
count systems (>128 logical cpu cores) but devices may only support 32 queues or less</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">I think Suman was saying, we have two options:
</span><o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">1.</span><span style="font-size:7.0pt;color:#1F497D">
</span><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">either don’t enable it at all if # cores != #queues or
</span><o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">2.</span><span style="font-size:7.0pt;color:#1F497D">
</span><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">protect both the list entries and doorbell writes before any accesses/updates if #cores != #queues.
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">So we will need to check to see if we need to use locks or not in the performance path based on whether
#cores and # queues DO match or not if we want to enable concurrent channels no matter what. It’s probably better than option 1) above if this is a configuration that can come up frequently.</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Thanks,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Judy</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"> </span><o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:10.0pt;font-family:"Tahoma",sans-serif">From:</span></b><span style="font-size:10.0pt;font-family:"Tahoma",sans-serif"> nvmewin [</span><a href="mailto:nvmewin-bounces@lists.openfabrics.org" target="_blank"><span style="font-size:10.0pt;font-family:"Tahoma",sans-serif">mailto:nvmewin-bounces@lists.openfabrics.org</span></a><span style="font-size:10.0pt;font-family:"Tahoma",sans-serif">]
<b>On Behalf Of </b>Robles, Raymond C<br>
<b>Sent:</b> Thursday, June 09, 2016 2:32 PM<br>
<b>To:</b> Iuliu Rus; </span><a href="mailto:suman.p@samsung.com" target="_blank"><span style="font-size:10.0pt;font-family:"Tahoma",sans-serif">suman.p@samsung.com</span></a><span style="font-size:10.0pt;font-family:"Tahoma",sans-serif"><br>
<b>Cc:</b> </span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:10.0pt;font-family:"Tahoma",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:10.0pt;font-family:"Tahoma",sans-serif"><br>
<b>Subject:</b> Re: [nvmewin] review comments for patch from Google</span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Hi Iuliu/Suman,</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">I’m of the opinion that any solution provided should be as robust as possible. In the case Suman points
out, concurrent channels should still be enabled even if the number of cores and queues do not match. This is especially true for large core count systems (>128 logical cpu cores) but devices may only support 32 queues or less.
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thank you…</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Ray</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">From:</span></b><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"> Iuliu Rus
</span><a href="mailto:[mailto:iuliur@google.com]" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">[mailto:iuliur@google.com]</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">
<br>
<b>Sent:</b> Thursday, June 09, 2016 2:09 PM<br>
<b>To:</b> </span><a href="mailto:suman.p@samsung.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">suman.p@samsung.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"><br>
<b>Cc:</b> Robles, Raymond C <</span><a href="mailto:raymond.c.robles@intel.com" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">raymond.c.robles@intel.com</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">>;
</span><a href="mailto:nvmewin@lists.openfabrics.org" target="_blank"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">nvmewin@lists.openfabrics.org</span></a><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"><br>
<b>Subject:</b> Re: review comments for patch from Google</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Which one do you prefer? Our google driver has the synchronization in place already for a different reason. Or, I can follow your second suggestion.<o:p></o:p></p>
</div>
</div>
</div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<div>
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">On Thu, Jun 9, 2016 at 7:39 AM, SUMAN PRAKASH B <<a href="mailto:suman.p@samsung.com" target="_blank">suman.p@samsung.com</a>> wrote:<o:p></o:p></p>
</div>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt">
<div>
<div>
<div>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Hi Iuliu,</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Please find my review comments below:</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">1. If the number of Cores and Queues supported by device are not same, then setting STOR_PERF_CONCURRENT_CHANNELS will corrupt the submission queue. For example, if device has 4 queues and connected
to target PC which has 8 logical processors, and if STOR_PERF_CONCURRENT_CHANNELS is set, all the 8 logical processors will get startIo calls concurrently. As per OFA driver NUMA design, the IO Queue 1 will be shared by core 0 and core 4. There is possibility
that core 0 and core 4 will get startio() calls concurrently and will access/update the listenty and queue 1 doorbell register at same time, which will lead to listentry and doorbell register corruption.
<br>
This problem can be address by following ways:<br>
a. Protect the listentry and doorbell before access/update.<br>
b. Do not set STOR_PERF_CONCURRENT_CHANNELS when number of cores and queues are not same.</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">2. We can use pRMT->NumActiveCores which is updated in NVMeEnumNumaCores() instead of using KeQueryActiveProcessorCountEx/KeQueryActiveProcessorCount.</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Please let us know your opinion.</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Thanks,<br>
Suman</span><o:p></o:p></p>
<p> <o:p></o:p></p>
<p>------- <b>Original Message</b> -------<o:p></o:p></p>
<p><b>Sender</b> : Robles, Raymond C<<a href="mailto:raymond.c.robles@intel.com" target="_blank">raymond.c.robles@intel.com</a>><o:p></o:p></p>
<p><b>Date</b> : Jun 09, 2016 03:56 (GMT+05:30)<o:p></o:p></p>
<p><b>Title</b> : Re: [nvmewin] Happy New Year... and status update<o:p></o:p></p>
<p> <o:p></o:p></p>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">Thank you Iuliu!</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060;background:yellow">Reviewing companies, please try to target the review completion for 2 weeks from
today (6/22/16).</span><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060">
</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#002060"> </span><o:p></o:p></p>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</blockquote>
</div>
</div>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</div>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</div>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">...<br>
<br>
[Message clipped] <o:p></o:p></p>
</blockquote>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
</div>
</div>
</div>
</blockquote>
</div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
</div>
</div>
</div>
</blockquote>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
</div>
</body>
</html>