
#include <Winsock2.h>
#include <stdio.h>
#include <stdlib.h>

// Functions defenitions to avoid the dependency on mswsock.h (allow building using the ddk only)
typedef VOID
(PASCAL FAR
GetAcceptExSockaddrsType) (
    IN PVOID lpOutputBuffer,
    IN DWORD dwReceiveDataLength,
    IN DWORD dwLocalAddressLength,
    IN DWORD dwRemoteAddressLength,
    OUT struct sockaddr **LocalSockaddr,
    OUT LPINT LocalSockaddrLength,
    OUT struct sockaddr **RemoteSockaddr,
    OUT LPINT RemoteSockaddrLength
    );

typedef BOOL
(WINAPI FAR AcceptExType) (
    IN SOCKET sListenSocket,
    IN SOCKET sAcceptSocket,
    IN PVOID lpOutputBuffer,
    IN DWORD dwReceiveDataLength,
    IN DWORD dwLocalAddressLength,
    IN DWORD dwRemoteAddressLength,
    OUT LPDWORD lpdwBytesReceived,
    IN LPOVERLAPPED lpOverlapped
    );

#define SO_CONNECT_TIME             0x700C
GetAcceptExSockaddrsType *pGetAcceptExSockaddrs = NULL;
AcceptExType *pAcceptEx = NULL;


HANDLE g_hStdout;
CONSOLE_SCREEN_BUFFER_INFO g_ConsoleScreenBufferInfo;

#define ColorPrintf(Color, format) \
{ \
    SetConsoleTextAttribute(g_hStdout, Color); \
    printf format; \
    SetConsoleTextAttribute(g_hStdout, g_ConsoleScreenBufferInfo.wAttributes);\
}\


int AssertBreak() {
/*
    DebugBreak();
    return 1;
*/

    ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Assert has happened, program exiting") );
    
    exit(3);

}


__int64
GetTimeStamp( void )
{
	LARGE_INTEGER	tick_count, frequency;

	if( !QueryPerformanceFrequency( &frequency ) )
		return( 0 );

	if( !QueryPerformanceCounter( &tick_count ) )
		return( 0 );

	return( 1000000 * tick_count.QuadPart / frequency.QuadPart );
}


#define ASSERT(expr) (void) ((expr) ||  AssertBreak())


#define AF_INET_FAMILY PF_INET

#define sleep(x) Sleep(1000 * (x));
#define CloseSocket closesocket

#define GetTimeMs() (double)GetTimeStamp() / 1000


const int g_NumberOfThreads = 2;

HANDLE g_ComplitionPort = NULL;
HANDLE g_pThreads[g_NumberOfThreads];

int ServerHandleNewAccept(SOCKET newfd, void *FirstBuffer, int BufferLen);

struct sockaddr addressFromString(char *address, int port)
{
    unsigned int b1,b2,b3,b4;

    struct sockaddr_in socket_address;
    struct sockaddr *socket_address_ptr = (struct sockaddr*)&socket_address;
    
    memset(&socket_address, 0, sizeof(socket_address));
    socket_address.sin_family = AF_INET;
    socket_address.sin_port = htons((u_short)port);


    sscanf(address, "%d.%d.%d.%d",&b1, &b2, &b3, &b4);
    socket_address.sin_addr.s_addr = b4 * 256 * 256 *256 +
                                     b3 * 256 * 256 + 
                                     b2 * 256 +
                                     b1;
                                     
    return *socket_address_ptr;
}

enum AvailableOperations {
    SEND,
    RECV,
    ACCEPTEX
};

const int LOCALADDRESSLENGTH = 26; // 26 is the minimum
const int DWREMOTEADDRESSLENGTH = 22; //(0 => 10114) 16=>ERROR_INSUFFICIENT_BUFFER, 22 is the minimum

struct OverlappedSend {

// General Data
    OVERLAPPED Overlapped;
    AvailableOperations Operation;
    SOCKET Socket;
    void *Buffer; // The data to send
    int  BufferSize;    

// Used for send and receive    
    DWORD NumberOfBytesOut; // The number of bytes sent / received
    int PostExactRecvPackats; // Used only on recv to tell the number of packets
    
// used for AcceptEx
    SOCKET sAcceptSocket;


// Used for general bookkiping about what is going on.

    static __int64  s_RemainingData; // The total amount of data that wasn't compleated yet
    static __int64  s_SendsInAir; // The number of bytes data that was sent, but not yet acknowledged as success/failure
    static CRITICAL_SECTION s_Lock;
    static int s_NumberOfOverlappedAccepts;
    static HANDLE s_OperationsDoneEvent;
    static int s_NumberOfOperatopnsInAir;  // Number of overlapped operations in air
    
};

__int64  OverlappedSend::s_RemainingData;
__int64  OverlappedSend::s_SendsInAir;
CRITICAL_SECTION OverlappedSend::s_Lock;
HANDLE OverlappedSend::s_OperationsDoneEvent;
int OverlappedSend::s_NumberOfOverlappedAccepts;
int OverlappedSend::s_NumberOfOperatopnsInAir;


VOID IncreaseRecvSocketSize(SOCKET s, int RecvBufferSize)
{
    printf("Increasing socket size to %d\n", RecvBufferSize);
    int RealRecvBufferSize;
    int err = setsockopt(s, SOL_SOCKET, SO_RCVBUF,  (char *)&RecvBufferSize, sizeof (int));
    if ( err != 0 ) {
        ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Error setsockopt failed err=%d", WSAGetLastError()));
        goto cleanup;
    }
    int Size = sizeof (int);
    err = getsockopt(s, SOL_SOCKET, SO_RCVBUF,  (char *)&RealRecvBufferSize, &Size);
    if ( err != 0 ) {
        ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Error setsockopt failed err=%d", WSAGetLastError()));
        goto cleanup;
    }
    if (RealRecvBufferSize != RecvBufferSize) {
        ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Error: RealSendBufferSize = %d", RealRecvBufferSize));
    }
cleanup:    
    return;
}


void PrintConnectTime(SOCKET s)
{
    INT seconds;
    INT bytes = sizeof(seconds);
    INT err = getsockopt( s , SOL_SOCKET, SO_CONNECT_TIME,
                          (char *)&seconds, (PINT)&bytes );
    if ( err != NO_ERROR ) {
        printf( "getsockopt(SO_CONNECT_TIME) failed: %ld\n", WSAGetLastError( ) );
        exit(1);
    }
    printf("Seconds = %d\n", seconds);
}

DWORD WINAPI WorkerThreadFunc( LPVOID lpParam )
{
    BOOL ret;
    DWORD NumberOfBytes;
    ULONG_PTR CompletionKey;
    OVERLAPPED* lpOverlapped;
    OverlappedSend *pOverLappedSend;
    int iRet;
    AvailableOperations Operation;
    
    while (TRUE) {
        ret = GetQueuedCompletionStatus(
                                g_ComplitionPort,
                                &NumberOfBytes,
                                &CompletionKey,
                                &lpOverlapped,
                                INFINITE);
        if(ret == 0) {
            DWORD gle = GetLastError();
            ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Error GetQueuedCompletionStatus failed gle = %d\n", gle) );
            // Nothing more to send - signal compleation and exit
            // There is a bug here, there might be other overlapped operations pending ??????
            ASSERT(FALSE);
            SetEvent(pOverLappedSend->s_OperationsDoneEvent);
            continue;
        }        
        
        pOverLappedSend = CONTAINING_RECORD(lpOverlapped, OverlappedSend, Overlapped);
        // Work on the object itself:

        int WaitForComplition;
        bool TakeNextAction;

        if (pOverLappedSend == NULL) {
            // This is a direct command to quit,
            printf("Got a command to stop\n");
            break;
        }
        
        EnterCriticalSection(&pOverLappedSend->s_Lock);
        Operation = pOverLappedSend->Operation;
        ASSERT((Operation == SEND) ||(Operation == RECV) || Operation ==ACCEPTEX);

        
        if((Operation == SEND) ||(Operation == RECV)) {
            ASSERT(pOverLappedSend->s_RemainingData >= NumberOfBytes);
            pOverLappedSend->s_RemainingData -= NumberOfBytes;
            if (Operation == SEND) {
                OverlappedSend::s_SendsInAir -= NumberOfBytes;
            } else {
                if (NumberOfBytes != 0) {
                    // NumberOfBytes == 0 means that is the first overlapped and not a real complition
                    OverlappedSend::s_SendsInAir -= pOverLappedSend->BufferSize;
                }
            }
            ASSERT(OverlappedSend::s_SendsInAir >= 0);
            OverlappedSend::s_NumberOfOperatopnsInAir--;
            ASSERT(OverlappedSend::s_NumberOfOperatopnsInAir >= 0);
        } else {
/*
        We are not counting them for now
            if (pOverLappedSend->sAcceptSocket != -1) {
                pOverLappedSend->s_NumberOfOverlappedAccepts--;
                ASSERT(pOverLappedSend->s_NumberOfOverlappedAccepts >= 0);
            }
*/            
        }

        WaitForComplition  = Operation == SEND ? 1 : 0;

        if((Operation == SEND) ||(Operation == RECV)) {

            TakeNextAction = pOverLappedSend->s_RemainingData - OverlappedSend::s_SendsInAir * WaitForComplition > 0;
            if (TakeNextAction != false) {
                OverlappedSend::s_NumberOfOperatopnsInAir++;
                OverlappedSend::s_SendsInAir += pOverLappedSend->BufferSize;
            }else {
            if ((OverlappedSend::s_RemainingData == 0) &&
                (OverlappedSend::s_NumberOfOperatopnsInAir == 0)){
                    printf("Finished all operations, function can return\n");
                    SetEvent(pOverLappedSend->s_OperationsDoneEvent);
            }

            }            
        } else {
            // For acceptex we continue forever.
            TakeNextAction = true;
        }
        LeaveCriticalSection(&pOverLappedSend->s_Lock);
        if (Operation == ACCEPTEX && (pOverLappedSend->sAcceptSocket != -1)) {
            // for Acceptex, a new socket was just created, we have to work on it
            // in any case
           
            unsigned char *LocalSockaddr;
            unsigned char *RemoteSockaddr;
            int l1, l2;

            pGetAcceptExSockaddrs(
                    pOverLappedSend->Buffer, 
                    pOverLappedSend->BufferSize,
                    LOCALADDRESSLENGTH,
                    DWREMOTEADDRESSLENGTH,
                    (LPSOCKADDR*)&LocalSockaddr,
                    &l1,
                    (LPSOCKADDR*)&RemoteSockaddr,
                    &l2
                );
            ASSERT(l1 == 16);
            ASSERT(l2 == 16);
            ASSERT(LocalSockaddr[0] == 2);
            ASSERT(LocalSockaddr[1] == 0);
            ASSERT(RemoteSockaddr[0] == 2);
            ASSERT(RemoteSockaddr[1] == 0);
            
            printf("Recieved a connection, our address %d.%d.%d.%d:%d, their address %d.%d.%d.%d:%d\n", 
                LocalSockaddr[4], LocalSockaddr[5], LocalSockaddr[6], LocalSockaddr[7],
                256 * LocalSockaddr[2] + LocalSockaddr[3],
                RemoteSockaddr[4], RemoteSockaddr[5], RemoteSockaddr[6], RemoteSockaddr[7],
                256 * RemoteSockaddr[2] + RemoteSockaddr[3]);

            
//?????? what is the size that we should send here ????????

//??? probably NumberOfBytes
        printf("Acceptex has returned %d bytes\n", NumberOfBytes);
        ASSERT(NumberOfBytes == pOverLappedSend->BufferSize);

            ServerHandleNewAccept(pOverLappedSend->sAcceptSocket, pOverLappedSend->Buffer, pOverLappedSend->BufferSize);

        }

        // The expression bellow tells that for send one has to stop sending,
        // once it has sended all data, while for receive he has to wait for compleation
        // of all operations

        if (TakeNextAction) {
            // do the next send
            WSABUF Buffers;
            Buffers.len = pOverLappedSend->BufferSize;
            Buffers.buf = (char *) pOverLappedSend->Buffer;
            
            DWORD dwFlags = 0;

            if (Operation == SEND) {
                
                iRet = WSASend(
                    pOverLappedSend->Socket,
                    &Buffers,
                    1,
                    &pOverLappedSend->NumberOfBytesOut,
                    dwFlags,
                    &pOverLappedSend->Overlapped,
                    NULL
                    );
                    ASSERT((iRet ==0) || (iRet == SOCKET_ERROR && WSAGetLastError() == WSA_IO_PENDING ));
            } else if (Operation == RECV){
                iRet = WSARecv(
                    pOverLappedSend->Socket,
                    &Buffers,
                    1,
                    &pOverLappedSend->NumberOfBytesOut,
                    &dwFlags,
                    &pOverLappedSend->Overlapped,
                    NULL
                    );
                    ASSERT((iRet ==0) || (iRet == SOCKET_ERROR && WSAGetLastError() == WSA_IO_PENDING ));
                
            } else {
                pOverLappedSend->sAcceptSocket = socket (AF_INET_FAMILY , SOCK_STREAM, IPPROTO_TCP);
                if (pOverLappedSend->sAcceptSocket == -1) {
                    printf("Error creating socket\n");
                    return 1;
                }

                static DWORD g_BytesReceived ;

//Sleep(10000); // I wan't to have a connection before the accept ex

                iRet = pAcceptEx(
                    pOverLappedSend->Socket,
                    pOverLappedSend->sAcceptSocket,
                    pOverLappedSend->Buffer, 
                    pOverLappedSend->BufferSize,
                    LOCALADDRESSLENGTH,
                    DWREMOTEADDRESSLENGTH,
                    &g_BytesReceived, //??????
                    &pOverLappedSend->Overlapped
                    );
                ASSERT((iRet == TRUE) || (iRet == 0 && WSAGetLastError() == WSA_IO_PENDING ));
            }

        }      

    }
    return 0;
}

VOID ConnectToCompleationPort(SOCKET newfd)
{
    ASSERT(g_ComplitionPort != NULL);
    HANDLE x;
    x = CreateIoCompletionPort((HANDLE)newfd, g_ComplitionPort, NULL, 0);
    ASSERT(x == g_ComplitionPort);
}

void CreateComplitionPort()
{

    ASSERT(g_ComplitionPort == NULL);
    g_ComplitionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, NULL, 0);
    if(g_ComplitionPort == NULL) {
        printf("Create complition port failed err=%d", GetLastError());
        exit(1);
    }

    for (int i=0; i <g_NumberOfThreads;i++ ) {
        // Create the threads that will work on the complitions
        g_pThreads[i] = CreateThread(NULL, 0 ,WorkerThreadFunc ,NULL, 0,NULL );       
    }
    InitializeCriticalSection(&OverlappedSend::s_Lock);
    OverlappedSend::s_OperationsDoneEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
    ASSERT(OverlappedSend::s_OperationsDoneEvent != NULL);

    OverlappedSend::s_NumberOfOverlappedAccepts = 0;
    OverlappedSend::s_RemainingData = 0;

}

void CloseComplitionPort()
{
    BOOL ret;
    // Post a complition thread end message
    for (int i=0; i<g_NumberOfThreads ;i++ ) {
        // Sending an overlapped of NULL works, since the
        // receiver tests first for end of test
        BOOL  ret = PostQueuedCompletionStatus(
                              g_ComplitionPort,
                              0,
                              NULL,
                              NULL
                                );        
        ASSERT(ret != 0);
        
        
    }
    WaitForMultipleObjects(g_NumberOfThreads,g_pThreads, TRUE, INFINITE);
    ASSERT(OverlappedSend::s_NumberOfOverlappedAccepts == 0);
    if(OverlappedSend::s_RemainingData != 0) {
        ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Finishd with errors %I64d remaining bytes\n", OverlappedSend::s_RemainingData) );
    }
    CloseHandle(g_ComplitionPort);
    g_ComplitionPort = NULL;
    for (int i=0; i <g_NumberOfThreads;i++ ) {
        // Create the threads that will work on the complitions
        BOOL ret = CloseHandle(g_pThreads[i]);
        ASSERT(ret != 0);
    }
    ret = CloseHandle(OverlappedSend::s_OperationsDoneEvent);
    ASSERT(ret != 0);
    DeleteCriticalSection(&OverlappedSend::s_Lock);
    
}

// Today this works for send and receive
void SendDataOverlapped(
    SOCKET newfd, 
    AvailableOperations Operation,
    int NumberOfOverlappedSend, 
    __int64 NumberOfBuffers, 
    __int64 BufferSize, 
    BOOL SendPacketNumber, 
    int PostExactRecvPackats
    )
{
    int i ;
    VOID *buffer = NULL;
    double elapsed;
    __int64 acc = 0;
    OverlappedSend *pOverLappedSend = NULL;
    BOOL ret;

    ASSERT(NumberOfOverlappedSend > 0);

    int OperationSize;
    if (Operation == SEND || PostExactRecvPackats) {
        OperationSize = (size_t)BufferSize;
    } else {
        OperationSize = 400000;
    }

    printf("Starting to %s asyncroniously (%d overlapped) %I64d messages of size %I64d\n", 
        Operation == SEND ? "send" : "recv" , NumberOfOverlappedSend, NumberOfBuffers, BufferSize );

    OverlappedSend **ppOverlappedSendOperations = new OverlappedSend *[NumberOfOverlappedSend];
    ASSERT(ppOverlappedSendOperations != NULL);

    ConnectToCompleationPort(newfd);
    pOverLappedSend->s_RemainingData = NumberOfBuffers * BufferSize;
    OverlappedSend::s_SendsInAir = 0;
    OverlappedSend::s_NumberOfOperatopnsInAir = NumberOfOverlappedSend;
    
    double Start = GetTimeMs();

    for(i = 0; i < NumberOfOverlappedSend; i++) {

        buffer = new char [(size_t)BufferSize];
        if (buffer == NULL) {
            printf("Error allocating buffer\n");
            exit(1);
        }

        pOverLappedSend = new OverlappedSend;
        ASSERT(pOverLappedSend != NULL);
        memset(pOverLappedSend, 0, sizeof OverlappedSend);
        ppOverlappedSendOperations[i] = pOverLappedSend;
        pOverLappedSend->Operation = Operation;

        // Set it's fields
                     
        pOverLappedSend->Socket = newfd;
        pOverLappedSend->Buffer = buffer;
        pOverLappedSend->BufferSize = (size_t)BufferSize;
        pOverLappedSend->PostExactRecvPackats = PostExactRecvPackats;

        // Post the message that will do the first send
        ret = PostQueuedCompletionStatus(
                              g_ComplitionPort,
                              (DWORD)0,
                              NULL,
                              &pOverLappedSend->Overlapped
                              );
        ASSERT(ret != 0);

    }

    // wait for the send 
    WaitForSingleObject(pOverLappedSend->s_OperationsDoneEvent, INFINITE);
    elapsed = (GetTimeMs() - Start) / 1000;
    
    if (pOverLappedSend->s_RemainingData == 0) {    
        ColorPrintf(FOREGROUND_GREEN | FOREGROUND_INTENSITY, ("Finishd %s correctly %f mbytes/sec\n", Operation == SEND ? "sending" : "receiving", NumberOfBuffers * BufferSize /elapsed/1024/1024) );
    } else {
        ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Finishd %s with errors %I64d remaining bytes\n", Operation == SEND ? "sending" : "receiving", pOverLappedSend->s_RemainingData) );
    }
//Cleanup:
    for(i = 0; i < NumberOfOverlappedSend; i++) {
        delete []ppOverlappedSendOperations[i]->Buffer;
        delete ppOverlappedSendOperations[i];
    }
    delete ppOverlappedSendOperations;
}



void SendData(SOCKET newfd, __int64 NumberOfBuffers, __int64 BufferSize, BOOL SendPacketNumber)
{
    int i ;
    double d;
    CHAR *buffer = NULL;
    double elapsed;
    __int64 acc = 0;

    buffer = new char [(size_t)BufferSize];
    if (buffer == NULL) {
        printf("Error allocating buffer\n");
        exit(1);
    }

    printf("Starting to send syncroniously %I64d messages of size %I64d\n", NumberOfBuffers, BufferSize );

    i = 0;

    for (i = 0; i < 10; i++) {
        buffer[i] = 'a' + i;
    }
    double Start = GetTimeMs();

    for (d=0; d < NumberOfBuffers; d++) {
        int j;
        if (SendPacketNumber) {
            memset(buffer, (char) d, (size_t)BufferSize);
        }
        
        j = send(newfd, buffer, (size_t)BufferSize, 0 );
            acc += j;
        if (j!=BufferSize) {
            ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Error send not compleated totaly sent %I64d (last send = %d, gle = %d)\n", acc, j,  WSAGetLastError()) );
            goto Cleanup;
        }
    }
    elapsed = (GetTimeMs() - Start) / 1000;
    ColorPrintf(FOREGROUND_GREEN | FOREGROUND_INTENSITY, ("Finishd sending correctly %f mbytes/sec (%f seconds)\n", acc/elapsed/1024/1024, elapsed ) );
Cleanup:
    free(buffer);
}

#if 0
void RecvData(SOCKET newfd, __int64 NumberOfBuffers, __int64 BufferSize, int  PostExactRecvPackats )
{
    int ReadSize;    
    int read1 = 1;
    unsigned char *p;
    double elapsed;
    __int64 acc = 0;
    HANDLE hFile;
    DWORD dwBytesWritten = 0;
    int reminder = 0;
    int i;
    BOOL ret;
    int WriteLocation = 0;
    if (PostExactRecvPackats) {
        ReadSize = (int)BufferSize;
    } else {
        ReadSize = 400000;
    }

    hFile = CreateFile("z:\\data.bin",  // file name 
        GENERIC_READ | GENERIC_WRITE, // open for read/write 
        0,                            // do not share 
        NULL,                         // default security 
        CREATE_ALWAYS,                // overwrite existing file
        FILE_FLAG_NO_BUFFERING,        // normal file 
        NULL);                        // no template 
    if (hFile == INVALID_HANDLE_VALUE) 
    { 
        printf("Could not open file.");
        return ;
    } 

    printf("Starting to recv syncroniously %I64d messages of size %I64d Posting recv of %d\n", NumberOfBuffers, BufferSize, ReadSize);
    
    char *buffer = new char [ReadSize];
    if (buffer == NULL) {
        printf("Error allocating memory");
        exit(4);
    }

    double  Start = GetTimeMs();

    while ((read1 != -1) /*&& (acc < (double)BufferSize * NumberOfBuffers)*/) {
        read1 = recv(newfd, buffer, ReadSize, 0);
        if (read1 == -1 || read1 == 0) {
            printf("Finsihed reading, total read %I64d bytes last read =%d\n", acc, read1 );
            break;
        }
        else {
            acc += read1;

            reminder += (read1 % 512);

            ret = WriteFile(hFile, buffer, read1 - (read1 % 512), 
                &dwBytesWritten, NULL);
            if (ret ==0 ) {
                printf("Write failed error = %d\n", GetLastError());
                break;
            }
            WriteLocation += dwBytesWritten;
            if (WriteLocation > 250 * 1024*1024) {
                SetFilePointer(hFile, 0, 0, FILE_BEGIN);
                WriteLocation = 0;
            }


//          printf("read returned %d \"%c%c%c%c\"\n",read1,
//          buffer[0],buffer[1], buffer[2], buffer[3]);
        }
    }

    if (acc != (BufferSize * NumberOfBuffers)) {
        ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY,("Error, expected to read %I64d but read %I64d",
            BufferSize * NumberOfBuffers, acc)  );
        elapsed = (GetTimeMs() - Start) / 1000;
        ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY,("Finishd reading correctly %lf mbytes/sec (time = %lf)\n", acc / elapsed/1024/1024, elapsed));
        
    } else {
        elapsed = (GetTimeMs() - Start) / 1000;
        ColorPrintf(FOREGROUND_GREEN | FOREGROUND_INTENSITY,("Finishd reading correctly %lf mbytes/sec (time = %lf)\n", acc / elapsed/1024/1024, elapsed));
        printf("Totaly wrote %I64d, reminder=%d\n",acc,reminder);        
    }
    delete []buffer;
    CloseHandle(hFile);
}
#else
void RecvData(SOCKET newfd, __int64 NumberOfBuffers, __int64 BufferSize, int  PostExactRecvPackats )
{
    int ReadSize;    
    int read1 = 1;
    unsigned char *p;
    double elapsed;
    __int64 acc = 0;
    int i;
    double  Start = GetTimeMs();
    if (PostExactRecvPackats) {
        ReadSize = (int)BufferSize;
    } else {
        ReadSize = 400000;
    }

    printf("Starting to recv syncroniously %I64d messages of size %I64d Posting recv of %d\n", NumberOfBuffers, BufferSize, ReadSize);
    
    char *buffer = new char [ReadSize];
    if (buffer == NULL) {
        printf("Error allocating memory");
        exit(4);
    }

    while ((read1 != -1) /*&& (acc < (double)BufferSize * NumberOfBuffers)*/) {
        read1 = recv(newfd, buffer, ReadSize, 0);
        if (read1 == -1 || read1 == 0) {
            printf("Finsihed reading, total read %I64d bytes last read =%d\n", acc, read1 );
            break;
        }
        else {
            acc += read1;

//          printf("read returned %d \"%c%c%c%c\"\n",read1,
//          buffer[0],buffer[1], buffer[2], buffer[3]);
        }
    }

    if (acc != (BufferSize * NumberOfBuffers)) {
        ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY,("Error, expected to read %I64d but read %I64d",
            BufferSize * NumberOfBuffers, acc)  );
        elapsed = (GetTimeMs() - Start) / 1000;
        ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY,("Finishd reading correctly %lf mbytes/sec (time = %lf)\n", acc / elapsed/1024/1024, elapsed));
        
    } else {
        elapsed = (GetTimeMs() - Start) / 1000;
        ColorPrintf(FOREGROUND_GREEN | FOREGROUND_INTENSITY,("Finishd reading correctly %lf mbytes/sec (time = %lf)\n", acc / elapsed/1024/1024, elapsed));
    }
    delete []buffer;
}
#endif
void PrintUsage(char *name)
{
    printf("The program might be used in client or server mode\n");
    printf("usage is %s <server> <port number> <UseOverlappedAccept> <OverlappedAcceptDataSize>\n", name);
    printf("usage is %s <client> <remote_ip> <remote_port number> <local_port> <option number> <WritePacketNumber> <UseOverlappedSend> <UseOverlappedRecv> <PostExactRecvPackats> <Number of packets> <PacketSize>\n", name);
    printf("The option will also be sent to the remote side\n");
    printf("Available options are:\n");
    printf("    1 - just receive packets forever (the other side will send xxx bytes yyy times)\n");
    printf("    2 - send xxx times yyy bytes (based on input from network)\n");
    printf("    3 - measure the latency of the network (pingpong) xxx times yyy bytes \n");
    printf("    WritePacketNumber can be 0 (no) or 1 (yes)\n");
    printf("    UseOverlappedSend can be 0 (no) or > 0 (number of operations to use)\n");
    printf("    UseOverlappedRecv can be 0 (no) or > 0 (number of operations to use)\n");
    printf("    PostExactRecvPackats can be 0 or 1 (If it is 0, Big receive packets will be used, if 1 buffe size will be used)\n");    
    printf("    UseOverlappedAccept can be 0 (Use Accept) or > 0 (number of overlapped operations to use in AcceptEx)\n");    
    printf("    OverlappedAcceptDataSize size of the buffer to have for receive\n");    
    printf("\nExamples to the usage of the program are:\n");
    printf("%s server 2222 0 0\n", name);
    printf("%s client 11.4.8.133 2222 1000 1 0 0 0 0 100 65536\n", name);
    printf("%s pingpong 11.4.8.133 2222 10000 10\n", name);
    
}


// Where are we running?
VOID PrintProvider(SOCKET s)
{
    WSAPROTOCOL_INFO  ProtocolInfo;
    int size1 = sizeof ProtocolInfo;
    int ret = getsockopt(s, SOL_SOCKET, SO_PROTOCOL_INFO , (char *)&ProtocolInfo, &size1);
    if (ret != 0) {
        printf("getsockopt failed\n");
            return;
    } 
    if (ProtocolInfo.dwServiceFlags1 & XP1_IFS_HANDLES) {
        printf("ipoib\n"); 
    } else {
        if (!strncmp("SDP",ProtocolInfo.szProtocol,3 )) {
            printf("running on sdp\n");
        } else if(!strncmp("MSAFD Tcpip",ProtocolInfo.szProtocol,11 )){
            printf("running on wsd\n");
        } else {
            printf("running on unknown provider %s", ProtocolInfo.szProtocol);
        }
    }
}

// Keep recieving until all data is received
int FullRecv(SOCKET s, char* buf, int len, int flags)
{
    int Total = 0, ret;
    while (Total != len) {
        ret = recv(s, buf, len, flags);
        if (ret == 0 || ret == -1) {
            break;
        }
        Total += ret;
    }
    return Total;

}

const int NUMBEROFWARMUPCYCLES = 200;

int PingPong(int argc, char *argv[])
{
    struct sockaddr server_addr;
    int port = atoi(argv[3]);
    int  ret;
    SOCKET s;
    int BufferSize, NumberOfBuffers;
    int i;
    char *buffer = NULL;
    double Start = 0;

    NumberOfBuffers = atoi(argv[4]);
    BufferSize = atoi(argv[5]);
    
    printf("PingPong test Using port %d NumberOfBuffers = %d , BufferSize = %d \n", 
        port, 
        NumberOfBuffers,
        BufferSize
        );


    buffer =  new char [BufferSize];
    if (buffer == NULL) {
        printf("Error allocating memory");
        exit(4);
    }

    server_addr = addressFromString(argv[2], port);

    s = socket (AF_INET_FAMILY, SOCK_STREAM, 0);
    if (s == -1) {
        printf("Error creating socket\n");
        goto Cleanup;
    }

    ret = connect(s, &server_addr, sizeof(server_addr));
    if (ret ==0) {
        printf("connect succeeded\n");
    } else {
        printf("connect failed errno = %d\n", WSAGetLastError());
        ret = 1;
        goto Cleanup;
    }

    PrintProvider(s);

    // send data to remote host

    char message[60];
    sprintf(message, "3,%d,%d,", NumberOfBuffers, BufferSize );

    ret = send(s, message, strlen (message), 0);
    if (ret != strlen (message)) {
        printf("Error in send winsock error=%d\n", WSAGetLastError());
        ret = 1;
        goto Cleanup;
    } 
   
    for(i =0 ; i < NumberOfBuffers; i++) {

        if (i == NUMBEROFWARMUPCYCLES) {
            // Give the test 200 iterations for warm up
            Start = GetTimeMs();
        }

        ret = send(s, buffer, (size_t)BufferSize, 0 );
        if (ret != BufferSize) {
            ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Error send not compleated i = %d, gle = %d)\n", i,  WSAGetLastError()) );
            goto Cleanup;
        }

        ret = FullRecv(s, buffer, (size_t)BufferSize, 0);
        if (ret != BufferSize) {
            ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Error recv not compleated i = %d, gle = %d)\n", i,  WSAGetLastError()) );
            goto Cleanup;
        }
    }
    double elapsed = (GetTimeMs() - Start) / 1000;
    if (NumberOfBuffers >= 1000) {
        ColorPrintf(FOREGROUND_GREEN | FOREGROUND_INTENSITY,("Pingpong finishd message size %d  latency %lfus\n", BufferSize,   elapsed / (NumberOfBuffers - NUMBEROFWARMUPCYCLES) * 1000000 / 2 ));
    } else {
        ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY,("Pingpong finishd, to get results please use at least 1000 buffers !!!)\n"));
    }
    
Cleanup:
    delete []buffer;
    CloseSocket(s);

    return 0;
}

int ServerPingPong(SOCKET s, int NumberOfBuffers, int BufferSize)
{
    int i, ret;
    char *buffer = NULL;

    buffer =  new char [BufferSize];
    if (buffer == NULL) {
        printf("Error allocating memory");
        exit(4);
    }
    for(i =0 ; i < NumberOfBuffers; i++) {

        ret = FullRecv(s, buffer, (size_t)BufferSize, 0);
        if (ret != BufferSize) {
            ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Error recv not compleated i = %d, gle = %d)\n", i,  WSAGetLastError()) );
            goto Cleanup;
        }

        ret = send(s, buffer, (size_t)BufferSize, 0 );
        if (ret != BufferSize) {
            ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Error send not compleated i = %d, gle = %d)\n", i,  WSAGetLastError()) );
            goto Cleanup;
        }

    }
Cleanup:
    delete []buffer;
    return ret;


}

int Connect(int argc, char *argv[])
{
    struct sockaddr server_addr, client_addr;
    int remote_port = atoi(argv[3]);
    int locale_port = atoi(argv[4]);
    int  ret;
    SOCKET s;
    int Send = 0;
    int BufferSize, NumberOfBuffers;
    int result = 0;
    bool SendPacketNumber;
    int  NumberOfOverlappedSend = 0;
    int  NumberOfOverlappedRecv = 0;
    int  PostExactRecvPackats = 0;


    if (argv[5][0] == '1') {
        Send = 1;
    }else if (argv[5][0] == '2') {
        Send = 0;
    } else {
        printf("Error - don't know if to send or receive\n");
        exit(1);
    }

    if (argv[6][0] == '0') {
        SendPacketNumber = false;
    }else if (argv[6][0] == '1') {
        SendPacketNumber = true;
    } else {
        printf("Error - don't know if to send the packet number\n");
        exit(1);
    }

    NumberOfOverlappedSend = atoi(argv[7]);

    if ((NumberOfOverlappedSend > 0 ) && (SendPacketNumber == true)) {
        printf("Warning, Sending packet number is not supported yet in overlapped send\n");
        exit(1);
    }

    NumberOfOverlappedRecv = atoi(argv[8]);

    PostExactRecvPackats = atoi(argv[9]);
        
    NumberOfBuffers = atoi(argv[10]);
    BufferSize = atoi(argv[11]);
    
    printf("Using remote port %d %s %s %s %s %s NumberOfBuffers = %d , BufferSize = %d \n", 
        remote_port, 
        Send == 1 ? "sending" : "recieving", 
        SendPacketNumber ? "Packets Are filed with data" : "Packets are not filled with data",
        NumberOfOverlappedSend > 0 ? "Using overlapped send" : "not using overlapped send", 
        NumberOfOverlappedRecv > 0 ? "Using overlapped recv" : "not using overlapped recv",
        PostExactRecvPackats ? "Posting the exact recv buffers" : "Posting big recv buffers", 
        NumberOfBuffers,
        BufferSize
        );

    server_addr = addressFromString(argv[2], remote_port);

    s = socket (AF_INET_FAMILY, SOCK_STREAM, 0);
    if (s == SOCKET_ERROR) {
        printf("Error %d creating socket\n", GetLastError());
        goto cleanup;
    }

    client_addr = addressFromString("0.0.0.0", locale_port);
    ret =   bind(s, (struct sockaddr*)&client_addr, sizeof(client_addr));
    if (ret < 0) {
        printf("Error in bind\n" );
        goto cleanup;
    }
    
    

    ret = connect(s, &server_addr, sizeof(server_addr));
    if (ret ==0) {
        printf("connect succeeded\n");
    } else {
        printf("connect failed errno = %d\n", WSAGetLastError());
            result = 1;
            goto cleanup;
    }

    PrintProvider(s);

    // Send the other side the correct information: 
    // send(=1), NumberOfOverlappedSend ,NumberOfBuffers, BufferSize, SendPacketNumber
    // or receive(=2),NumberOfOverlappedSend ,NumberOfBuffers, BufferSize, SendPacketNumber
    char message[60];
    sprintf(message, "%d,%d,%d,%d,%d,%d,%d,", 
        Send ? 1 : 2,
        NumberOfOverlappedSend ,
        NumberOfOverlappedRecv,
        PostExactRecvPackats,
        NumberOfBuffers, 
        BufferSize, 
        SendPacketNumber ? 1 : 0
        );

//    Sleep(15000);

    ret = send(s, message, strlen (message), 0);
    if (ret != strlen (message)) {
        printf("Error in send winsock error=%d\n", WSAGetLastError());
        result = 1;
        goto cleanup;
    } 
    
    if (Send) {
        if (NumberOfOverlappedSend > 0 ) {
            SendDataOverlapped(s, SEND,NumberOfOverlappedSend, NumberOfBuffers, BufferSize, SendPacketNumber, PostExactRecvPackats);
        } else {
            SendData(s, NumberOfBuffers, BufferSize, SendPacketNumber);
        }
    } else {
        if (NumberOfOverlappedRecv > 0 ) {
            SendDataOverlapped(s, RECV,NumberOfOverlappedRecv, NumberOfBuffers, BufferSize, SendPacketNumber, PostExactRecvPackats);
        } else {
            RecvData(s, NumberOfBuffers, BufferSize, PostExactRecvPackats);
        }
    }
cleanup:
    ret = CloseSocket(s);
    if (ret == 0) {
        printf("CloseSocket succeeded\n");
    } else {
        printf("CloseSocket failed\n");
    }

    return result;
}

int ReadNumber(SOCKET fd, char *pBuffer, int BufferSize, int *pCurrentPlace)
{
    const int DataSize = 20;
    char Data[DataSize];
    int i = 0, ret;
    ASSERT(BufferSize >= *pCurrentPlace);
//    printf("ReadNumber called\n");

    while (1) {
        if (*pCurrentPlace < BufferSize) {
            Data[i] = pBuffer[(*pCurrentPlace)++];
        } else {
            ret = recv(fd,&Data[i],1,0);
            if (ret != 1) {
                printf("Error reading data 1 ret=%d\n", ret);
                exit(1);
            }
        }
        printf("%c",Data[i]);
        if (Data[i] == ',') break;
        i++;
        if (i == DataSize) {
            printf("Error reading data 2\n");
            exit(1);
        }
    }
    return atoi(Data);
}

int ServerHandleNewAccept(SOCKET newfd, VOID *FirstBuffer, int BufferLen)
{
    int UseOverlappedSend = 0;
    int UseOverlappedRecv = 0;
    int  PostExactRecvPackats = 0;
    BOOL SendPacketNumber = 0;

    int option;
    int ret;
    int BufferSize, NumberOfBuffers;

    int CurrentPlace = 0;


    option = ReadNumber(newfd, (char *)FirstBuffer, BufferLen, &CurrentPlace);
    if (option == 1 || option == 2) {
        UseOverlappedSend = ReadNumber(newfd, (char *)FirstBuffer, BufferLen, &CurrentPlace);
        UseOverlappedRecv = ReadNumber(newfd, (char *)FirstBuffer, BufferLen, &CurrentPlace);
        PostExactRecvPackats = ReadNumber(newfd, (char *)FirstBuffer, BufferLen, &CurrentPlace);
        NumberOfBuffers = ReadNumber(newfd, (char *)FirstBuffer, BufferLen, &CurrentPlace);
        BufferSize = ReadNumber(newfd, (char *)FirstBuffer, BufferLen, &CurrentPlace);
        SendPacketNumber = ReadNumber(newfd, (char *)FirstBuffer, BufferLen, &CurrentPlace);
    } else {
        if (option != 3) {
            printf("Unknown option\n");
            goto Cleanup;
        }
        NumberOfBuffers = ReadNumber(newfd, (char *)FirstBuffer, BufferLen, &CurrentPlace);
        BufferSize = ReadNumber(newfd, (char *)FirstBuffer, BufferLen, &CurrentPlace);
    }

    if (CurrentPlace != BufferLen) {
        ColorPrintf(FOREGROUND_RED | FOREGROUND_INTENSITY, ("Error First buffer size was too big CurrentPlace = %d BufferLen = %d\n", CurrentPlace, BufferLen) );
        exit(1);
    }

    printf("%s ,%d overlapped Send, %d overlapped recv, NumberOfBuffers = %d , BufferSize =%d PacketsFilled=%s \n", 
        option == 2 ? "sending" : "recieving",
        UseOverlappedSend,
        UseOverlappedRecv,
        NumberOfBuffers,
        BufferSize,
        SendPacketNumber ? "true" : "false"
        );
    
    if (option == 1) {
        if (UseOverlappedRecv > 0) {
            SendDataOverlapped(newfd, RECV, UseOverlappedRecv, NumberOfBuffers, BufferSize, SendPacketNumber, PostExactRecvPackats);
        } else {
            RecvData(newfd, NumberOfBuffers, BufferSize, PostExactRecvPackats);
        }
    }
    if (option == 2) {
        if (UseOverlappedSend > 0) {
            SendDataOverlapped(newfd, SEND, UseOverlappedSend, NumberOfBuffers, BufferSize, SendPacketNumber, PostExactRecvPackats);
        } else {
            SendData(newfd, NumberOfBuffers, BufferSize, SendPacketNumber);
        }
    }

    if (option == 3) {
        ServerPingPong(newfd, NumberOfBuffers, BufferSize);
    }
Cleanup:
    
    ret = CloseSocket(newfd);
    if (ret == 0) {
        printf("CloseSocket succeeded\n");
    } else {
        printf("CloseSocket failed\n");
    }

    return 0;

}

int Server(int argc, char *argv[])
{

    u_short port;
    SOCKET s, newfd;
    int ret;
    struct sockaddr_in my_addr, client_addr;
    int addr_len = sizeof(client_addr);
    int UseOverlappedAccept = 0;
    int OverlappedAcceptRecvSize = 0;
    int i;

    port = (u_short)atoi(argv[2]);
    UseOverlappedAccept = atoi(argv[3]);
    if (UseOverlappedAccept > 0) {
        OverlappedAcceptRecvSize = atoi(argv[4]);
    }

    printf("Using port %d, %d overlapped accepts Accept Recv packet size = %d\n", port, UseOverlappedAccept, OverlappedAcceptRecvSize);
    
    memset(&my_addr, 0, sizeof(my_addr));
    my_addr.sin_family = AF_INET;
    my_addr.sin_port = htons(port);
    s = socket (AF_INET_FAMILY , SOCK_STREAM, IPPROTO_TCP);
    if (s == -1) {
        printf("Error creating socket %d\n", GetLastError());
        return 1;
    }
    //IncreaseRecvSocketSize(s, 4 * 1024*1024);

    ret =	bind(s, (struct sockaddr*)&my_addr, sizeof(my_addr));
    if (ret < 0) {
        printf("Error in bind\n" );
        return 2;
    }

    ret = listen(s, 5);
    if (UseOverlappedAccept == 0) {
        for(;;) {
            struct sockaddr name;
            int namelen;
            unsigned char *p;

            newfd  = accept(s, (struct sockaddr *)&client_addr, &addr_len);
            namelen = sizeof(name);
            getpeername(newfd, &name, &namelen);
            p = (unsigned char *) &client_addr;
            printf("received a client %d from %d.%d.%d.%d:%d\n", newfd, 
                p[4], p[5],p[6], p[7], p[2] *256 +  p[3] ) ;

            

            ServerHandleNewAccept(newfd, NULL,0);

        }
    }else {
        // We shuold be using overlapped accepts (currently we post them all)
//            Sleep(20000);// ????? Can this function return immidiatly ??????????
        OverlappedSend **ppOverlappedSendOperations = new OverlappedSend *[UseOverlappedAccept];

        ConnectToCompleationPort(s);
        
        OverlappedSend *pOverLappedSend = NULL;            
        ASSERT(ppOverlappedSendOperations != NULL);

        OverlappedSend::s_NumberOfOverlappedAccepts = UseOverlappedAccept;

        // In any case, we only post two accept operations.           
        for (i=0; i <UseOverlappedAccept; i++ ) {
            pOverLappedSend = new OverlappedSend;
            ASSERT(pOverLappedSend != NULL);
            memset(pOverLappedSend, 0, sizeof OverlappedSend);
            ppOverlappedSendOperations[i] = pOverLappedSend;
            pOverLappedSend->Operation = ACCEPTEX;
            pOverLappedSend->Socket = s;
            pOverLappedSend->sAcceptSocket = -1;
            pOverLappedSend->BufferSize = OverlappedAcceptRecvSize;
            pOverLappedSend->Buffer = new char [pOverLappedSend->BufferSize + LOCALADDRESSLENGTH + DWREMOTEADDRESSLENGTH];
            ASSERT(pOverLappedSend->Buffer != NULL);
            
            // Post the message that will do the first send
            ret = PostQueuedCompletionStatus(
                                  g_ComplitionPort,
                                  (DWORD)0,
                                  NULL,
                                  &pOverLappedSend->Overlapped
                                  );
            ASSERT(ret != 0);
        }

        //Actually we should wait here for ever. (at least for now)
        Sleep(-1);
        ASSERT(FALSE);

        // In the case that we don't want to wait for ever, some cleaning
        // is required, (bellow is an example).
/*    
        for(i = 0; i < UseOverlappedAccept; i++) {
            delete []ppOverlappedSendOperations[i]->Buffer;
            delete ppOverlappedSendOperations[i];
        }
        delete ppOverlappedSendOperations;
*/

    }

    return 0;
}


int __cdecl main(int argc, char *argv[])
{

    int error;
    g_hStdout = GetStdHandle(STD_OUTPUT_HANDLE);
    GetConsoleScreenBufferInfo(g_hStdout, &g_ConsoleScreenBufferInfo);    

    if (argc < 3) {
        PrintUsage(argv[0]);
        return 1;
    }

    WSADATA wsaData;

    error = WSAStartup( MAKEWORD( 2, 2 ), &wsaData);
    if (error) {
        printf("Error in WSAStartup: winsock error=%d\n", WSAGetLastError());
        return 1;
    }

    HMODULE hMswsock2 = LoadLibrary("mswsock.dll");
    if (hMswsock2 == NULL) {
        printf("Error in loading mswsock.dll error=%d\n", WSAGetLastError());
        return 1;
    }

    pGetAcceptExSockaddrs = (GetAcceptExSockaddrsType *)GetProcAddress(hMswsock2, "GetAcceptExSockaddrs");
    pAcceptEx =  (AcceptExType *)GetProcAddress(hMswsock2, "AcceptEx");

    if (pGetAcceptExSockaddrs == NULL || pAcceptEx == NULL ) {
        printf("Error in loading mswsock.dll FUNCTIONS error=%d\n", WSAGetLastError());
        return 1;
    }

    CreateComplitionPort();
    if (!_stricmp(argv[1],"client")) {
        Connect(argc, argv);
        //sleep(2);
    } else if (!_stricmp(argv[1],"server")) {
        Server(argc, argv);
    } else if (!_stricmp(argv[1],"pingpong")) {
        PingPong(argc, argv);
    } else {
        printf("Unknown option %s", argv[1]);
        PrintUsage(argv[0]);
    }
    //printf("calling WSACleanup\n ");
    int ret = WSACleanup ();
    //printf("ret = %d", ret);
    CloseComplitionPort();
    return 0;
}


LONGLONG usleep(LONGLONG us)
{
    LONGLONG j;
    for (j=0; j < us * 325; j++);
    return j;
}
LONGLONG ddd;
int __cdecl main3(int argc, char *argv[])
{
    ddd = usleep(5*1000*1000);
    return 0;
}


int __cdecl main4(int argc, char *argv[])
{

    int error;
    g_hStdout = GetStdHandle(STD_OUTPUT_HANDLE);
    GetConsoleScreenBufferInfo(g_hStdout, &g_ConsoleScreenBufferInfo);    
    struct sockaddr server_addr;

    WSADATA wsaData;

    error = WSAStartup( MAKEWORD( 2, 2 ), &wsaData);
    if (error) {
        printf("Error in WSAStartup: winsock error=%d\n", WSAGetLastError());
        return 1;
    }
    SOCKET s;
    server_addr = addressFromString("11.4.12.63", 5000);

    s = socket (AF_INET_FAMILY, SOCK_DGRAM, 0);
    if (s == -1) {
            printf("Error creating socket\n");
            goto cleanup;
    }

    char * buf = new char [100];

    for (int i=0; i < 100000; i++) {
    
        int ret = sendto(s, buf, 100, 0, &server_addr, sizeof (server_addr));
        if (ret != 100) {
            printf("send failed");
        }
        usleep(30);
    }
cleanup:    
    return 0;

}
