/*
 * OSU One Sided MPI_Put Bandwidth test v2.2
 */

/*
 * Copyright (C) 2003-2006 the Network-Based Computing Laboratory
 * (NBCL), The Ohio State University.
 *
 */

#define GPU 0

#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <assert.h>
#include "mpi.h"
#include "cuda.h"
#include "cudatest.h"

#define MYBUFSIZE (150000000)   /* ~= 100M Bytes */
#define MAX_REQ_NUM 100

/* Note we have a upper limit for buffer size, so be extremely careful
 * if you want to change the loop size or warm up size */
#define MAX_SIZE (1<<22)
#define WINDOW_SIZE (32)

char        s_buf1[MAX_SIZE + 4096];
char        r_buf1[MYBUFSIZE + 4096];
MPI_Request request[MAX_REQ_NUM];
CUdeviceptr dptr_send, dptr_recv, gpu_send_buf, gpu_recv_buf;
static int checkbuf = 0;

static void usage(void) {
    printf("osu_put_bw_cuda [-c]\n");
}

int main (int argc, char *argv[])
{
    int         myid, numprocs, i, j, k;
    int         size, page_size;
    char       *host_send_buf, *host_recv_buf;
    double      t_start = 0.0, t_end = 0.0, t = 0.0;
    int         destrank;
    int c;

    MPI_Group   comm_group, group;
    MPI_Win     win;
    int         color;
    MPI_Comm    newcomm;

    MPI_Init (&argc, &argv);
    MPI_Comm_size (MPI_COMM_WORLD, &numprocs);
    MPI_Comm_rank (MPI_COMM_WORLD, &myid);

    printf("Sleeping for 15  pid=%d\n", getpid());
    sleep(15);

    page_size = getpagesize ();
    host_send_buf =
        (char *) (((unsigned long) s_buf1 + (page_size - 1)) / page_size *
                  page_size);
    host_recv_buf =
        (char *) (((unsigned long) r_buf1 + (page_size - 1)) / page_size *
                  page_size);
    assert ((host_send_buf != NULL) && (host_recv_buf != NULL));
    assert (MAX_SIZE * WINDOW_SIZE < MYBUFSIZE);

    /* Load up the send buffer in case we plan on checking the results.
       Just store the index in each location. */
    for (i = 0; i < MAX_SIZE >> 2; i++) {
        ((int *)host_send_buf)[i] = i;
    }
    for (i = 0; i < MYBUFSIZE >> 2; i++) {
        ((int *)host_recv_buf)[i] = -1;
    }

    if (myid == 0) {
        fprintf (stdout,
                 "# OSU MPI2 MPI_Put Bandwidth Test (Version 2.2)\n");
        fprintf (stdout, "# Size\t\tBandwidth (MillionBytes/s) \n");
        fflush(stdout);
    }

    if ((myid == 0) || (myid == 1)) {
        color = 0;
    } else {
        color = 1;
    }
    MPI_Comm_split(MPI_COMM_WORLD, color, 0, &newcomm);
    MPI_Comm_group (newcomm, &comm_group);

    /* Only ranks 0 and 1 play */
    if ((myid == 0) || (myid == 1)) {
        size = 1;

	/* Window creation and test */
	if (myid == 0) {
	    MPI_Win_create (host_recv_buf, size, 1, MPI_INFO_NULL,
			    newcomm, &win);
	    destrank = 1;
	    MPI_Group_incl (comm_group, 1, &destrank, &group);
	    MPI_Win_start (group, 0, win);
	    MPI_Put (host_send_buf, size, MPI_CHAR, 1, size, size,
		     MPI_CHAR, win);
	    MPI_Win_complete (win);
	} else {
	    MPI_Win_create (host_recv_buf, size * WINDOW_SIZE, 1, MPI_INFO_NULL,
			    newcomm, &win);
	    destrank = 0;
	    MPI_Group_incl (comm_group, 1, &destrank, &group);
	    MPI_Win_post (group, 0, win);
	    MPI_Win_wait (win);
	}

	MPI_Barrier (newcomm);
	MPI_Win_free (&win);
    }
    MPI_Barrier (MPI_COMM_WORLD);
    MPI_Finalize ();
    if (0 == myid) {
        printf("MPITEST_results: PASSED\n");
    }
    return 0;
}