mpi

#include <sched.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <pthread.h>
#include <sys/time.h>
#include <time.h>
#include <math.h>
#include <mpi.h>
#define _REENTRANT

//mpicxx -o flab4_itog flab4_itog.cpp
//mpiexec -np 2 ./flab4_itog 205 (1029) 2 100

int t;
int N;//размер матрицы
int n_blocks;//число потоков (число блоков)
int block_size;//размер блока
int edging_size;//размер окаймления
int M;
int n; int m; int _time_;
int intBuf[3];

double *A;
double *B;
double *X;
double *b;
double *x;
double *A_block;
double *A_ver;
double *A_gor;
double *A_koef;
double *a_block;
double *a_ver;
double *a_gor;
double *a_koef;

double *A2;
double *B2;
double *X2;
double *x2;

double *A1;
double *B1;
double *X1;

FILE *f;
struct timeval tv1, tv2;
double t_posl,t_par;

//lab.c
int Length;
int Lambda;
int Ct;
int p;
double material;
double k1;
double k2;
double k3;
double dt;
double dx;
int tLayers;
int nodesNum;//число узлов
int threadsNum;//число потоков (число блоков)
double simulationTime;
int nodesInBlock;//размер блока
int edgingSize;//размер окаймления
double maxT = 0;
double minT = 100;


struct arg_fwd_struct //структура для прямого хода Гаусса
{
   int topleft;
   int downright;
   int dim;
   double *matr;
   double *vec;
   bool *result;
};

struct arg_bwd_struct //структура для обратного хода Гаусса
{
   int topleft;
   int downright;
   int dim;
   double *matr;
   double *vec;
   double *res;
};

//lab.c
//get variable index in matrix by node index
int getIndexVar(int node){
    if(node == 0)
        return nodesNum  - edgingSize;
    if(node == nodesNum - 1)
        return nodesNum - 1;
    if(node % (nodesInBlock + 1) == 0)
        return nodesInBlock * threadsNum + node / (nodesInBlock + 1);
    return node - 1 - (int) (node / (nodesInBlock + 1));
}

//get node index by variable index from matrix
int getIndexNode(int var){
    if(var == 0)
        return 1;
    if(var < nodesInBlock * threadsNum){
        return var + 1 + (int) (var / nodesInBlock);
    } else {
        return (nodesInBlock + 1) * (var - nodesInBlock * threadsNum);
    }
}

void initConditions(){
    int i;
    //X =  (double *) calloc(nodesNum, sizeof(double));
    for(i = 0; i < nodesNum; i++)
        X[i] = 0;
}

void boundConditions1(int node, double T){
    A[getIndexVar(node) * nodesNum + getIndexVar(node)] = 1;
    B[getIndexVar(node)] = T;
}

void boundConditions2(){
    double dT = 50;
    //////// left border
    /* -->[########] */
    //A[getIndexVar(0) * nodesNum + getIndexVar(0)] = (double) 1/dx;
    //A[getIndexVar(0) * nodesNum + getIndexVar(1)] = -(double) 1/dx;
    //B[getIndexVar(0)] = dT;

    /* <--[########]*/
    /*A[getIndexVar(0) * nodesNum + getIndexVar(0)] = -(double) 1/dx;
    A[getIndexVar(0) * nodesNum + getIndexVar(1)] = (double) 1/dx;
    B[getIndexVar(0)] = dT;*/

    ////////right border
    /* [########]<-- */
    A[getIndexVar(nodesNum - 1) * nodesNum + getIndexVar(nodesNum - 1)] = (double) 1/dx;
    A[getIndexVar(nodesNum - 1) * nodesNum + getIndexVar(nodesNum - 2)] = -(double) 1/dx;
    B[getIndexVar(nodesNum - 1)] = dT;

    /* [########]--> */
    //A[getIndexVar(nodesNum - 1) * nodesNum + getIndexVar(nodesNum - 1)] = -(double) 1/dx;
    //A[getIndexVar(nodesNum - 1) * nodesNum + getIndexVar(nodesNum - 2)] = (double) 1/dx;
    //B[getIndexVar(nodesNum - 1)] = dT;
}

void checkMaxMin(){
    int i;
    for(i = 0; i < nodesNum; i++){
        if(X[i] > maxT){
            maxT = X[i];
            continue;
        }
        if(X[i] < minT){
            minT = X[i];
            continue;
        }
    }
}

void printToGNU(int Tag){
    int i,j,t;
    int node;

    if(Tag == 0) {
        fprintf(f, "%s\n", "clear");
        fprintf(f, "%s\n", "plot 2 with lines");
        fprintf(f, "%s\n", "set size ratio -1");
        fprintf(f, "%s\n", "set nokey");

        fprintf(f, "%s%d%s\n","set xrange [-1:", nodesNum,"]");
        fprintf(f, "%s%d%s%d%s\n","set yrange [", -(int) nodesNum/4, ":", (int) nodesNum/2,"]");

        fprintf(f, "                                                                                                                 \n");
        fprintf(f, "%s\n", "set palette defined (0 \"#6699FF\", 100 \"#FF0000\")");
        return;
    }

    if(Tag == 1){
        fprintf(f, "%s\n", "plot \"-\" using 2:1:3 with image");
        for(j = 0; j < nodesNum; j++){
            for(i = 0; i < (int) (Length/(4*dx)); i++){
                fprintf(f, "%d\t%d\t%f\n", i, j, X[getIndexVar(j)]);
            }
        }
        fprintf(f, "%s\n", "EOF");
        //fprintf(f, "%s\n", "pause 0.0001");
    }

    if(Tag == 2){
        fseek(f, 100, SEEK_SET);
        fprintf(f, "%s%f%s%f%s\n", "set cbrange [", minT , ":", maxT , "]");
        return;
    }
}

void fillSLAU(){
    int i, j;

    //memset(A, 0, sizeof(A));
    for(i = 0; i < nodesNum; i++)
        for(j = 0; j < nodesNum; j++)
            A[i * nodesNum + j] = 0;


    for(i = 0; i < nodesNum; i++){
        if(i == 0){
            boundConditions1(i, 50);
            continue;
        }
        if(i == nodesNum - 1){
            boundConditions1(i, 100);
            continue;
        }
        /*if(i == (int) (nodesNum / 2)){
            boundConditions1(i, 200);
            continue;
        }*/

        //insert if here to add conditions 1st type

        A[getIndexVar(i) * nodesNum + getIndexVar(i)] = k1;
        A[getIndexVar(i) * nodesNum + getIndexVar(i + 1)] = k2;
        A[getIndexVar(i) * nodesNum + getIndexVar(i - 1)] = k2;
        B[getIndexVar(i)] = k3 * X[getIndexVar(i)];

    }
    //boundConditions2();
}

void prepare(){
    int i;

    printf("\n");
    material = (double) Lambda/(Ct * p);
    printf("material koefficient = %.10f\n", material);

    //A = (double *) calloc(nodesNum*nodesNum, sizeof(double));
    //B = (double *) calloc(nodesNum, sizeof(double));

    nodesInBlock = (nodesNum - 2 - (threadsNum - 1)) / threadsNum;
    printf("nodes to block = %d\n", nodesInBlock);
    edgingSize = nodesNum - nodesInBlock * threadsNum;
    printf("edging size = %d\n", edgingSize);

    dx = (double) Length/(nodesNum - 1);
    tLayers = (int) simulationTime/ dt;

    k1 = ((double) 1/dt + (double) (2*material/(dx*dx)));
    k2 = - (double) material/(dx*dx);
    k3 = (double) 1/dt;

    printf("dx = %f\n", dx);
    printf("number of t calculations = %d\n", tLayers);
    printf("k1 = %f\n", k1);
    printf("k2 = %f\n", k2);
    printf("k3 = %f\n", k3);

    //initConditions();
}
//lab.c


void print_matrix(double *A,double *B,int N) //вывод матриц A и B
{
   for (int i = 0; i < N; i++)
   {
      for (int j = 0; j < N; j++)
      printf ("%.3f\t",A[N*i+j]);
      printf ("\t\t%.3f\n",B[i]);
   }
}

void print_result(double *X,int N) //вывод результата X
{
   for (int i = 0; i < N; i++)
   printf ("x[%d]=%.3f",i,X[i]);
}

void* gauss_forward(void *argument)
{//метод Гаусса - прямой ход для послед. вычислений
   int i, j, k;
   int tl, nonzero;
   double c, aa;
   double *swap_row;
   double swap_b;

   arg_fwd_struct *arg = (arg_fwd_struct *)argument;

   for (tl = arg->topleft; tl < arg->downright; tl++)
   {
      if (!arg->matr[arg->dim*tl+tl]){
         if (tl == arg->downright - 1){
            *(arg->result) = false;
           pthread_exit(0);}

         for (k = tl + 1, nonzero = 0; k < arg->downright; k++)
            if (arg->matr[arg->dim*k+tl]){
               nonzero = k;
               break;}

         if (!nonzero){
            *(arg->result) = false;
            pthread_exit(0);}

         swap_row=(double *)calloc(arg->dim, sizeof(double));
         for(int j=0;j<arg->dim;j++)
         swap_row[j] = arg->matr[arg->dim*tl+j];
         for(int j=0;j<arg->dim;j++)
         arg->matr[arg->dim*tl+j] = arg->matr[arg->dim*nonzero+j];
         for(int j=0;j<arg->dim;j++)
         arg->matr[arg->dim*nonzero+j] = swap_row[j];

         swap_b = arg->vec[tl];
         arg->vec[tl] = arg->vec[nonzero];
         arg->vec[nonzero] = swap_b;
         delete [] swap_row;
      }

      c = arg->matr[arg->dim*tl+tl];
      for (j = tl; j < arg->downright; j++) arg->matr[arg->dim*tl+j] /= c;
      arg->vec[tl] /= c;

      for (i = tl + 1; i < arg->downright; i++)
      {
         c = arg->matr[arg->dim*i+tl];
         for (j = tl; j < arg->downright; j++) arg->matr[arg->dim*i+j] -= c * arg->matr[arg->dim*tl+j];
         arg->vec[i] -= c * arg->vec[tl];
      }
   }
   *(arg->result) = true;
}

void* gauss_backward(void *argument)
{ //Обратный ход метода Гаусса для послед. вычислений
   int i, j;
   double res1;

   arg_bwd_struct *arg = (arg_bwd_struct *)argument;

   for (i = arg->downright - 1; i >= arg->topleft; i--)
   {
      arg->res[i] = arg->vec[i];
      for (j = arg->downright - 1; j > i; j--)
      {
         res1 = arg->res[j];
         arg->res[i] -= res1 * arg->matr[(arg->dim)*i+j];
      }
   }
}

void* gauss_forward_par()
{//метод Гаусса - прямой ход для парал. вычислений

   int i, k;
   int tl, nonzero;
   double c, aa;
   double *swap_row;
   double *swap_row_v;
   double swap_b;

   for (tl = 0; tl < n; tl++)
   {
         if (!a_block[n*tl+tl]){
         if (tl == n - 1){exit(1);}

         for (k = tl + 1, nonzero = 0; k < n; k++)
            if (a_block[n*k+tl]){
               nonzero = k;
               break;}

         if (!nonzero){exit(1);}

         swap_row=(double *)calloc(n, sizeof(double));
     swap_row_v=(double *)calloc(n, sizeof(double));
         for(int j=0;j<n;j++)
         swap_row[j] = a_block[n*tl+j];
         for(int j=0;j<n;j++)
         a_block[n*tl+j] = a_block[n*nonzero+j];
         for(int j=0;j<n;j++)
         a_block[n*nonzero+j] = swap_row[j];

         for(int j=0;j<m;j++)
         swap_row_v[j] = a_ver[m*tl+j];
         for(int j=0;j<m;j++)
         a_ver[m*tl+j] = a_ver[m*nonzero+j];
         for(int j=0;j<m;j++)
         a_ver[m*nonzero+j] = swap_row_v[j];

         swap_b = b[tl];
         b[tl] = b[nonzero];
         b[nonzero] = swap_b;
         delete [] swap_row;
      }

      c = a_block[n*tl+tl];
      for (int j = tl; j < n; j++) a_block[n*tl+j] /= c;
      for (int j = 0; j < m; j++) a_ver[m*tl+j] /= c;
      b[tl] /= c;

      for (i = tl + 1; i < n; i++)
      {
         c = a_block[n*i+tl];
         for (int j = tl; j < n; j++) a_block[n*i+j] -= c * a_block[n*tl+j];
         for (int j=0;j<m;j++) a_ver[m*i+j] -= c * a_ver[m*tl+j];
         b[i] -= c * b[tl];
      }

      for(i=0;i<m;i++)
      {
    a_koef[n*i+tl] = a_gor[n*i+tl];
    aa=a_gor[n*i+tl];
    for (int j = tl; j < n; j++) a_gor[n*i+j] -= aa * a_block[n*tl+j];
      }
  }
}

void* gauss_backward_par()
{//Обратный ход метода Гаусса для парал. вычислений
   int i;
   double res1;

   for (i = n - 1; i >=0; i--)
   {
      x[i] = b[i];
      for (int j=0;j<m;j++)
      {
        res1 = x2[j];
        x[i] -= a_ver[m*i+j] * res1;
      }

      for (int j = n - 1; j > i; j--)
      {
         res1 = x[j];
         x[i] -= res1 * a_block[n*i+j];
      }
   }
}

int main (int argc, char **argv)
{

int myrank;
int total;
MPI_Init (&argc,&argv);
MPI_Comm_size (MPI_COMM_WORLD,&total);
MPI_Comm_rank (MPI_COMM_WORLD,&myrank);

if (!myrank){
    if (argc<4) {printf("Input Error!\n");exit(1);}

//lab.c
        Length = 1;
    dt = 1;
    //material
    p = 10000;//8900;//7800
    Ct = 235;//390;//460
    Lambda = 420;//390;//45
    //nodesNum - threadsNum - 3 % threadsNum == 0
    nodesNum = atoi(argv[1]);//151;//9;//13;
    threadsNum = atoi(argv[2]);//2;//2;//4;
    simulationTime = atoi(argv[3]);//100;
    if((nodesNum - 2 - (threadsNum - 1)) % threadsNum != 0) {
        printf("\nwrong number of nodes and threads!\n");
        return 1;
    }
    prepare();
//lab.c

    block_size = nodesInBlock;
    edging_size = edgingSize;
    n_blocks = threadsNum;
    M=block_size* n_blocks;
    N = (block_size* n_blocks)+edging_size;


    A=(double *)calloc(N*N, sizeof(double));
    B=(double *)calloc(N, sizeof(double));
    X=(double *)calloc(N, sizeof(double));
    initConditions();
    A1=(double *)calloc(N*N, sizeof(double));
    B1=(double *)calloc(N, sizeof(double));
    X1=(double *)calloc(N, sizeof(double));
    A_koef=(double *)calloc(n_blocks*block_size*edging_size, sizeof(double));
    A_block=(double *)calloc(n_blocks*block_size*block_size, sizeof(double));
    A_ver=(double *)calloc(n_blocks*block_size*edging_size, sizeof(double));
    A_gor=(double *)calloc(n_blocks*block_size*edging_size, sizeof(double));
    A2=(double *)calloc(edging_size*edging_size, sizeof(double));
    B2=(double *)calloc(edging_size, sizeof(double));
    X2=(double *)calloc(edging_size, sizeof(double));

    //Заполнение intBuf
    intBuf[0]=block_size;
    intBuf[1]=edging_size;
    intBuf[2]=tLayers;

//lab.c
    f = fopen("gnu", "w");
    printToGNU(0);
//lab.c
}//myrank

    MPI_Bcast((void *)intBuf,3,MPI_INT,0,MPI_COMM_WORLD);
    n=intBuf[0];
    m=intBuf[1];
    _time_=intBuf[2];

for(t = 0; t < _time_; t++){ //главный цикл


 if (!myrank){

        fillSLAU(); //lab.c
        //print_matrix(A,B,N);
        //printf("\n");

    for (int i=0; i<N; i++){//заполнение матриц A1 и B1
        for (int j=0; j<N; j++){A1[i*N+j]=A[i*N+j];}
        B1[i]=B[i];
    }

   //заполнение матрицы A_block
    int k=0; int shift=0; int count=0;
    for (int i=0; i<n_blocks*block_size;i++)
    {
    for (int j=0+shift; j<block_size+shift;j++)
    {A_block[k]=A[N*i+j];k+=1;}
    count+=1;
    if (count==block_size)
    {shift+=block_size;count=0;}
    }
     //printf("\n");
     //print_result(A_block,n_blocks*block_size*block_size);


   //заполнение матрицы A_ver
    k=0;
    for (int i=0; i<N-edging_size; i++){
        for (int j=N-edging_size; j<N; j++)
        {A_ver[k]=A[N*i+j];k+=1;}
    }
     //printf("\n");
     //print_result(A_ver,n_blocks*block_size*edging_size);

   //заполнение матрицы A_gor
    k=0;shift=0;count=0;
    for (int l=0;l<n_blocks;l++)
    {
    for (int i=N-edging_size; i<N;i++)
    {
    for (int j=0+shift; j<block_size+shift;j++)
    {A_gor[k]=A[N*i+j];k+=1;}
    count+=1;
    if (count==edging_size)
    {shift+=block_size;count=0;}
    }
    }
     //printf("\n");
     //print_result(A_gor,n_blocks*block_size*edging_size);


 //Последовательный расчет
    arg_fwd_struct *arg_fwd_posl;
    arg_fwd_posl = new arg_fwd_struct[1];
    bool *status_posl;
    status_posl = new bool[1];

       arg_fwd_posl[0].topleft = 0; // индекс левого верхнего угла текущего блока
       arg_fwd_posl[0].downright = N;// индекс ниж. прав. угла тек. блока +1(индекс лев. верх. угла след.бл.)
       arg_fwd_posl[0].dim = N;// размер всей матрицы
       arg_fwd_posl[0].matr = A1;// указатель на матрицу
       arg_fwd_posl[0].vec = B1;// указатель на вектор правых частей
       arg_fwd_posl[0].result = &status_posl[0];// указатель на переменную, в которой будет содержаться исход прямого прохода (удача или нет)
gettimeofday(&tv1, NULL);
    gauss_forward((void*)&arg_fwd_posl[0]);
    X1[N-1] = B1[N-1];
gettimeofday(&tv2, NULL);
t_posl+=(tv2.tv_sec-tv1.tv_sec)+(tv2.tv_usec-tv1.tv_usec)/1000000.0;
    //print_matrix(A1,B1,N);

    arg_bwd_struct *arg_bwd_posl;
    arg_bwd_posl = new arg_bwd_struct[1];

      arg_bwd_posl[0].topleft = 0;
      arg_bwd_posl[0].downright = N;
      arg_bwd_posl[0].dim = N;
      arg_bwd_posl[0].matr = A1;
      arg_bwd_posl[0].vec = B1;
      arg_bwd_posl[0].res = X1;
gettimeofday(&tv1, NULL);
    gauss_backward((void*)&arg_bwd_posl[0]);
gettimeofday(&tv2, NULL);
t_posl +=(tv2.tv_sec-tv1.tv_sec)+(tv2.tv_usec-tv1.tv_usec)/1000000.0;
    //printf("\n");print_result(X1,N);
   delete [] arg_fwd_posl;
   delete [] arg_bwd_posl;
   delete [] status_posl;
//printf("\ntime_posl = %f\n",t_posl);

} //myrank


//Параллельный расчет

/*MPI_Bcast((void *)intBuf,3,MPI_INT,0,MPI_COMM_WORLD);
n=intBuf[0];
m=intBuf[1];
_time_=intBuf[2];*/

a_block =(double *)calloc(n*n, sizeof(double));
a_ver=(double *)calloc(n*m, sizeof(double));
a_gor=(double *)calloc(m*n, sizeof(double));
a_koef=(double *)calloc(m*n, sizeof(double));
b=(double *)calloc(n, sizeof(double));
x=(double *)calloc(n, sizeof(double));
x2=(double *)calloc(m, sizeof(double));

if (!myrank){gettimeofday(&tv1, NULL);}

MPI_Scatter((void *)A_block,n*n,MPI_DOUBLE,(void *)a_block,n*n,MPI_DOUBLE,0,MPI_COMM_WORLD);
MPI_Scatter((void *)A_ver,n*m,MPI_DOUBLE,(void *)a_ver,n*m,MPI_DOUBLE,0,MPI_COMM_WORLD);
MPI_Scatter((void *)A_gor,m*n,MPI_DOUBLE,(void *)a_gor,m*n,MPI_DOUBLE,0,MPI_COMM_WORLD);
MPI_Scatter((void *)B,n,MPI_DOUBLE,(void *)b,n,MPI_DOUBLE,0,MPI_COMM_WORLD);

gauss_forward_par();

MPI_Gather((void *)a_block,n*n,MPI_DOUBLE,(void *)A_block,n*n,MPI_DOUBLE,0,MPI_COMM_WORLD);
MPI_Gather((void *)a_ver,n*m,MPI_DOUBLE,(void *)A_ver,n*m,MPI_DOUBLE,0,MPI_COMM_WORLD);
MPI_Gather((void *)a_gor,m*n,MPI_DOUBLE,(void *)A_gor,m*n,MPI_DOUBLE,0,MPI_COMM_WORLD);
MPI_Gather((void *)b,n,MPI_DOUBLE,(void *)B,n,MPI_DOUBLE,0,MPI_COMM_WORLD);
MPI_Gather((void *)a_koef,m*n,MPI_DOUBLE,(void *)A_koef,m*n,MPI_DOUBLE,0,MPI_COMM_WORLD);

if (!myrank){gettimeofday(&tv2, NULL);
t_par +=(tv2.tv_sec-tv1.tv_sec)+(tv2.tv_usec-tv1.tv_usec)/1000000.0;
/*printf("\n");print_result(A_block,n_blocks*block_size*block_size);

printf("\n");print_result(A_ver,n_blocks*block_size*edging_size);

printf("\n");print_result(A_gor,n_blocks*block_size*edging_size);

printf("\n");print_result(A_koef,n_blocks*block_size*edging_size);

printf("\n");print_result(B,N);*/
}//myrank

//Обработка Аnn
if (!myrank){ gettimeofday(&tv1, NULL);

   //Заполняем матрицу A2 и B2
   for (int i = N-edging_size; i < N; i++){
    for (int k=0;k<edging_size;k++) {
        A2[edging_size*(i-N+edging_size)+k]=A[N*i+(M+k)];
        B2[i-N+edging_size]=B[i];
    }
    }

    //Преобразуем матрицу B2
    for (int i = 0; i < edging_size; i++){
    int shift=0; int count=0;
    for (int j=0;j<M;j++)
    {
        B2[i] -= A_koef[block_size*i+j+shift] * B[j];
        count+=1;
        if (count==block_size)
        {count=0; shift+=block_size*(edging_size-1);}
    }
    }

    //Преобразуем матрицу A2
    for (int i = 0; i < edging_size; i++){
    for (int k=0;k<edging_size;k++){
        int shift=0; int count=0;
        for (int j=0;j<M;j++)
        {
            A2[edging_size*i+k] -= A_koef[block_size*i+j+shift] * A_ver[edging_size*j+k];
            count+=1;
            if (count==block_size)
            {count=0; shift+=block_size*(edging_size-1);}
        }
    }
    }

gettimeofday(&tv2, NULL);
t_par +=(tv2.tv_sec-tv1.tv_sec)+(tv2.tv_usec-tv1.tv_usec)/1000000.0;

//Прямой ход для Ann
    arg_fwd_struct *arg_fwd;
    arg_fwd = new arg_fwd_struct[1];
    bool *status;
    status = new bool[1];

       arg_fwd[0].topleft =0;
       arg_fwd[0].downright =edging_size;
       arg_fwd[0].dim = edging_size;
       arg_fwd[0].matr = A2;
       arg_fwd[0].vec = B2;
       arg_fwd[0].result = &status[0];
gettimeofday(&tv1, NULL);
    gauss_forward((void*)&arg_fwd[0]);
    X2[edging_size-1] = B2[edging_size-1];
gettimeofday(&tv2, NULL);
t_par +=(tv2.tv_sec-tv1.tv_sec)+(tv2.tv_usec-tv1.tv_usec)/1000000.0;

//Обратный ход для Ann
    arg_bwd_struct *arg_bwd;
    arg_bwd = new arg_bwd_struct[1];

      arg_bwd[0].topleft = 0;
      arg_bwd[0].downright = edging_size;
      arg_bwd[0].dim = edging_size;
      arg_bwd[0].matr = A2;
      arg_bwd[0].vec = B2;
      arg_bwd[0].res = X2;
gettimeofday(&tv1, NULL);
    gauss_backward((void*)&arg_bwd[0]);
    for (int i=N-1,j=edging_size-1;j>=0;j--){X[i]=X2[j];i--;}
gettimeofday(&tv2, NULL);
t_par +=(tv2.tv_sec-tv1.tv_sec)+(tv2.tv_usec-tv1.tv_usec)/1000000.0;
   delete [] arg_fwd;
   delete [] arg_bwd;
   delete [] status;
} //myrank

if (!myrank){gettimeofday(&tv1, NULL);
memcpy(x2,X2,sizeof(double)*m);}

MPI_Bcast((void *)x2,m,MPI_DOUBLE,0,MPI_COMM_WORLD);
gauss_backward_par();
MPI_Gather((void *)x,n,MPI_DOUBLE,(void *)X,n,MPI_DOUBLE,0,MPI_COMM_WORLD);

if (!myrank){gettimeofday(&tv2, NULL);
t_par +=(tv2.tv_sec-tv1.tv_sec)+(tv2.tv_usec-tv1.tv_usec)/1000000.0;
//printf("\n");
//print_result(X,N);

//lab.c
        checkMaxMin();
        if(maxT > 10000){
            printf("errr\n");
            exit(1);//return;
        }

        //printNodesFromMatrix(OldResult);
        if(t % (int)( _time_*0.01) == 0)
            printToGNU(1);
        //getchar();
//lab.c
}//myrank
}// all end

if (!myrank){
//lab.c
    printToGNU(2);
    printf("max T = %f\n", maxT);
    printf("min T = %f\n", minT);
//lab.c
printf("\ntime_posl = %f\n",t_posl);
printf("\ntime_par = %f\n",t_par);
double t= t_posl/t_par;
printf("speed = %f\n",t);
}//myrank

if(!myrank){
   delete [] X;
   delete [] B;
   delete [] A;
   delete [] X1;
   delete [] B1;
   delete [] A1;
   delete [] A_koef;
   delete [] A_block;
   delete [] A_ver;
   delete [] A_gor;
   delete [] a_koef;
   delete [] a_block;
   delete [] a_ver;
   delete [] a_gor;
   delete [] b;
   delete [] x;
   delete [] A2;
   delete [] B2;
   delete [] X2;
   printf("\nENDL\n");
 }
    MPI_Finalize();
    exit(0);
}