{{-351, -260, -148, -159, 1}.
 {-197, -106, 6, -5, 155}
 {-194, -103, 9, -2, 158}
 {-104, -13, 99, 88, 248}
 {28, 119, 231, 220, 380}}

For[i = 1, i < Length[diffList] + 1, i++,
  setMin1 = {};
  setMin2 = {};
  setMin3 = {};
  setMin4 = {};
  setMin5 = {};
  For[j = 1, j < Length[diffList[[i]]] + 1, j++,
    setMin1 = Append[setMin1, Abs[diffList[[i, j, 1]]]];
    setMin2 = Append[setMin2, Abs[diffList[[i, j, 2]]]];
    setMin3 = Append[setMin3, Abs[diffList[[i, j, 3]]]];
    setMin4 = Append[setMin4, Abs[diffList[[i, j, 4]]]];
    setMin5 = Append[setMin5, Abs[diffList[[i, j, 5]]]];]]

mat = {{-351, -260, -148, -159,   1},
       {-197, -106,    6,   -5, 155},
       {-194, -103,    9,   -2, 158},
       {-104,  -13,   99,   88, 248},
       {  28,  119,  231,  220, 380}};
Map[Min[Abs[#]] &, Transpose[mat]]

findminabs[v_] := Min[Abs[v]];
Map[findminabs, Transpose[mat]]

m =
  {{-351, -260, -148, -159, 1},
   {-197, -106, 6, -5, 155},
   {-194, -103, 9, -2, 158},
   {-104, -13, 99, 88, 248},
   {28, 119, 231, 220, 380}};

Min /@ Abs[Transpose[m]]