Brainer

function TShape.Validate: Boolean;

  { .: LinesCross :. }
  function LinesCross(x0, y0, x1, y1, x2, y2, x3, y3: Integer): Boolean;
  var
    D, AB, CD: Single;
  begin
        D := (x1 - x0) * (y3 - y2) - (y1 - y0) * (x3 - x2);
          if (Abs(D) < 0.001) then
    begin
      Result := False;
      exit;
    end;
          AB := ((y0 - y2) * (x3 - x2) -(x0 - x2) * (y3 - y2)) / D;
          if (AB > 0.0) and (AB < 1.0) then
    begin
      CD := ((y0 - y2) * (x1 - x0) - (x0 - x2) * (y1 - y0)) / D;
                  if (CD > 0.0) and (CD < 1.0) then
      begin
        Result := False;
        exit;
      end;
    end;
    Result := True;
  end;

var
  I, J: Integer;
begin
  Result := True;

  for I := 0 to FPoints.Count -4 do
    for J := I + 1 to FPoints.Count -2 do
    begin
      if LinesCross(TShapeVec(FPoints.Items[I]).X,
        TShapeVec(FPoints.Items[I]).Y,
        TShapeVec(FPoints.Items[I + 1]).X,
        TShapeVec(FPoints.Items[I + 1]).Y,
        TShapeVec(FPoints.Items[J]).X,
        TShapeVec(FPoints.Items[J]).Y,
        TShapeVec(FPoints.Items[J + 1]).X,
        TShapeVec(FPoints.Items[J + 1]).Y) then
      begin
        Result := False;
        exit;
      end;
    end;
end;