<td>
    <input gender="Female" id="5c6434a8590a0654a0002b5a" academicyearid="5b879ec6590a062ba4005fdb" type="checkbox">
    </td>

    <td>
    <input gender="Female" id="5c6434a8590a0654a0002b0a" academicyearid="5b879ec6590a062ba4005fdb" type="checkbox">
    </td>

     <td>
    <input gender="Male" id="5c6434a8590a0654a0002b04" academicyearid="5b879ec6590a062ba4005fdb" type="checkbox">
    </td>

$('.commoncheckbox input:checked').each(function () {

          var gender = $(this).attr('gender');
          if(gender != othergender)// just an idea
          alert("only students of one gender can be selected");

    });