<td>
  <form class="insCat" action="#" name="insertCat">
    <input name="categorienaam" type="text">
    <input class="insert" type="submit" value="nieuwe categorie">
  </form>
</td>

jQuery(".insert").click(function(){

  str = jQuery("form.insCat").serialize();
  console.log(str);

  jQuery.ajax({
    type: "POST",
    data: str + "&cmd=insert",
    url: "ajax_handler.php",
    success: function(msg){}
  });

  jQuery("div.result").empty();

});

jQuery(".insert").click(function(e){
            e.preventDefault();

            ...your code...

            return false;
    });

jQuery("form.insCat").bind("submit", yourFunction);

var yourFunction = function(event){
  // do whatever you like then...
  event.preventDefault();
};