<apex:repeat value="{!newAttachments}" var="newAtt">
                    <apex:outputLabel value="Upload photo"/>
                    <apex:inputFile id="inputfile" value="{!newAtt.body}" filename="{!newAtt.name}"/><br/>
                </apex:repeat><br/>
                <apex:commandButton value="Add More" action="{!addMore}"  onclick="checkFileSize();"/>
 <script>
    function checkFileSize(){
    if ($("[id$=inputfile]")[0].files.length > 0) {
    //validate file attachment is not > 500k
    var uploadFileSize = $("[id$=inputfile]")[0].files[0].size;
    if (uploadFileSize > 1024 * 1024 * .5) {
        alert('File uploads must be less than 500k in size. Your file is: '+(Math.round(uploadFileSize/1024)) + 'k');
    }
    else {

    }
}
else {
    // no file attached, if not required, call save method
}
        }
    </script>

function checkFileSize(event){
  if ($("[id$=inputfile]")[0].files.length > 0) {
    //validate file attachment is not > 500k
    var uploadFileSize = $("[id$=inputfile]")[0].files[0].size;
    if (uploadFileSize > 2**19) {
      alert('File uploads must be less than 500k in size. Your file is: '+(uploadFileSize>>10) + 'k');
      event.preventDefault();
    }
    // ... rest of code

onclick="checkFileSize(event);"