$form['captcha'] = array(
  '#type' => 'captcha',
  '#captcha_type' => 'recaptcha/reCAPTCHA',
  '#required' => TRUE,
);

$form['actions']['submit'] = [
  '#name' =>'submit',
  '#type' => 'submit',
  '#value' => $this->t('Submit'),
  '#ajax' => [
    'callback' => '::ajaxSubmitForm',
    'event' => 'click',
  ],
];

public function ajaxSubmitForm(array &$form, FormStateInterface $form_state) {

    $response = new AjaxResponse();

    if ($form_state->hasAnyErrors()) {
      $response->addCommand(new ReplaceCommand('#form-modal-form', $form));
    }

    return $response;
}