<form>
   <ul>
     <li>
      <ul>
       <li class="size-value"></li>
       <li class="size-value"></li>
       <li class="size-value"></li>
       <li class="size-value"></li>
     </ul>
    </li>
  </ul>
</form>
<div class="mt10">
   <input type="submit" class="modalAddToBagButton">
</div>

}).on('click', '.modalAddToBagButton', function(e) {
            e.preventDefault();
            // Add if/else statement here. if list items have a class selected, form can be submitted. if no list items have class selected, then display this error message.
            $(this).closest("#dialog-addToBag").find('form').submit();
        });

.on('click', '.modalAddToBagButton', function(e) {
            e.preventDefault();
            var x = document.getElementsByClassName("es-value");
            var i = x.length;
            var selected = false;
            while(i--) {
                if (x[i].hasAttribute("selected")) {
                    selected = true;
                }
            }
            if(selected == false) {
                //Displays error
                $("#errormessage").show();
            } else {

                $(this).closest("#dialog-addToBag").find('form').submit();
            }
        });

<div id="errorDiv"></div>

}).on('click', '.modalAddToBagButton', function(e) {
        e.preventDefault();

       var x = document.getElementsByClassName("size-value");
       var i = x.length;

       var selected = false;

       while(i--)
       {
           if (x[i].hasAttribute("selected"))
           {
              selected = true;
           }
       }

       if(selected == false)
       {
           //Displays error
           document.getElementById("errorDiv").innerHTML = "Please select a size.";

       } else {

          //Submit form
          $(this).closest("#dialog-addToBag").find('form').submit();
       }
});

<form id="clothes-sizes">
    <ul>
        <li>
            <ul>
                <li class="size-value"></li>
                <li class="size-value"></li>
                <li class="size-value"></li>
                <li class="size-value"></li>
            </ul>
        </li>
    </ul>

    <div class="mt10">
        <input type="button" class="modalAddToBagButton">
    </div>

</form>

var form = $("#clothes-sizes");
var btn = form.find(".modalAddToBagButton");

btn
.unbind("click")
.bind("click", function ()
{
    var selectedSizes = form.find("ul li.selected");

    if (!selectedSizes.length)
    {
        alert("Please select a size.");

        return false;
    }

    form.submit();
});