<aui:button name="YES" value="YES" onClick="javascript:yes();"/>
<aui:script>
    function yes(){
        alert('pop');
        Liferay.Util.getOpener().<portlet:namespace />closePopup('<portlet:namespace />shahbaj');
    }
</aui:script>

<input type="button" onclick="hidePopup();" value="Cancel" />

function hidePopup(){
    AUI().ready('aui-dialog', function(A){
        A.DialogManager.hideAll();
    });
}

AUI().ready(function(A) {
        AUI().use('aui-dialog', 'aui-io', function(A) {
                var url = 'http://localhost/url';
    Liferay.Util.openWindow(
            {
                dialog: {
                    cache: false,
                    width:800,
                    modal: true
                },
                   id:'shahbaj',
                uri: url
            }
        );
 Liferay.provide(
        window,
       'closePopup',
        function(popupIdToClose) {
            var dialog = Liferay.Util.getWindow(popupIdToClose);
            dialog.destroy(); // You can try toggle/hide whatever You want
        },
        ['aui-base','aui-dialog','aui-dialog-iframe']
    );
        });
});

<aui:button cssClass="close-panel" type="cancel" value="close" />