$this->loadLayout()
    ->_initLayoutMessages('checkout/session')
    ->_initLayoutMessages('catalog/session');

 $category = Mage::getModel('catalog/category')->load(11); //here 11 is category id
 $block    = $this->getLayout()->createBlock('catalog/product_list')->setCategory($category)->setTemplate('catalog/product/list.phtml');
 $this->getLayout()->getBlock('content')->append($block);
 $this->renderLayout();