Stopping product saving process in observer
<catalog_product_prepare_save>
        <observers>
            <brands_product_save_observer>
                <type>singleton</type>
                <class>brands/observer</class>
                <method>saveProductBrand</method>
            </brands_product_save_observer>
        </observers>
    </catalog_product_prepare_save>

public function saveProductBrand($observer) {
    $product = $observer->getProduct();
    $categoryIds = $product->getCategoryIds();
    if (isset($categoryIds)) {
        foreach ($categoryIds as $categoryId) {
            $isBrandCategory = Mage::getModel('brands/navigation')->isBrandCategory($categoryId);
            if ($isBrandCategory)
                $brandCategories[] = $categoryId;
        }
        if (isset($brandCategories)) {
            $brandId = Mage::getModel('brands/navigation')->getBrand($brandCategories[0]);
            if ($brandId) {
                $attribute = Mage::getModel('eav/config')->getAttribute('catalog_product', 140);
                foreach ($attribute->getSource()->getAllOptions(true, true) as $option) {
                    $attributeArray[$option['label']] = $option['value'];
                }
                $categoryName = Mage::getModel('catalog/category')->load($brandId)->getName();
                $product->setData('brand', $attributeArray[$categoryName]);
            }
        } else {
            Mage::getSingleton('adminhtml/session')->addError(Mage::helper('catalog')->__('Please add this product to a brand in the "Categories" tab.'));

            HERE SOME CODE TO TELL MAGENTO TO STOP SAVING THE PRODUCT

            return;
        }
    }
}

Mage::throwException(Mage::helper('adminhtml')->__('You totally failed at that.'));

function saveAction()
{
    ...
    $product = $this->_initProductSave();

    try {
        $product->save();
        $productId = $product->getId();
    ...
}

function save()
{
    if( $error_detection )
    {
        return $this;
    }
    else
    {
        return parent::save();
    }
}