<models>
    ...
    <catalog>
        <rewrite>
            <layer>Custom_Module_Model_Layer</layer>
        </rewrite>
    </catalog>
    ...
</models>

class Custom_Module_Model_Layer extends Mage_Catalog_Model_Layer
{
    /**
     * Initialize product collection
     *
     * @param Mage_Catalog_Model_Resource_Eav_Mysql4_Product_Collection $collection
     * @return Mage_Catalog_Model_Layer
     */
    public function prepareProductCollection($collection)
    {
        $collection
        ->addAttributeToSelect(Mage::getSingleton('catalog/config')->getProductAttributes())
        ->addMinimalPrice()
        ->addFinalPrice()
        ->addTaxPercents()
        ->addUrlRewrite($this->getCurrentCategory()->getId());

        Mage::getSingleton('catalog/product_status')->addVisibleFilterToCollection($collection);
        Mage::getSingleton('catalog/product_visibility')->addVisibleInCatalogFilterToCollection($collection);

        // this filters products which has are in stock or out of stock and custom_attribute enabled

        Mage::getSingleton('custom_module/custom')->addCustomFilter($collection);

    }
}

class Custom_Module_Model_Custom extends Mage_Core_Model_Abstract
{
    public function addCustomFilter($collection){
        $this->_getResource()->addCustomFilter($collection);
        return $this;
    }
}

class Custom_Module_Model_Resource_Custom extends Mage_Core_Model_Resource_Db_Abstract{
    public function addCustomFilter($collection)
    {
        $websiteId = Mage::app()->getStore($collection->getStoreId())->getWebsiteId();
        $joinCondition = $this->_getReadAdapter()
            ->quoteInto('e.entity_id = stock_status_index.product_id'
                . ' AND stock_status_index.website_id = ?', $websiteId
            );

        $joinCondition .= $this->_getReadAdapter()->quoteInto(
            ' AND stock_status_index.stock_id = ?',
            Mage_CatalogInventory_Model_Stock::DEFAULT_STOCK_ID
        );

        $collection->getSelect()
            ->join(
                array('stock_status_index' => 'cataloginventory_stock_status'),
                $joinCondition,
                array()
            );
            // ->where('stock_status_index.stock_status=?', Mage_CatalogInventory_Model_Stock_Status::STATUS_IN_STOCK);

        $attributeCode = 'custom_attribute';
        $attribute = Mage::getSingleton('eav/config')->getAttribute(Mage_Catalog_Model_Product::ENTITY, $attributeCode);
        // get custom attribute values
        $collection->getSelect()
            ->join(
                array("custom_attribute" => $attribute->getBackendTable()),
                "e.entity_id = custom_attribute.entity_id AND custom_attribute.attribute_id = " . $attribute->getId(),
                array()
            )
            // filter out in stock products and not in stock products with custom_attribute = 1
            ->where('stock_status_index.stock_status=' . Mage_CatalogInventory_Model_Stock_Status::STATUS_IN_STOCK . ' OR (stock_status_index.stock_status=' . Mage_CatalogInventory_Model_Stock_Status::STATUS_OUT_OF_STOCK . ' AND custom_attribute.value = 1)');

        return $this;
    }
}