if ( is_feed() ) {
    add_filter( 'post_gallery', 'wpse63980_gallery_shortcode', 99999, 2 );
    function wpse63980_gallery_shortcode( $html, $attr )
    {

         extract( shortcode_atts( array(
              'order'      => 'ASC',
              'orderby'    => 'menu_order ID',
              'id'         => $post->ID,
              'itemtag'    => 'dl',
              'icontag'    => 'dt',
              'captiontag' => 'dd',
              'columns'    => 3,
              'size'       => 'thumbnail',
              'include'    => '',
              'exclude'    => ''
         ), $attr ) );

         // get_posts(), get_attachments(), etc. here

         return $html;
    }
}