I had created a custom taxonomy "group", and new post type named "topic". there are some terms under custom taxonomy "group".

Use the following code, i can create a new entry from frontend in category. But it doesn't work in my custom taxonomy.

Code:

<?php
/**
 * Display the create new topic link on category page.
 */
function my_new_topic_link( $text = '', $before = '', $after = '') {
    if ( ! is_user_logged_in() )
        return;

    global $wp_query;
    $node_id = $wp_query->get_queried_object_id();

    if ( is_category() ) {
        $url = my_get_page_url_by_slug( 'new', 'node=' . $node_id );
    } else {
        $url = my_get_page_url_by_slug( 'new' );
    }

    if ( '' == $text )
        $text = __( 'Create New Topic' );

    $link = $before . '<a rel="nofollow" class="btn" href="' . $url . '">' . $text . '</a>' . $after;

    echo $link;
}

/**
 * The create new topic form.
 */
function my_new_topic_form() {
    ob_start();

    my_error_messages();
?>
<form id="vp-new-topic" action="" method="post">
    <fieldset>
    <?php if ( empty( $_GET['node'] ) ) { ?>
        <p><?php wp_dropdown_categories( array( 'show_option_all' => __( 'Select Node' ) ) ); ?></p>
    <?php } ?>
        <p><input type="text" name="topic_title" placeholder="<?php _e( 'Topic title here' ); ?>" id="topic_title" class="form-field wider" value="" /></p>
        <p><textarea name="topic_content_filtered" id="topic_content_filtered" class="form-field wider" rows="10" cols="20" ></textarea></p>
        <p>
            <input type="hidden" name="action" value="new_topic" />
            <input type="hidden" name="new_topic_nonce" value="<?php echo wp_create_nonce( 'new-topic-nonce' ); ?>" />
            <input type="submit" name="submit" class="btn" value="<?php _e( 'Create Topic' ); ?>" />
        </p>
    </fieldset>
</form>
<?php
  echo ob_get_clean();
}

/**
 * Do the create new topic form process.
 */
function my_do_create_new_topic() {
    if ( empty( $_POST['action'] ) || ( 'new_topic' != $_POST['action'] ) )
        return;

    if ( isset( $_POST['topic_title'] ) && wp_verify_nonce( $_POST['new_topic_nonce'], 'new-topic-nonce' ) ) {
        if ( !empty( $_GET['node'] ) )
            $node_id = (int) $_GET['node'];
        else
            $node_id = $_POST['cat'];

        $title = $_POST['topic_title'];
        $content_filtered = strip_tags( $_POST['topic_content_filtered'] );
        $user_id = get_current_user_id();

        // Node empty
        if ( empty( $node_id ) || ( 0 == $node_id ) ) {
            my_errors()->add( 'node_empty', __( 'Please select a node' ) );
        }

        // Node id invalid
        if ( !empty( $node_id ) ) {
            $node = get_term_by( 'id', $node_id, 'category');
            if ( !$node ) {
                my_errors()->add( 'node_invalid', 'Please do not cheating me.' );
            }
        }

        // Title empty
        if ( empty( $title ) ) {
            my_errors()->add( 'topic_title_empty', __( 'Please enter the topic title' ) );
        }

        // Content empty
        if ( empty( $content_filtered ) ) {
            my_errors()->add( 'topic_content_empty', __( 'Please write something useful' ) );
        }

        $errors = my_errors()->get_error_messages();
        if( empty( $errors ) ) {
            $data = array(
                        'post_author' => $user_id,
                        'post_category' => (array) $node_id,
                        'post_title' => $title,
                        'post_content_filtered' => $content_filtered,
                        'post_status' => 'publish'
                    );

            $data['post_content'] = $content_filtered;

            $topic = wp_insert_post( $data );

            if ( $topic ) {
                $topic_url = get_permalink( $topic );
                wp_redirect( $topic_url );
                exit;
            }
        }
    }
}
add_action( 'template_redirect', 'my_do_create_new_topic' );

/**
 * The topic edit links.
 */
function my_edit_topic_links() {
    $topic_id = get_the_ID();

    if ( !is_user_logged_in() || !current_user_can( 'edit_post', $topic_id ) )
        return;

    $pub_time = get_the_time( 'U' );
    $time_diff = abs( current_time( 'timestamp' ) - $pub_time );

    // Cannot edit topic 15mins after created
    if ( 900 < $time_diff )
        return;

    $edit_url = add_query_arg( 'edit', 'true', get_permalink() );
    $link = '<span class="topic-edit">';
    $link .= '<a rel="nofollow" href="' . $edit_url . '">' . __( 'edit' ) . '</a>';
    $link .= '</span>';

    echo $link;
}

/**
 * If the current page the edit topic page.
 */
function my_is_edit() {
    if ( is_single() && isset( $_GET['edit'] ) && 'true' == $_GET['edit'] )
        return true;

    return false;
}

/**
 * The edit topic form.
 */
function my_edit_topic_form() {
    $pub_time = get_the_time( 'U' );
    $time_diff = abs( current_time( 'timestamp' ) - $pub_time );
    // Cannot edit topic 15mins after created
    if ( 900 < $time_diff ) {
        echo '<p>' . __( 'You cannot edt this topic 15 minites after created.' ) . '</p>';
        return;
    }

    ob_start();
    $topic_id = get_query_var( 'p' );
    $topic = get_post( $topic_id );

    my_error_messages();
?>
<form id="vp-edit-topic" action="" method="post">
    <fieldset>
        <p><input type="text" name="topic_title" id="topic_title" class="form-field wider" value="<?php echo $topic->post_title ?>" /></p>
        <p><textarea name="topic_content_filtered" id="topic_content_filtered" class="form-field wider" rows="10" cols="20" ><?php echo $topic->post_content_filtered; ?></textarea></p>
        <p>
            <input type="hidden" name="action" value="edit_topic" />
            <input type="hidden" name="edit_topic_nonce" value="<?php echo wp_create_nonce( 'edit-topic-nonce' ); ?>" />
            <input type="submit" name="submit" class="btn" value="<?php _e( 'Submit Changes' ); ?>" />
        </p>
    </fieldset>
</form>
<?php
    echo ob_get_clean();
}

/**
 * Do the edit topic form process.
 */
function my_do_edit_topic() {
    if ( empty( $_POST['action'] ) || ( 'edit_topic' != $_POST['action'] ) )
        return;

    if ( isset( $_POST['topic_title'] ) && wp_verify_nonce( $_POST['edit_topic_nonce'], 'edit-topic-nonce' ) ) {
        $topic_id = get_query_var( 'p' );
        $title = $_POST['topic_title'];
        $content_filtered = strip_tags( $_POST['topic_content_filtered'] );

        // Title empty
        if ( empty( $title ) ) {
            my_errors()->add( 'topic_title_empty', __( 'Please enter the topic title' ) );
        }

        // Content empty
        if ( empty( $content_filtered ) ) {
            my_errors()->add( 'topic_content_empty', __( 'Please write something useful' ) );
        }

        $errors = my_errors()->get_error_messages();
        if( empty( $errors ) ) {
            $data = array(
                        'ID' => $topic_id,
                        'post_title' => $title,
                        'post_content_filtered' => $content_filtered,
                    );

            $data['post_content'] = $content_filtered;

            $topic = wp_update_post( $data );

            if ( $topic ) {
                $topic_url = get_permalink( $topic );
                wp_redirect( $topic_url );
                exit;
            }
        }
    }
}
add_action( 'template_redirect', 'my_do_edit_topic' );