add_action('wp', 'check_search_restrict_numbers_only');
function check_search_restrict_numbers_only() {

    global $wp_query;

    if (!$s = get_search_query())
        return false;

    if (!is_numeric($s)) {
        $wp_query->set_404();
        status_header(404);
        get_template_part(404);
        exit();
    }

}