add_filter( 'woocommerce_related_products_args', 'rare_related_products_by_custom_taxonomy');
function rare_related_products_by_custom_taxonomy( $args ) {

    global $post;

    $tags = wp_get_post_terms( $post->ID, "custom_taxonomy" );
    foreach ( $tags as $tag ) {
            $tags_array[] .= $tag->term_id;
    }

  unset( $args['post__in'] );
  $args['tax_query'] = array(
            array(
                'taxonomy' => 'custom_taxonomy',
                'field'    => 'id',
                'terms'    => $tags_array,
            ),
        );
  return $args;

}