<aura:attribute name="LstOfAddedUser" type="List" description="Hold the list of added userUser Data"/>
<aura:attribute name="UserdataList" type="List"/>
<aura:iteration items="{!v.UserdataList}" var="userData" indexVar="i">
    <a onclick="{!c.removeRow}" data-index="{!i}">
        <lightning:icon variant="error" iconName="utility:delete" class="slds-icon slds-icon_small" size="small" alternativeText="icon"/>
        <span class="slds-assistive-text">Delete Icon</span>
    </a>
</aura:iteration>

removeRow : function(cmp, event, helper) {
 var index = event.target.dataset.index;
    var ListOfUsr=cmp.get("v.LstOfAddedUser");
    ListOfUsr.splice(index, 1);
    //some custom logic below
    var action = cmp.get("c.deleteRow");
    var pageNumber = cmp.get("v.PageNumber");
    var pageSize = cmp.get("v.pageSize");
    if(ListOfUsr!=null && ListOfUsr!=''){
        action.setParams({
            "selectedIds":JSON.stringify(ListOfUsr.map(thisItem => thisItem.Id)),
            "pageNumber": pageNumber,
            "pageSize": pageSize,

        });
        action.setCallback(this, function(response){
            var state = response.getState();
            if(state ==="SUCCESS")
            {
                var resultData = response.getReturnValue();
                cmp.set("v.UserdataList", resultData.userdataList);
                cmp.set("v.PageNumber", resultData.pageNumber);
                cmp.set("v.TotalRecords", resultData.totalRecords);
                cmp.set("v.RecordStart", resultData.recordStart);
                cmp.set("v.RecordEnd", resultData.recordEnd);
                cmp.set("v.TotalPages", Math.ceil(resultData.totalRecords / pageSize));
            }
        });
        $A.enqueueAction(action);
    }
}

var index = event.target.dataset.index;

<aura:iteration items="{!v.UserdataList}" var="userData" indexVar="i">
    <a onclick="{!c.removeRow}" id="{!userData.id}">
        <lightning:icon variant="error" iconName="utility:delete" class="slds-icon slds-icon_small" size="small" alternativeText="icon"/>
        <span class="slds-assistive-text">Delete Icon</span>
    </a>
</aura:iteration>

<aura:iteration items="{!v.UserdataList}" var="userData" indexVar="i">
  <lightning:buttonIcon variant="error"
     iconName="utility:delete"
    class="slds-icon slds-icon_small"
    size="small"
    alternativeText="icon"
    value="{!userData.id}"
    onclick="{!c.removeRow}"/>
  <span class="slds-assistive-text">Delete Icon</span>
</aura:iteration>

removeRow : function(cmp, event, helper) {
 var id= event.getSource().get('v.value');
//use this id to find relevant record and update your list.