from people in xDoc.Descendants("People")
where (int?) Convert.ToInt16(people.Element("active").Value) == 1 &&
    (int?)Convert.ToInt16(people.Element("id").Value) != 130 &&  // <-- to optimize
    (int?)Convert.ToInt16(people.Element("id").Value) != 252 &&  // <-- to optimize
    (int?)Convert.ToInt16(people.Element("id").Value) != 264 &&  // <-- to optimize
    (int?)Convert.ToInt16(people.Element("id").Value) != 302 &&  // <-- to optimize
    people.Descendants("roleIds").Descendants("int")
        .Select(p => (int?)Convert.ToInt16(p.Value)).FirstOrDefault() == 3  // <-- to optimize
select new PeopleClass
{
    ResAnId = (int) Convert.ToInt16(people.Element("id").Value),
    DpInit = (string)people.Element("clientCode").Value,
    ResAnFName = (string)people.Element("firstName").Value,
    ResAnLName = (string)people.Element("lastName").Value,
    ResAnName = String.Format("{0} {1}", (string)people.Element("firstName").Value,
        (string)people.Element("lastName").Value),
    RoleId = people.Descendants("roleIds").Descendants("int")
        .Select(p => (int?) Convert.ToInt16(p.Value)).FirstOrDefault(),  // <-- to optimize
    DivisionId = (int?) Convert.ToInt16(people.Element("divisionId").Value)
}).ToList();

var analystRoleId = 3;
var testAnalystIds = new[] { 130, 252, 264, 302 };

var activeAnalysts =
    from p in xdoc.Descendants("People")
    let id = (int)p.Element("id")
    let isActive = (int)p.Element("active") == 1
    let isAnalyst = p.Descendants("roleIds").Elements().Any(e => (int)e == analystRoleId)
    let firstName = (string)p.Element("firstName")
    let lastName = (string)p.Element("lastName")
    where isActive && isAnalyst && !testAnalystIds.Contains(id)
    select new PeopleClass {
        ResAnId = id,
        DpInit = (string)p.Element("clientCode"),
        ResAnFName = firstName,
        ResAnLName = lastName,
        ResAnName = $"{firstName} {lastName}",
        RoleId = analystRoleId,
        DivisionId = (int?)p.Element("divisionId")
    };