Set<string> appnameSet = new set<string>();
for(test1__c test : trigger.new)
{
   appnameSet.add(test.appName__c);
}

List<test1__c> testList = new List<test1__c >();

for(test1__c  tt: [select appName__c from Test1__c where APPName__c IN :appnameSet])
{
      for(test1__c  tt1: trigger.new)
      {
          if(tt1.appName__c == tt.appName__c )
          {
               tt1.addError('Duplicate app name found') ;
          }
      }

}

List<Test1__c> testListinsert = new List<Test1__c>()
testListinsert.add(new Test1__c(name="One",appName__c = 'APP1'));
testListinsert.add(new Test1__c(name="Two",appName__c = 'APP1')); //Duplicate
testListinsert.add(new Test1__c(name="Three",appName__c = 'APP1'));//Duplicate
insert testListinsert;

Map<String, String> mapDup = Map<String, String>();

//first level filter if existing records to be inserted contains duplicates.
for(test1__c test : trigger.new)
{
   mapDup.containsKey(test.appName__c)
   {
       test.addError('Duplicate app name found') ;
   }
   else
   {
       mapDup.put(test.appName__c, test.appName__c);
   }
}

//now check if records to be inserted is exist in database.
AggregateResult[] lstResult = [ SELECT APPName__c, COUNT(Id),
                                FROM Test1__c
                                WHERE APPName__c IN:mapDup.keySet()
                                GROUP BY APPName__c
                                HAVING COUNT(Id)>0];

if(lstResult.size()>0)
{
    trigger.new[0].addError('Duplicate app name found');

}