FILES=(path/to/*.txt)

shopt -s extglob
files=(path/to/!(manifest).txt)

declare -a files=()
for file in /path/to/files/*
do
  ! [[ -e "$file" ]] || [[ "$file" = */manifest.txt ]] || files+=("$file")
done

FILES=($(ls /path/to/*.txt | grep -wv '^manifest.txt$'))

$ touch f{1..6}.txt manifest.txt
$ ls *.txt
f1.txt      f3.txt      f5.txt      manifest.txt
f2.txt      f4.txt      f6.txt

declare -A exclude
for f in f1.txt f5.txt manifest.txt; do
    exclude[$f]=1
done

files=()
for fn in *.txt; do
    [[ ${exclude[$fn]} ]] && continue
    files+=("$fn")
done

$ echo "${files[@]}"
f2.txt f3.txt f4.txt f6.txt