y = SpeechSynthesize["Yanny"]
l = SpeechSynthesize["Laurel"]

yad = AudioData[y][[1]];
lad = AudioData[l][[1]];

lengthDiff = Length[lad] - Length[yad]
ad1 = Join[Table[0., {lengthDiff}], yad];
yad2 = Join[yad, Table[0., {lengthDiff}]];

len = Length[yad1]
yadl = Table[
   RandomChoice[{yad1, yad2}] +
    RandomVariate[NormalDistribution[0, 0.01], len], {100}];
ladl = Table[
   lad + RandomVariate[NormalDistribution[0, 0.01], len], {100}];

enc = NetInitialize@
  NetChain[{len/8, Tanh, len/16, Tanh, len/64, Ramp, 2},
   "Input" -> len]

td = Join[# -> {1., 0.} & /@ yadl, # -> {0., 1.} & /@ ladl,
   Table[RandomVariate[NormalDistribution[0, RandomReal[{.0, .2}]],
      len] -> {0., 0.}, {100}]];

enc = NetTrain[enc, td]