Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def resultToProgress(name: String, result: Array[Row], approxCols: Seq[Int]): Double = {
- var retVal = Double.PositiveInfinity
- val N: Int = 3 // smoothing factor
- if(approxCols.length == 1) {
- if (!lossHistory.contains(name)) {
- lossHistory.put(name, ArrayBuffer[Seq[Double]]())
- }
- val poolLossHist: ArrayBuffer[Seq[Double]] =
- lossHistory.getOrElse(name, ArrayBuffer[Seq[Double]]())
- // if we have only one approx col
- if (result.length == 1) {
- // if we only have one row in the results
- val estAndBounds = result(0).toSeq(approxCols(0)).asInstanceOf[GenericRowWithSchema]
- val estimation = estAndBounds(0).asInstanceOf[Double]
- poolLossHist.append(Seq(estimation))
- if(poolLossHist.size > 1) {
- val poolLossDiffHist = (1 until poolLossHist.size).map { i =>
- Math.abs(poolLossHist(i)(0) - poolLossHist(i - 1)(0))
- }
- val max = poolLossDiffHist.max
- val poolLossDiffNormalized = poolLossDiffHist.map( x => x / max)
- val lastN = poolLossDiffNormalized.slice(poolLossDiffNormalized.size -
- Math.min(N, poolLossDiffNormalized.size), poolLossDiffNormalized.size)
- val avg = lastN.sum / lastN.size
- retVal = avg
- }
- } else if (!result.isEmpty) {
- // if not, average all of the groups
- val estimations = result.map { row =>
- row.toSeq(approxCols(0)).asInstanceOf[GenericRowWithSchema](0).asInstanceOf[Double]
- }
- // estimations is a sequence of estimations for each group
- poolLossHist.append(estimations)
- if(poolLossHist.size > 1) {
- // poolLossDiffHist is a sequence of differences for each group
- val poolLossDiffHist = (1 until poolLossHist.size).map { i =>
- poolLossHist(i).zip(poolLossHist(i-1)).map { case (x: Double, y: Double) =>
- Math.abs(x-y)
- }
- }
- // maxs is the max diff for each group
- val maxs = poolLossDiffHist.map( x => x.max )
- // poolLossDiffNormalized is the normalized diff for each group
- val poolLossDiffNormalized = poolLossDiffHist.zip(maxs).map{ case(group, max) =>
- group.map( x => x / max )
- }
- // lastN is the last N historical points, each of which has the normalized diff
- val lastN = poolLossDiffNormalized.slice(poolLossDiffNormalized.size -
- Math.min(N, poolLossDiffNormalized.size), poolLossDiffNormalized.size)
- val avgs = lastN.map { hist =>
- hist.sum / hist.size
- }
- retVal = avgs.sum / avgs.size
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement