Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- package org.vitalstar
- import org.apache.spark.mllib.optimization.LBFGS;
- import org.apache.spark.mllib.classification.LogisticRegressionModel
- import org.apache.spark.mllib.evaluation.BinaryClassificationMetrics
- import org.apache.spark.mllib.linalg.Vectors
- import org.apache.spark.mllib.optimization.{LBFGS, LogisticGradient, SquaredL2Updater}
- import org.apache.spark.mllib.util.MLUtils
- import org.scalatest.{FunSuite, BeforeAndAfterAll}
- import org.scalatest.junit.JUnitRunner
- import org.junit.Assert._
- import org.junit.runner.RunWith
- import org.apache.spark.{SparkConf, SparkContext}
- @RunWith(classOf[JUnitRunner])
- class justTrying extends FunSuite with BeforeAndAfterAll {
- def main(): Unit ={
- var testString = ""
- testString = "Hello World"
- }
- def lbfgstest(): Double ={
- val conf = new SparkConf()
- conf.setMaster("local")
- conf.setAppName("iLBFGS")
- val sc = new SparkContext(conf)
- val data = MLUtils.loadLibSVMFile(sc, "testdata/sample_libsvm_data.txt")
- val numFeatures = data.take(1)(0).features.size
- // Split data into training (60%) and test (40%).
- val splits = data.randomSplit(Array(0.6, 0.4), seed = 11L)
- // Append 1 into the training data as intercept.
- val training = splits(0).map(x => (x.label, MLUtils.appendBias(x.features))).cache()
- val test = splits(1)
- // Run training algorithm to build the model
- val numCorrections = 10
- val convergenceTol = 1e-4
- val maxNumIterations = 20
- val regParam = 0.1
- val initialWeightsWithIntercept = Vectors.dense(new Array[Double](numFeatures + 1))
- val (weightsWithIntercept, loss) = LBFGS.runLBFGS(
- training,
- new LogisticGradient(),
- new SquaredL2Updater(),
- numCorrections,
- convergenceTol,
- maxNumIterations,
- regParam,
- initialWeightsWithIntercept)
- val model = new LogisticRegressionModel(
- Vectors.dense(weightsWithIntercept.toArray.slice(0, weightsWithIntercept.size - 1)),
- weightsWithIntercept(weightsWithIntercept.size - 1))
- // Clear the default threshold.
- model.clearThreshold()
- // Compute raw scores on the test set.
- val scoreAndLabels = test.map { point =>
- val score = model.predict(point.features)
- (score, point.label)
- }
- // Get evaluation metrics.
- val metrics = new BinaryClassificationMetrics(scoreAndLabels)
- val auROC = metrics.areaUnderROC()
- println("Loss of each step in training process")
- loss.foreach(println)
- println("Area under ROC = " + auROC)
- return auROC
- }
- test("Testing String") {
- var testString = ""
- testString = "Hello World"
- assert(testString == "Hello World")
- }
- test("Testing LBFGS"){
- var au_lbfgs = lbfgstest()
- assert(au_lbfgs == 1.0)
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement