Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- SparkConf configuration = new SparkConf().setAppName("logisApp");
- SparkContext sparkContext = new SparkContext(configuration);
- Logger rootLogger = Logger.getRootLogger();
- rootLogger.setLevel(Level.ERROR);
- JavaRDD<LabeledPoint> allData = dataHelper.readTextFromfile(sparkContext, "path/to/all/data");;
- // Split initial RDD into two... [60% training data, 40%
- // testing data].
- JavaRDD<LabeledPoint>[] parts = allData.randomSplit(new double[] { 0.6, 0.4 }, 11L);
- JavaRDD<LabeledPoint> trainingData = parts[0];
- JavaRDD<LabeledPoint> valdidationData = parts[1];
- //
- // Run training algorithm to build the model.
- LogisticRegressionWithLBFGS logisticRegression = new LogisticRegressionWithLBFGS().setNumClasses(2);
- logisticRegression.optimizer().setUpdater(new L1Updater());
- final LogisticRegressionModel model = logisticRegression.run(trainingData.rdd());
- //
- // Compute raw scores on the test set.
- JavaRDD<Tuple2<Object, Object>> predict = valdidationData
- .map(new Function<LabeledPoint, Tuple2<Object, Object>>() {
- public Tuple2<Object, Object> call(LabeledPoint point) {
- Double prediction = model.predict(point.features());
- return new Tuple2<Object, Object>(prediction, point.label());
- }
- });
- //
- // Get evaluation metrics.
- BinaryClassificationMetrics metrics = new BinaryClassificationMetrics(predict.rdd());
- // Precision
- JavaRDD<Tuple2<Object, Object>> precision = metrics.precisionByThreshold().toJavaRDD();
- // F Score
- JavaRDD<Tuple2<Object, Object>> f1Score = metrics.fMeasureByThreshold().toJavaRDD();
- System.out.println(" F1 Score : " + f1Score.collect());
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement