Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- val modelFile = DistributedCacheFile(sumacArgs.model.getAbsolutePath)
- val featureFile = DistributedCacheFile(sumacArgs.features.getAbsolutePath)
- val trainFile = DistributedCacheFile(sumacArgs.trainFeatures.getAbsolutePath)
- @transient
- private lazy val ethnicityModel =
- EthnicityFS.loadModel(llModelFile = new File(modelFile.path), featureFile = new File(featureFile.path))
- @transient
- private lazy val validFeatures: Set[Int] = {
- Source.fromFile(trainFile.path).getLines.map { line =>
- line.toInt
- }.toSet
- }
- /**
- hadoop jar transform/target/scala-2.10/transform-assembly-0.1-SNAPSHOT.jar com.twitter.scalding.Tool -Dmapred.map.child.java.opts="-Xmx4500m -Dfile.encoding=UTF-8” -Dmapred.reduce.child.java.opts="-Xmx5500m -Dfile.encoding=UTF-8” -Dmapred.map.tasks=24 -Dmapred.reduce.tasks=8 com.qf.util.fbcrawlexplorer.EthFeatureExplorer --env dev --hdfs --inputFile /mapr/mapr-prod/data/rawJoins/tw/20141213_1418494555086/ --model /mapr/mapr-dev/data/exports3/models/eth2/ethnicityModel.ser.gz --features /mapr/mapr-dev/data/exports3/models/eth2/ethnicityFeatures.ser.gz --sample 0.2 --trainFeatures /mapr/mapr-dev/data/pierre/validFeatures.txt
- */
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement