Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #include "stdafx.h"
- #define BUFSIZE 4096
- #define VARNAME TEXT("MAHOUT_CP")
- int _tmain(int argc, _TCHAR* argv[]) {
- DWORD dwLength; LPTSTR pszBuffer;
- pszBuffer = (LPTSTR)malloc(BUFSIZE*sizeof(TCHAR));
- dwLength = GetEnvironmentVariable(VARNAME, pszBuffer, BUFSIZE);
- if (dwLength > 0) { _tprintf(TEXT("%sn"), pszBuffer); return 0; }
- return 1;
- }
- set SCALA_HOME=C:Progra~2scala
- set SPARK_HOME=C:spark
- set HADOOP_HOME=C:hadoop
- set MAHOUT_HOME=C:mahout
- set SPARK_SCALA_VERSION=2.10
- set MASTER=local[2]
- set MAHOUT_LOCAL=true
- set path=%SCALA_HOME%bin;%SPARK_HOME%bin;%PATH%
- cd /D %SPARK_HOME%
- set SPARK_CP=%SPARK_HOME%conf;%SPARK_HOME%libxxx.jar;...other jars...
- set MAHOUT_CP=%MAHOUT_HOME%libxxx.jar;...other jars...;%MAHOUT_HOME%xxx.jar;...other jars...;%SPARK_CP%;%MAHOUT_HOME%libsparkxxx.jar;%MAHOUT_HOME%libhadoopxxx.jar;%MAHOUT_HOME%srcconf;%JAVA_HOME%libtools.jar
- start "master0" "%JAVA_HOME%binjava" -cp "%SPARK_CP%" -Xms1g -Xmx1g org.apache.spark.deploy.master.Master --ip localhost --port 7077 --webui-port 8082 >>out-master0.log 2>>out-master0.err
- start "worker1" "%JAVA_HOME%binjava" -cp "%SPARK_CP%" -Xms1g -Xmx1g org.apache.spark.deploy.worker.Worker spark://localhost:7077 --webui-port 8083 >>out-worker1.log 2>>out-worker1.err
- ...you may add more workers here...
- cd /D %MAHOUT_HOME%
- "%JAVA_HOME%binjava" -Xmx4g -classpath "%MAHOUT_CP%" "org.apache.mahout.sparkbindings.shell.Main"
- https://mahout.apache.org/users/sparkbindings/play-with-shell.html
- "C:Program Files (x86)GoogleChromeApplicationchrome" --disable-web-security http://localhost:4040
- mkdir C:tmphive
- %HADOOP_HOME%binwinutils.exe chmod 777 /tmp/hive
- %SPARK_HOME%binspark-shell
- import sys
- import os
- spark_home = 'C:Apachespark-1.6.1'
- sys.path.insert(0, os.path.join(spark_home, 'python'))
- sys.path.insert(0, os.path.join(spark_home, 'pythonlibpyspark.zip'))
- sys.path.insert(0, os.path.join(spark_home, 'pythonlibpy4j-0.9-src.zip'))
- # Start a spark context:
- sc = pyspark.SparkContext()
- #
- lines = sc.textFile(os.path.join(spark_home, "README.md")
- pythonLines = lines.filter(lambda line: "Python" in line)
- pythonLines.first()
- C:UsersDesktopAsparkbin>spark-shell
- C:UsersDesktopAsparkbin>pyspark
- pip install findspark
- import findspark
- findspark.init()
- from pyspark import SparkContext
- from pyspark import SparkConf
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement