API tools faq
paste
Guest User

Samza Install Quick Guide

a guest
Mar 31st, 2015
122
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 12.45 KB | None | 0 0
raw download clone embed print report
  1. Install Samza
  2.  
  3. apt-get install openjdk-7-jdk openjdk-7-jre git maven curl
  4.  
  5. cd /opt
  6. git clone http://git-wip-us.apache.org/repos/asf/samza.git
  7. cd samza
  8. ./gradlew clean build
  9.  
  10. Install YARN
  11.  
  12. wget http://apache.rediris.es/hadoop/common/current/hadoop-2.6.0.tar.gz
  13. tar -xvf hadoop-2.6.0.tar.gz
  14.  
  15. export HADOOP_YARN_HOME=$(pwd)
  16. mkdir conf
  17. export HADOOP_CONF_DIR=$HADOOP_YARN_HOME/conf
  18.  
  19. cd hadoop-2.6.0
  20.  
  21. cp ./etc/hadoop/yarn-site.xml conf
  22. vi conf/yarn-site.xml
  23. <configuration>
  24. <property>
  25. <name>yarn.resourcemanager.hostname</name>
  26. <value>samza01</value>
  27. </property>
  28. </configuration>
  29.  
  30. curl http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/capacity-scheduler.xml?view=co > conf/capacity-scheduler.xml
  31.  
  32. cd /tmp
  33.  
  34. curl http://www.scala-lang.org/files/archive/scala-2.10.4.tgz > scala-2.10.4.tgz
  35. tar -xvf scala-2.10.4.tgz
  36. cp /tmp/scala-2.10.4/lib/scala-compiler.jar $HADOOP_YARN_HOME/share/hadoop/hdfs/lib
  37. cp /tmp/scala-2.10.4/lib/scala-library.jar $HADOOP_YARN_HOME/share/hadoop/hdfs/lib
  38. curl -L http://search.maven.org/remotecontent?filepath=org/clapper/grizzled-slf4j_2.10/1.0.1/grizzled-slf4j_2.10-1.0.1.jar > $HADOOP_YARN_HOME/share/hadoop/hdfs/lib/grizzled-slf4j_2.10-1.0.1.jar
  39. curl -L http://search.maven.org/remotecontent?filepath=org/apache/samza/samza-yarn_2.10/0.8.0/samza-yarn_2.10-0.8.0.jar > $HADOOP_YARN_HOME/share/hadoop/hdfs/lib/samza-yarn_2.10-0.8.0.jar
  40. curl -L http://search.maven.org/remotecontent?filepath=org/apache/samza/samza-core_2.10/0.8.0/samza-core_2.10-0.8.0.jar > $HADOOP_YARN_HOME/share/hadoop/hdfs/lib/samza-core_2.10-0.8.0.jar
  41.  
  42. vi $HADOOP_YARN_HOME/conf/core-site.xml
  43. <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
  44. <configuration>
  45. <property>
  46. <name>fs.http.impl</name>
  47. <value>org.apache.samza.util.hadoop.HttpFileSystem</value>
  48. </property>
  49. </configuration>
  50.  
  51. Define samza slave nodes
  52. scp -r . <slaveIP>:/opt/hadoop-2.6.0
  53. echo <slaveIP> >> conf/slaves
  54. sbin/start-yarn.sh
  55.  
  56. Create job
  57.  
  58. This was generated in Eclipse with Maven plugin.
  59.  
  60. Code
  61.  
  62. package samzafroga;
  63.  
  64. import java.util.Map;
  65. import org.apache.samza.config.Config;
  66. import org.apache.samza.system.IncomingMessageEnvelope;
  67. import org.apache.samza.system.OutgoingMessageEnvelope;
  68. import org.apache.samza.system.SystemStream;
  69. import org.apache.samza.task.MessageCollector;
  70. import org.apache.samza.task.StreamTask;
  71. import org.apache.samza.task.TaskContext;
  72. import org.apache.samza.task.TaskCoordinator;
  73.  
  74. public class job1 implements StreamTask {
  75. private final SystemStream OUTPUT_STREAM = new SystemStream("kafka", "samzaout");
  76.  
  77. public void process(IncomingMessageEnvelope envelope,
  78. MessageCollector collector,
  79. TaskCoordinator coordinator)
  80. {
  81. String msg = (String)envelope.getMessage();
  82. String outmsg = "-" + msg + "-SMZ";
  83. collector.send(new OutgoingMessageEnvelope(OUTPUT_STREAM, outmsg));
  84. }
  85. }
  86.  
  87.  
  88. pom.xml
  89.  
  90. <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  91. <modelVersion>4.0.0</modelVersion>
  92. <groupId>com.acio.samza</groupId>
  93. <artifactId>samzafroga</artifactId>
  94. <version>0.0.1</version>
  95. <name>samzafroga</name>
  96. <dependencies>
  97. <dependency>
  98. <groupId>org.apache.samza</groupId>
  99. <artifactId>samza-api</artifactId>
  100. </dependency>
  101. <dependency>
  102. <groupId>org.apache.samza</groupId>
  103. <artifactId>samza-core_2.10</artifactId>
  104. </dependency>
  105. <dependency>
  106. <groupId>org.apache.samza</groupId>
  107. <artifactId>samza-log4j</artifactId>
  108. </dependency>
  109. <dependency>
  110. <groupId>org.apache.samza</groupId>
  111. <artifactId>samza-shell</artifactId>
  112. </dependency>
  113. <dependency>
  114. <groupId>org.apache.samza</groupId>
  115. <artifactId>samza-yarn_2.10</artifactId>
  116. <exclusions>
  117. <exclusion>
  118. <artifactId>jdk.tools</artifactId>
  119. <groupId>jdk.tools</groupId>
  120. </exclusion>
  121. </exclusions>
  122. </dependency>
  123. <dependency>
  124. <groupId>org.apache.samza</groupId>
  125. <artifactId>samza-kv_2.10</artifactId>
  126. </dependency>
  127. <dependency>
  128. <groupId>org.apache.samza</groupId>
  129. <artifactId>samza-kv-rocksdb_2.10</artifactId>
  130. </dependency>
  131. <dependency>
  132. <groupId>org.apache.samza</groupId>
  133. <artifactId>samza-kafka_2.10</artifactId>
  134. </dependency>
  135. <dependency>
  136. <groupId>org.apache.kafka</groupId>
  137. <artifactId>kafka_2.10</artifactId>
  138. </dependency>
  139. <dependency>
  140. <groupId>org.schwering</groupId>
  141. <artifactId>irclib</artifactId>
  142. </dependency>
  143. <dependency>
  144. <groupId>org.slf4j</groupId>
  145. <artifactId>slf4j-api</artifactId>
  146. </dependency>
  147. <dependency>
  148. <groupId>org.slf4j</groupId>
  149. <artifactId>slf4j-log4j12</artifactId>
  150. </dependency>
  151. <dependency>
  152. <groupId>org.codehaus.jackson</groupId>
  153. <artifactId>jackson-jaxrs</artifactId>
  154. </dependency>
  155. <dependency>
  156. <groupId>org.apache.hadoop</groupId>
  157. <artifactId>hadoop-hdfs</artifactId>
  158. </dependency>
  159. <dependency>
  160. <groupId>org.eclipse.jetty</groupId>
  161. <artifactId>jetty-server</artifactId>
  162. <version>${jettyVersion}</version>
  163. </dependency>
  164. <dependency>
  165. <groupId>org.eclipse.jetty</groupId>
  166. <artifactId>jetty-webapp</artifactId>
  167. <version>${jettyVersion}</version>
  168. </dependency>
  169. </dependencies>
  170. <properties>
  171. <!-- maven specific properties -->
  172. <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  173. <samza.version>0.8.0</samza.version>
  174. <jettyVersion>7.6.16.v20140903</jettyVersion>
  175. </properties>
  176. <repositories>
  177. <repository>
  178. <id>apache-releases</id>
  179. <url>https://repository.apache.org/content/groups/public</url>
  180. </repository>
  181. <repository>
  182. <id>scala-tools.org</id>
  183. <name>Scala-tools Maven2 Repository</name>
  184. <url>https://oss.sonatype.org/content/groups/scala-tools</url>
  185. </repository>
  186. </repositories>
  187.  
  188. <pluginRepositories>
  189. <pluginRepository>
  190. <id>scala-tools.org</id>
  191. <name>Scala-tools Maven2 Repository</name>
  192. <url>http://scala-tools.org/repo-releases</url>
  193. </pluginRepository>
  194. </pluginRepositories>
  195. <build>
  196. <pluginManagement>
  197. <plugins>
  198. <plugin>
  199. <groupId>org.apache.rat</groupId>
  200. <artifactId>apache-rat-plugin</artifactId>
  201. <version>0.9</version>
  202. <configuration>
  203. <excludes>
  204. <exclude>*.patch</exclude>
  205. <exclude>**/target/**</exclude>
  206. <exclude>*.json</exclude>
  207. <exclude>.vagrant/**</exclude>
  208. <exclude>.git/**</exclude>
  209. <exclude>*.md</exclude>
  210. <exclude>docs/**</exclude>
  211. <exclude>config/**</exclude>
  212. <exclude>bin/**</exclude>
  213. <exclude>.gitignore</exclude>
  214. <exclude>**/.cache/**</exclude>
  215. <exclude>deploy/**</exclude>
  216. <exclude>**/.project</exclude>
  217. </excludes>
  218. </configuration>
  219. </plugin>
  220. </plugins>
  221. </pluginManagement>
  222. <plugins>
  223.  
  224. <!-- plugin to build the tar.gz file filled with examples -->
  225. <plugin>
  226. <artifactId>maven-assembly-plugin</artifactId>
  227. <version>2.3</version>
  228. <configuration>
  229. <descriptors>
  230. <descriptor>src/assembly/bin.xml</descriptor>
  231. </descriptors>
  232. <descriptorRefs>
  233. <descriptorRef>jar-with-dependencies</descriptorRef>
  234. </descriptorRefs>
  235. </configuration>
  236. <executions>
  237. <execution>
  238. <id>make-assembly</id>
  239. <phase>package</phase>
  240. <goals>
  241. <goal>single</goal>
  242. </goals>
  243. </execution>
  244. </executions>
  245. </plugin>
  246. </plugins>
  247. </build>
  248. <dependencyManagement>
  249. <dependencies>
  250. <dependency>
  251. <groupId>org.apache.samza</groupId>
  252. <artifactId>samza-api</artifactId>
  253. <version>${samza.version}</version>
  254. </dependency>
  255. <dependency>
  256. <groupId>org.apache.samza</groupId>
  257. <artifactId>samza-core_2.10</artifactId>
  258. <version>${samza.version}</version>
  259. </dependency>
  260. <dependency>
  261. <groupId>org.apache.samza</groupId>
  262. <artifactId>samza-log4j</artifactId>
  263. <version>${samza.version}</version>
  264. </dependency>
  265. <dependency>
  266. <groupId>org.apache.samza</groupId>
  267. <artifactId>samza-shell</artifactId>
  268. <version>${samza.version}</version>
  269. </dependency>
  270. <dependency>
  271. <groupId>org.apache.samza</groupId>
  272. <artifactId>samza-yarn_2.10</artifactId>
  273. <version>${samza.version}</version>
  274. </dependency>
  275. <dependency>
  276. <groupId>org.apache.samza</groupId>
  277. <artifactId>samza-kv_2.10</artifactId>
  278. <version>${samza.version}</version>
  279. </dependency>
  280. <dependency>
  281. <groupId>org.apache.samza</groupId>
  282. <artifactId>samza-kv-rocksdb_2.10</artifactId>
  283. <version>${samza.version}</version>
  284. </dependency>
  285. <dependency>
  286. <groupId>org.apache.samza</groupId>
  287. <artifactId>samza-kafka_2.10</artifactId>
  288. <version>${samza.version}</version>
  289. </dependency>
  290. <dependency>
  291. <groupId>org.apache.kafka</groupId>
  292. <artifactId>kafka_2.10</artifactId>
  293. <version>0.8.2.0</version>
  294. </dependency>
  295. <dependency>
  296. <groupId>org.schwering</groupId>
  297. <artifactId>irclib</artifactId>
  298. <version>1.10</version>
  299. </dependency>
  300. <dependency>
  301. <groupId>org.slf4j</groupId>
  302. <artifactId>slf4j-api</artifactId>
  303. <version>1.6.2</version>
  304. </dependency>
  305. <dependency>
  306. <groupId>org.slf4j</groupId>
  307. <artifactId>slf4j-log4j12</artifactId>
  308. <version>1.6.2</version>
  309. </dependency>
  310. <dependency>
  311. <groupId>org.codehaus.jackson</groupId>
  312. <artifactId>jackson-jaxrs</artifactId>
  313. <version>1.8.5</version>
  314. </dependency>
  315. <dependency>
  316. <groupId>org.apache.hadoop</groupId>
  317. <artifactId>hadoop-hdfs</artifactId>
  318. <version>2.6.0</version>
  319. </dependency>
  320. </dependencies>
  321. </dependencyManagement>
  322. </project>
  323.  
  324. src/assembly/bin.xml
  325.  
  326. <assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
  327. xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  328. xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
  329. <id>bin</id>
  330. <formats>
  331. <format>tar.gz</format>
  332. <format>zip</format>
  333. </formats>
  334. <fileSets>
  335. <fileSet>
  336. <directory>${project.basedir}</directory>
  337. <outputDirectory>/</outputDirectory>
  338. <includes>
  339. <include>README*</include>
  340. <include>LICENSE*</include>
  341. <include>NOTICE*</include>
  342. </includes>
  343. </fileSet>
  344. <fileSet>
  345. <directory>${project.build.directory}</directory>
  346. <outputDirectory>/</outputDirectory>
  347. <includes>
  348. <include>*.jar</include>
  349. </includes>
  350. </fileSet>
  351. <fileSet>
  352. <directory>${project.build.directory}/site</directory>
  353. <outputDirectory>docs</outputDirectory>
  354. </fileSet>
  355. </fileSets>
  356. </assembly>
  357.  
  358. Export as a .jar with dependencies.
  359.  
  360. Runnign a job
  361.  
  362. mkdir /opt/jobs
  363. cd /opt/jobs
  364. mkdir bin
  365. mkdir lib
  366. cp -P /opt/samza/samza-shell/src/main/bash/* bin/
  367.  
  368. cd /opt/jobs
  369. tar -czvf samzajob1.tar.gz bin lib
  370. mv samzajob1.tar.gz /var/www/html/jobs/
  371.  
  372.  
  373.  
  374. vi job1.properties
  375. task.class=samzafroga.job1
  376. job.name=samzafroga.job1
  377. job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
  378. yarn.package.path=http://<webIP>/jobs/samzajob1.tar.gz
  379.  
  380. systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
  381. systems.kafka.consumer.zookeeper.connect=kfk-kafka01:2181,kfk-kafka02:2181
  382. systems.kafka.producer.bootstrap.servers=kfk-kafka01:9092,kfk-kafka01:9093,kfk-kafka02:9092,kfk-kafka02:9093
  383. systems.kafka.producer.metadata.broker.list=kfk-kafka01:9092,kfk-kafka01:9093,kfk-kafka02:9092,kfk-kafka02:909
  384.  
  385. task.inputs=kafka.syslog
  386.  
  387. serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
  388. serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
  389.  
  390. systems.kafka.streams.syslog.samza.msg.serde=string
  391. systems.kafka.streams.samzaout.samza.msg.serde=string
  392.  
  393.  
  394. bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file://$PWD/ job1.properties
  395.  
  396.  
  397. Job will be seen in http://<samza01IP>:8088
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!