API tools faq
paste
Guest User

Untitled

a guest
Apr 13th, 2018
97
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 14.73 KB | None | 0 0
raw download clone embed print report
  1. {
  2. "AWSTemplateFormatVersion": "2010-09-09",
  3. "Description": "EMR Cluster for sqoop job",
  4. "Parameters": {
  5. "EMRClusterName": {
  6. "Description": "Name of the cluster",
  7. "Type": "String",
  8. "Default": "emrcluster"
  9. },
  10. "KeyName": {
  11. "Description": "Must be an existing Keyname",
  12. "Type": "String",
  13. "Default": "p-3"
  14. },
  15. "MasterInstacneType": {
  16. "Description": "Instance type to be used for the master instance.",
  17. "Type": "String",
  18. "Default": "m1.medium"
  19. },
  20. "CoreInstanceType": {
  21. "Description": "Instance type to be used for core instances.",
  22. "Type": "String",
  23. "Default": "m1.medium"
  24. },
  25. "NumberOfCoreInstances": {
  26. "Description": "Must be a valid number",
  27. "Type": "Number",
  28. "Default": 1
  29. },
  30. "SubnetID": {
  31. "Description": "Must be Valid public subnet ID",
  32. "Default": "subnet-2ee08274",
  33. "Type": "String"
  34. },
  35. "LogUri": {
  36. "Description": "Must be a valid S3 URL",
  37. "Default": "s3://aws-logs-552968783313-us-east-1/elasticmapreduce/",
  38. "Type": "String"
  39. },
  40. "ReleaseLabel": {
  41. "Description": "Must be a valid EMR release version",
  42. "Default": "emr-5.12.1",
  43. "Type": "String"
  44. }
  45. },
  46. "Mappings": {},
  47. "Conditions": {},
  48. "Resources": {
  49. "EMRCluster": {
  50. "DependsOn": [
  51. "EMRClusterServiceRole",
  52. "EMRClusterinstanceProfileRole",
  53. "EMRClusterinstanceProfile"
  54. ],
  55. "Type": "AWS::EMR::Cluster",
  56. "Properties": {
  57. "Applications": [
  58. {
  59. "Name": "Sqoop"
  60. }
  61. ],
  62. "Configurations": [],
  63. "Instances": {
  64. "Ec2KeyName": {
  65. "Ref": "KeyName"
  66. },
  67. "Ec2SubnetId": {
  68. "Ref": "SubnetID"
  69. },
  70. "MasterInstanceGroup": {
  71. "InstanceCount": 1,
  72. "InstanceType": {
  73. "Ref": "MasterInstacneType"
  74. },
  75. "Market": "ON_DEMAND",
  76. "Name": "Master"
  77. },
  78. "CoreInstanceGroup": {
  79. "InstanceCount": {
  80. "Ref": "NumberOfCoreInstances"
  81. },
  82. "InstanceType": {
  83. "Ref": "CoreInstanceType"
  84. },
  85. "Market": "ON_DEMAND",
  86. "Name": "Core",
  87. "AutoScalingPolicy": {
  88. "Constraints": {
  89. "MinCapacity": 1,
  90. "MaxCapacity": 3
  91. },
  92. "Rules": [
  93. {
  94. "Action": {
  95. "SimpleScalingPolicyConfiguration": {
  96. "ScalingAdjustment": 1,
  97. "CoolDown": 300,
  98. "AdjustmentType": "CHANGE_IN_CAPACITY"
  99. }
  100. },
  101. "Description": "",
  102. "Trigger": {
  103. "CloudWatchAlarmDefinition": {
  104. "MetricName": "YARNMemoryAvailablePercentage",
  105. "ComparisonOperator": "LESS_THAN",
  106. "Statistic": "AVERAGE",
  107. "Period": 300,
  108. "EvaluationPeriods": 1,
  109. "Unit": "PERCENT",
  110. "Namespace": "AWS/ElasticMapReduce",
  111. "Threshold": 15,
  112. "Dimensions": [
  113. {
  114. "Value": "${emr.clusterId}",
  115. "Key": "JobFlowId"
  116. }
  117. ]
  118. }
  119. },
  120. "Name": "Default-scale-out-1"
  121. },
  122. {
  123. "Action": {
  124. "SimpleScalingPolicyConfiguration": {
  125. "ScalingAdjustment": 1,
  126. "CoolDown": 300,
  127. "AdjustmentType": "CHANGE_IN_CAPACITY"
  128. }
  129. },
  130. "Description": "",
  131. "Trigger": {
  132. "CloudWatchAlarmDefinition": {
  133. "MetricName": "ContainerPendingRatio",
  134. "ComparisonOperator": "GREATER_THAN",
  135. "Statistic": "AVERAGE",
  136. "Period": 300,
  137. "EvaluationPeriods": 1,
  138. "Unit": "COUNT",
  139. "Namespace": "AWS/ElasticMapReduce",
  140. "Threshold": 0.75,
  141. "Dimensions": [
  142. {
  143. "Value": "${emr.clusterId}",
  144. "Key": "JobFlowId"
  145. }
  146. ]
  147. }
  148. },
  149. "Name": "Default-scale-out-2"
  150. },
  151. {
  152. "Action": {
  153. "SimpleScalingPolicyConfiguration": {
  154. "ScalingAdjustment": -1,
  155. "CoolDown": 300,
  156. "AdjustmentType": "CHANGE_IN_CAPACITY"
  157. }
  158. },
  159. "Description": "",
  160. "Trigger": {
  161. "CloudWatchAlarmDefinition": {
  162. "MetricName": "YARNMemoryAvailablePercentage",
  163. "ComparisonOperator": "GREATER_THAN",
  164. "Statistic": "AVERAGE",
  165. "Period": 300,
  166. "EvaluationPeriods": 1,
  167. "Unit": "PERCENT",
  168. "Namespace": "AWS/ElasticMapReduce",
  169. "Threshold": 75,
  170. "Dimensions": [
  171. {
  172. "Value": "${emr.clusterId}",
  173. "Key": "JobFlowId"
  174. }
  175. ]
  176. }
  177. },
  178. "Name": "Default-scale-in"
  179. }
  180. ]
  181. }
  182. },
  183. "TerminationProtected": false
  184. },
  185. "VisibleToAllUsers": true,
  186. "JobFlowRole": {
  187. "Ref": "EMRClusterinstanceProfile"
  188. },
  189. "ReleaseLabel": {
  190. "Ref": "ReleaseLabel"
  191. },
  192. "LogUri": {
  193. "Ref": "LogUri"
  194. },
  195. "Name": {
  196. "Ref": "EMRClusterName"
  197. },
  198. "AutoScalingRole": "EMR_AutoScaling_DefaultRole",
  199. "ServiceRole": {
  200. "Ref": "EMRClusterServiceRole"
  201. }
  202. },
  203. "Metadata": {
  204. "AWS::CloudFormation::Designer": {
  205. "id": "9752a730-1233-4688-bbfa-0d372d8b4b7b"
  206. }
  207. }
  208. },
  209. "EMRClusterServiceRole": {
  210. "Type": "AWS::IAM::Role",
  211. "Properties": {
  212. "AssumeRolePolicyDocument": {
  213. "Version": "2012-10-17",
  214. "Statement": [
  215. {
  216. "Effect": "Allow",
  217. "Principal": {
  218. "Service": [
  219. "elasticmapreduce.amazonaws.com"
  220. ]
  221. },
  222. "Action": [
  223. "sts:AssumeRole"
  224. ]
  225. }
  226. ]
  227. },
  228. "ManagedPolicyArns": [
  229. "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole"
  230. ],
  231. "Path": "/"
  232. },
  233. "Metadata": {
  234. "AWS::CloudFormation::Designer": {
  235. "id": "e5d7db8d-9b40-4c5c-bf75-052f7a77080f"
  236. }
  237. }
  238. },
  239. "EMRClusterinstanceProfileRole": {
  240. "Type": "AWS::IAM::Role",
  241. "Properties": {
  242. "AssumeRolePolicyDocument": {
  243. "Version": "2012-10-17",
  244. "Statement": [
  245. {
  246. "Effect": "Allow",
  247. "Principal": {
  248. "Service": [
  249. "ec2.amazonaws.com"
  250. ]
  251. },
  252. "Action": [
  253. "sts:AssumeRole"
  254. ]
  255. }
  256. ]
  257. },
  258. "ManagedPolicyArns": [
  259. "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role"
  260. ],
  261. "Path": "/"
  262. },
  263. "Metadata": {
  264. "AWS::CloudFormation::Designer": {
  265. "id": "b5c0eafa-3a18-4c65-a36f-3f0906c147c3"
  266. }
  267. }
  268. },
  269. "EMRClusterinstanceProfile": {
  270. "Type": "AWS::IAM::InstanceProfile",
  271. "Properties": {
  272. "Path": "/",
  273. "Roles": [
  274. {
  275. "Ref": "EMRClusterinstanceProfileRole"
  276. }
  277. ]
  278. },
  279. "Metadata": {
  280. "AWS::CloudFormation::Designer": {
  281. "id": "87c2277a-7e40-4492-b3f6-24883ca5988e"
  282. }
  283. }
  284. },
  285. "EMRS4SIT2": {
  286. "Type": "AWS::EMR::Step",
  287. "Properties": {
  288. "ActionOnFailure": "CONTINUE",
  289. "HadoopJarStep": {
  290. "Args": [
  291. "sqoop",
  292. "import, --driver, org.postgresql.Driver, --connect, jdbc:postgresql://pipeline.fhuwis2ii.us-east-1.rds.amazonaws.com/pipeline, --username, pipeline, --password, pipe_line, --table, actor, --target-dir, s3://pridesys-sqoop/tables/actor, --compress"
  293. ],
  294. "Jar": "command-runner.jar"
  295. },
  296. "Name": "table1Step",
  297. "JobFlowId": {
  298. "Ref": "EMRCluster"
  299. }
  300. },
  301. "Metadata": {
  302. "AWS::CloudFormation::Designer": {
  303. "id": "65a4c828-b7c6-424b-950b-d68c6d92e382"
  304. }
  305. }
  306. }
  307. },
  308. "Outputs": {},
  309. "Metadata": {
  310. "AWS::CloudFormation::Designer": {
  311. "b5c0eafa-3a18-4c65-a36f-3f0906c147c3": {
  312. "size": {
  313. "width": 60,
  314. "height": 60
  315. },
  316. "position": {
  317. "x": 60,
  318. "y": 90
  319. },
  320. "z": 1,
  321. "embeds": []
  322. },
  323. "87c2277a-7e40-4492-b3f6-24883ca5988e": {
  324. "size": {
  325. "width": 60,
  326. "height": 60
  327. },
  328. "position": {
  329. "x": 180,
  330. "y": 90
  331. },
  332. "z": 1,
  333. "embeds": [],
  334. "isassociatedwith": [
  335. "b5c0eafa-3a18-4c65-a36f-3f0906c147c3"
  336. ]
  337. },
  338. "e5d7db8d-9b40-4c5c-bf75-052f7a77080f": {
  339. "size": {
  340. "width": 60,
  341. "height": 60
  342. },
  343. "position": {
  344. "x": 60,
  345. "y": 210
  346. },
  347. "z": 1,
  348. "embeds": []
  349. },
  350. "9752a730-1233-4688-bbfa-0d372d8b4b7b": {
  351. "size": {
  352. "width": 60,
  353. "height": 60
  354. },
  355. "position": {
  356. "x": 180,
  357. "y": 210
  358. },
  359. "z": 1,
  360. "embeds": [],
  361. "dependson": [
  362. "b5c0eafa-3a18-4c65-a36f-3f0906c147c3"
  363. ]
  364. },
  365. "65a4c828-b7c6-424b-950b-d68c6d92e382": {
  366. "size": {
  367. "width": 60,
  368. "height": 60
  369. },
  370. "position": {
  371. "x": -60.83920716362772,
  372. "y": 189.01408632115496
  373. },
  374. "z": 0,
  375. "embeds": [],
  376. "isassociatedwith": [
  377. "9752a730-1233-4688-bbfa-0d372d8b4b7b"
  378. ]
  379. }
  380. }
  381. }
  382. }
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!